| @@ -25,16 +25,899 @@ · 24 unmodified lines |
| 25 | */ | 25 | */ |
| 26 | | 26 | |
| 27 | #include <linux/anon_inodes.h> | 27 | #include <linux/anon_inodes.h> |
| | 28 | #include <linux/sizes.h> |
| 28 | | 29 | |
| 29 | #include "i915_drv.h" | 30 | #include "i915_drv.h" |
| | 31 | #include "i915_oa_hsw.h" |
| | 32 | |
| | 33 | /* HW requires this to be a power of two, between 128k and 16M, though driver |
| | 34 | * is currently generally designed assuming the largest 16M size is used such |
| | 35 | * that the overflow cases are unlikely in normal operation. |
| | 36 | */ |
| | 37 | #define OA_BUFFER_SIZE SZ_16M |
| | 38 | |
| | 39 | #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) |
| | 40 | |
| | 41 | /* There's a HW race condition between OA unit tail pointer register updates and |
| | 42 | * writes to memory whereby the tail pointer can sometimes get ahead of what's |
| | 43 | * been written out to the OA buffer so far. |
| | 44 | * |
| | 45 | * Although this can be observed explicitly by checking for a zeroed report-id |
| | 46 | * field in tail reports, it seems preferable to account for this earlier e.g. |
| | 47 | * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles |
| | 48 | * in this situation. |
| | 49 | * |
| | 50 | * To give time for the most recent reports to land before they may be copied to |
| | 51 | * userspace, the driver operates as if the tail pointer effectively lags behind |
| | 52 | * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated |
| | 53 | * based on this constant in nanoseconds, the current OA sampling exponent |
| | 54 | * and current report size. |
| | 55 | * |
| | 56 | * There is also a fallback check while reading to simply skip over reports with |
| | 57 | * a zeroed report-id. |
| | 58 | */ |
| | 59 | #define OA_TAIL_MARGIN_NSEC 100000ULL |
| | 60 | |
| | 61 | /* frequency for checking whether the OA unit has written new reports to the |
| | 62 | * circular OA buffer... |
| | 63 | */ |
| | 64 | #define POLL_FREQUENCY 200 |
| | 65 | #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) |
| | 66 | |
| | 67 | /* The maximum exponent the hardware accepts is 63 (essentially it selects one |
| | 68 | * of the 64bit timestamp bits to trigger reports from) but there's currently |
| | 69 | * no known use case for sampling as infrequently as once per 47 thousand years. |
| | 70 | * |
| | 71 | * Since the timestamps included in OA reports are only 32bits it seems |
| | 72 | * reasonable to limit the OA exponent where it's still possible to account for |
| | 73 | * overflow in OA report timestamps. |
| | 74 | */ |
| | 75 | #define OA_EXPONENT_MAX 31 |
| | 76 | |
| | 77 | #define INVALID_CTX_ID 0xffffffff |
| | 78 | |
| | 79 | |
| | 80 | /* XXX: beware if future OA HW adds new report formats that the current |
| | 81 | * code assumes all reports have a power-of-two size and ~(size - 1) can |
| | 82 | * be used as a mask to align the OA tail pointer. |
| | 83 | */ |
| | 84 | static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { |
| | 85 | [I915_OA_FORMAT_A13] = { 0, 64 }, |
| | 86 | [I915_OA_FORMAT_A29] = { 1, 128 }, |
| | 87 | [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, |
| | 88 | /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ |
| | 89 | [I915_OA_FORMAT_B4_C8] = { 4, 64 }, |
| | 90 | [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, |
| | 91 | [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, |
| | 92 | [I915_OA_FORMAT_C4_B8] = { 7, 64 }, |
| | 93 | }; |
| | 94 | |
| | 95 | #define SAMPLE_OA_REPORT (1<<0) |
| 30 | | 96 | |
| 31 | struct perf_open_properties { | 97 | struct perf_open_properties { |
| 32 | u32 sample_flags; | 98 | u32 sample_flags; |
| 33 | | 99 | |
| 34 | u64 single_context:1; | 100 | u64 single_context:1; |
| 35 | u64 ctx_handle; | 101 | u64 ctx_handle; |
| | 102 | |
| | 103 | /* OA sampling state */ |
| | 104 | int metrics_set; |
| | 105 | int oa_format; |
| | 106 | bool oa_periodic; |
| | 107 | int oa_period_exponent; |
| | 108 | }; |
| | 109 | |
| | 110 | /* NB: This is either called via fops or the poll check hrtimer (atomic ctx) |
| | 111 | * |
| | 112 | * It's safe to read OA config state here unlocked, assuming that this is only |
| | 113 | * called while the stream is enabled, while the global OA configuration can't |
| | 114 | * be modified. |
| | 115 | * |
| | 116 | * Note: we don't lock around the head/tail reads even though there's the slim |
| | 117 | * possibility of read() fop errors forcing a re-init of the OA buffer |
| | 118 | * pointers. A race here could result in a false positive !empty status which |
| | 119 | * is acceptable. |
| | 120 | */ |
| | 121 | static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv) |
| | 122 | { |
| | 123 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| | 124 | u32 oastatus2 = I915_READ(GEN7_OASTATUS2); |
| | 125 | u32 oastatus1 = I915_READ(GEN7_OASTATUS1); |
| | 126 | u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| | 127 | u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| | 128 | |
| | 129 | return OA_TAKEN(tail, head) < |
| | 130 | dev_priv->perf.oa.tail_margin + report_size; |
| | 131 | } |
| | 132 | |
| | 133 | /** |
| | 134 | * Appends a status record to a userspace read() buffer. |
| | 135 | */ |
| | 136 | static int append_oa_status(struct i915_perf_stream *stream, |
| | 137 | char __user *buf, |
| | 138 | size_t count, |
| | 139 | size_t *offset, |
| | 140 | enum drm_i915_perf_record_type type) |
| | 141 | { |
| | 142 | struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; |
| | 143 | |
| | 144 | if ((count - *offset) < header.size) |
| | 145 | return -ENOSPC; |
| | 146 | |
| | 147 | if (copy_to_user(buf + *offset, &header, sizeof(header))) |
| | 148 | return -EFAULT; |
| | 149 | |
| | 150 | (*offset) += header.size; |
| | 151 | |
| | 152 | return 0; |
| | 153 | } |
| | 154 | |
| | 155 | /** |
| | 156 | * Copies single OA report into userspace read() buffer. |
| | 157 | */ |
| | 158 | static int append_oa_sample(struct i915_perf_stream *stream, |
| | 159 | char __user *buf, |
| | 160 | size_t count, |
| | 161 | size_t *offset, |
| | 162 | const u8 *report) |
| | 163 | { |
| | 164 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 165 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| | 166 | struct drm_i915_perf_record_header header; |
| | 167 | u32 sample_flags = stream->sample_flags; |
| | 168 | |
| | 169 | header.type = DRM_I915_PERF_RECORD_SAMPLE; |
| | 170 | header.pad = 0; |
| | 171 | header.size = stream->sample_size; |
| | 172 | |
| | 173 | if ((count - *offset) < header.size) |
| | 174 | return -ENOSPC; |
| | 175 | |
| | 176 | buf += *offset; |
| | 177 | if (copy_to_user(buf, &header, sizeof(header))) |
| | 178 | return -EFAULT; |
| | 179 | buf += sizeof(header); |
| | 180 | |
| | 181 | if (sample_flags & SAMPLE_OA_REPORT) { |
| | 182 | if (copy_to_user(buf, report, report_size)) |
| | 183 | return -EFAULT; |
| | 184 | } |
| | 185 | |
| | 186 | (*offset) += header.size; |
| | 187 | |
| | 188 | return 0; |
| | 189 | } |
| | 190 | |
| | 191 | /** |
| | 192 | * Copies all buffered OA reports into userspace read() buffer. |
| | 193 | * @stream: An i915-perf stream opened for OA metrics |
| | 194 | * @buf: destination buffer given by userspace |
| | 195 | * @count: the number of bytes userspace wants to read |
| | 196 | * @offset: (inout): the current position for writing into @buf |
| | 197 | * @head_ptr: (inout): the current oa buffer cpu read position |
| | 198 | * @tail: the current oa buffer gpu write position |
| | 199 | * |
| | 200 | * Returns 0 on success, negative error code on failure. |
| | 201 | * |
| | 202 | * Notably any error condition resulting in a short read (-ENOSPC or |
| | 203 | * -EFAULT) will be returned even though one or more records may |
| | 204 | * have been successfully copied. In this case it's up to the caller |
| | 205 | * to decide if the error should be squashed before returning to |
| | 206 | * userspace. |
| | 207 | * |
| | 208 | * Note: reports are consumed from the head, and appended to the |
| | 209 | * tail, so the head chases the tail?... If you think that's mad |
| | 210 | * and back-to-front you're not alone, but this follows the |
| | 211 | * Gen PRM naming convention. |
| | 212 | */ |
| | 213 | static int gen7_append_oa_reports(struct i915_perf_stream *stream, |
| | 214 | char __user *buf, |
| | 215 | size_t count, |
| | 216 | size_t *offset, |
| | 217 | u32 *head_ptr, |
| | 218 | u32 tail) |
| | 219 | { |
| | 220 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 221 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| | 222 | u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; |
| | 223 | int tail_margin = dev_priv->perf.oa.tail_margin; |
| | 224 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); |
| | 225 | u32 mask = (OA_BUFFER_SIZE - 1); |
| | 226 | u32 head; |
| | 227 | u32 taken; |
| | 228 | int ret = 0; |
| | 229 | |
| | 230 | if (WARN_ON(!stream->enabled)) |
| | 231 | return -EIO; |
| | 232 | |
| | 233 | head = *head_ptr - gtt_offset; |
| | 234 | tail -= gtt_offset; |
| | 235 | |
| | 236 | /* The OA unit is expected to wrap the tail pointer according to the OA |
| | 237 | * buffer size and since we should never write a misaligned head |
| | 238 | * pointer we don't expect to read one back either... |
| | 239 | */ |
| | 240 | if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE || |
| | 241 | head % report_size) { |
| | 242 | DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n", |
| | 243 | head, tail); |
| | 244 | dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| | 245 | dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| | 246 | *head_ptr = I915_READ(GEN7_OASTATUS2) & |
| | 247 | GEN7_OASTATUS2_HEAD_MASK; |
| | 248 | return -EIO; |
| | 249 | } |
| | 250 | |
| | 251 | |
| | 252 | /* The tail pointer increases in 64 byte increments, not in report_size |
| | 253 | * steps... |
| | 254 | */ |
| | 255 | tail &= ~(report_size - 1); |
| | 256 | |
| | 257 | /* Move the tail pointer back by the current tail_margin to account for |
| | 258 | * the possibility that the latest reports may not have really landed |
| | 259 | * in memory yet... |
| | 260 | */ |
| | 261 | |
| | 262 | if (OA_TAKEN(tail, head) < report_size + tail_margin) |
| | 263 | return -EAGAIN; |
| | 264 | |
| | 265 | tail -= tail_margin; |
| | 266 | tail &= mask; |
| | 267 | |
| | 268 | for (/* none */; |
| | 269 | (taken = OA_TAKEN(tail, head)); |
| | 270 | head = (head + report_size) & mask) { |
| | 271 | u8 *report = oa_buf_base + head; |
| | 272 | u32 *report32 = (void *)report; |
| | 273 | |
| | 274 | /* All the report sizes factor neatly into the buffer |
| | 275 | * size so we never expect to see a report split |
| | 276 | * between the beginning and end of the buffer. |
| | 277 | * |
| | 278 | * Given the initial alignment check a misalignment |
| | 279 | * here would imply a driver bug that would result |
| | 280 | * in an overrun. |
| | 281 | */ |
| | 282 | if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { |
| | 283 | DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); |
| | 284 | break; |
| | 285 | } |
| | 286 | |
| | 287 | /* The report-ID field for periodic samples includes |
| | 288 | * some undocumented flags related to what triggered |
| | 289 | * the report and is never expected to be zero so we |
| | 290 | * can check that the report isn't invalid before |
| | 291 | * copying it to userspace... |
| | 292 | */ |
| | 293 | if (report32[0] == 0) { |
| | 294 | DRM_ERROR("Skipping spurious, invalid OA report\n"); |
| | 295 | continue; |
| | 296 | } |
| | 297 | |
| | 298 | ret = append_oa_sample(stream, buf, count, offset, report); |
| | 299 | if (ret) |
| | 300 | break; |
| | 301 | |
| | 302 | /* The above report-id field sanity check is based on |
| | 303 | * the assumption that the OA buffer is initially |
| | 304 | * zeroed and we reset the field after copying so the |
| | 305 | * check is still meaningful once old reports start |
| | 306 | * being overwritten. |
| | 307 | */ |
| | 308 | report32[0] = 0; |
| | 309 | } |
| | 310 | |
| | 311 | *head_ptr = gtt_offset + head; |
| | 312 | |
| | 313 | return ret; |
| | 314 | } |
| | 315 | |
| | 316 | static int gen7_oa_read(struct i915_perf_stream *stream, |
| | 317 | char __user *buf, |
| | 318 | size_t count, |
| | 319 | size_t *offset) |
| | 320 | { |
| | 321 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 322 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| | 323 | u32 oastatus2; |
| | 324 | u32 oastatus1; |
| | 325 | u32 head; |
| | 326 | u32 tail; |
| | 327 | int ret; |
| | 328 | |
| | 329 | if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) |
| | 330 | return -EIO; |
| | 331 | |
| | 332 | oastatus2 = I915_READ(GEN7_OASTATUS2); |
| | 333 | oastatus1 = I915_READ(GEN7_OASTATUS1); |
| | 334 | |
| | 335 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| | 336 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| | 337 | |
| | 338 | /* XXX: On Haswell we don't have a safe way to clear oastatus1 |
| | 339 | * bits while the OA unit is enabled (while the tail pointer |
| | 340 | * may be updated asynchronously) so we ignore status bits |
| | 341 | * that have already been reported to userspace. |
| | 342 | */ |
| | 343 | oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; |
| | 344 | |
| | 345 | /* We treat OABUFFER_OVERFLOW as a significant error: |
| | 346 | * |
| | 347 | * - The status can be interpreted to mean that the buffer is |
| | 348 | * currently full (with a higher precedence than OA_TAKEN() |
| | 349 | * which will start to report a near-empty buffer after an |
| | 350 | * overflow) but it's awkward that we can't clear the status |
| | 351 | * on Haswell, so without a reset we won't be able to catch |
| | 352 | * the state again. |
| | 353 | * |
| | 354 | * - Since it also implies the HW has started overwriting old |
| | 355 | * reports it may also affect our sanity checks for invalid |
| | 356 | * reports when copying to userspace that assume new reports |
| | 357 | * are being written to cleared memory. |
| | 358 | * |
| | 359 | * - In the future we may want to introduce a flight recorder |
| | 360 | * mode where the driver will automatically maintain a safe |
| | 361 | * guard band between head/tail, avoiding this overflow |
| | 362 | * condition, but we avoid the added driver complexity for |
| | 363 | * now. |
| | 364 | */ |
| | 365 | if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { |
| | 366 | ret = append_oa_status(stream, buf, count, offset, |
| | 367 | DRM_I915_PERF_RECORD_OA_BUFFER_LOST); |
| | 368 | if (ret) |
| | 369 | return ret; |
| | 370 | |
| | 371 | DRM_ERROR("OA buffer overflow: force restart\n"); |
| | 372 | |
| | 373 | dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| | 374 | dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| | 375 | |
| | 376 | oastatus2 = I915_READ(GEN7_OASTATUS2); |
| | 377 | oastatus1 = I915_READ(GEN7_OASTATUS1); |
| | 378 | |
| | 379 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| | 380 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| | 381 | } |
| | 382 | |
| | 383 | if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { |
| | 384 | ret = append_oa_status(stream, buf, count, offset, |
| | 385 | DRM_I915_PERF_RECORD_OA_REPORT_LOST); |
| | 386 | if (ret) |
| | 387 | return ret; |
| | 388 | dev_priv->perf.oa.gen7_latched_oastatus1 |= |
| | 389 | GEN7_OASTATUS1_REPORT_LOST; |
| | 390 | } |
| | 391 | |
| | 392 | ret = gen7_append_oa_reports(stream, buf, count, offset, |
| | 393 | &head, tail); |
| | 394 | |
| | 395 | /* All the report sizes are a power of two and the |
| | 396 | * head should always be incremented by some multiple |
| | 397 | * of the report size. |
| | 398 | * |
| | 399 | * A warning here, but notably if we later read back a |
| | 400 | * misaligned pointer we will treat that as a bug since |
| | 401 | * it could lead to a buffer overrun. |
| | 402 | */ |
| | 403 | WARN_ONCE(head & (report_size - 1), |
| | 404 | "i915: Writing misaligned OA head pointer"); |
| | 405 | |
| | 406 | /* Note: we update the head pointer here even if an error |
| | 407 | * was returned since the error may represent a short read |
| | 408 | * where some some reports were successfully copied. |
| | 409 | */ |
| | 410 | I915_WRITE(GEN7_OASTATUS2, |
| | 411 | ((head & GEN7_OASTATUS2_HEAD_MASK) | |
| | 412 | OA_MEM_SELECT_GGTT)); |
| | 413 | |
| | 414 | return ret; |
| | 415 | } |
| | 416 | |
| | 417 | static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) |
| | 418 | { |
| | 419 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 420 | |
| | 421 | /* We would wait indefinitely if periodic sampling is not enabled */ |
| | 422 | if (!dev_priv->perf.oa.periodic) |
| | 423 | return -EIO; |
| | 424 | |
| | 425 | /* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it |
| | 426 | * just performs mmio reads of the OA buffer head + tail pointers and |
| | 427 | * it's assumed we're handling some operation that implies the stream |
| | 428 | * can't be destroyed until completion (such as a read()) that ensures |
| | 429 | * the device + OA buffer can't disappear |
| | 430 | */ |
| | 431 | return wait_event_interruptible(dev_priv->perf.oa.poll_wq, |
| | 432 | !dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)); |
| | 433 | } |
| | 434 | |
| | 435 | static void i915_oa_poll_wait(struct i915_perf_stream *stream, |
| | 436 | struct file *file, |
| | 437 | poll_table *wait) |
| | 438 | { |
| | 439 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 440 | |
| | 441 | poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); |
| | 442 | } |
| | 443 | |
| | 444 | static int i915_oa_read(struct i915_perf_stream *stream, |
| | 445 | char __user *buf, |
| | 446 | size_t count, |
| | 447 | size_t *offset) |
| | 448 | { |
| | 449 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 450 | |
| | 451 | return dev_priv->perf.oa.ops.read(stream, buf, count, offset); |
| | 452 | } |
| | 453 | |
| | 454 | /* Determine the render context hw id, and ensure it remains fixed for the |
| | 455 | * lifetime of the stream. This ensures that we don't have to worry about |
| | 456 | * updating the context ID in OACONTROL on the fly. |
| | 457 | */ |
| | 458 | static int oa_get_render_ctx_id(struct i915_perf_stream *stream) |
| | 459 | { |
| | 460 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 461 | struct i915_vma *vma; |
| | 462 | int ret; |
| | 463 | |
| | 464 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); |
| | 465 | if (ret) |
| | 466 | return ret; |
| | 467 | |
| | 468 | /* As the ID is the gtt offset of the context's vma we pin |
| | 469 | * the vma to ensure the ID remains fixed. |
| | 470 | * |
| | 471 | * NB: implied RCS engine... |
| | 472 | */ |
| | 473 | vma = i915_gem_context_pin_legacy(stream->ctx, 0); |
| | 474 | if (IS_ERR(vma)) { |
| | 475 | ret = PTR_ERR(vma); |
| | 476 | goto unlock; |
| | 477 | } |
| | 478 | |
| | 479 | dev_priv->perf.oa.pinned_rcs_vma = vma; |
| | 480 | |
| | 481 | /* Explicitly track the ID (instead of calling i915_ggtt_offset() |
| | 482 | * on the fly) considering the difference with gen8+ and |
| | 483 | * execlists |
| | 484 | */ |
| | 485 | dev_priv->perf.oa.specific_ctx_id = i915_ggtt_offset(vma); |
| | 486 | |
| | 487 | unlock: |
| | 488 | mutex_unlock(&dev_priv->drm.struct_mutex); |
| | 489 | |
| | 490 | return ret; |
| | 491 | } |
| | 492 | |
| | 493 | static void oa_put_render_ctx_id(struct i915_perf_stream *stream) |
| | 494 | { |
| | 495 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 496 | |
| | 497 | mutex_lock(&dev_priv->drm.struct_mutex); |
| | 498 | |
| | 499 | i915_vma_unpin(dev_priv->perf.oa.pinned_rcs_vma); |
| | 500 | dev_priv->perf.oa.pinned_rcs_vma = NULL; |
| | 501 | |
| | 502 | dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; |
| | 503 | |
| | 504 | mutex_unlock(&dev_priv->drm.struct_mutex); |
| | 505 | } |
| | 506 | |
| | 507 | static void |
| | 508 | free_oa_buffer(struct drm_i915_private *i915) |
| | 509 | { |
| | 510 | mutex_lock(&i915->drm.struct_mutex); |
| | 511 | |
| | 512 | i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); |
| | 513 | i915_vma_unpin(i915->perf.oa.oa_buffer.vma); |
| | 514 | i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); |
| | 515 | |
| | 516 | i915->perf.oa.oa_buffer.vma = NULL; |
| | 517 | i915->perf.oa.oa_buffer.vaddr = NULL; |
| | 518 | |
| | 519 | mutex_unlock(&i915->drm.struct_mutex); |
| | 520 | } |
| | 521 | |
| | 522 | static void i915_oa_stream_destroy(struct i915_perf_stream *stream) |
| | 523 | { |
| | 524 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 525 | |
| | 526 | BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); |
| | 527 | |
| | 528 | dev_priv->perf.oa.ops.disable_metric_set(dev_priv); |
| | 529 | |
| | 530 | free_oa_buffer(dev_priv); |
| | 531 | |
| | 532 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); |
| | 533 | intel_runtime_pm_put(dev_priv); |
| | 534 | |
| | 535 | if (stream->ctx) |
| | 536 | oa_put_render_ctx_id(stream); |
| | 537 | |
| | 538 | dev_priv->perf.oa.exclusive_stream = NULL; |
| | 539 | } |
| | 540 | |
| | 541 | static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) |
| | 542 | { |
| | 543 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); |
| | 544 | |
| | 545 | /* Pre-DevBDW: OABUFFER must be set with counters off, |
| | 546 | * before OASTATUS1, but after OASTATUS2 |
| | 547 | */ |
| | 548 | I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */ |
| | 549 | I915_WRITE(GEN7_OABUFFER, gtt_offset); |
| | 550 | I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ |
| | 551 | |
| | 552 | /* On Haswell we have to track which OASTATUS1 flags we've |
| | 553 | * already seen since they can't be cleared while periodic |
| | 554 | * sampling is enabled. |
| | 555 | */ |
| | 556 | dev_priv->perf.oa.gen7_latched_oastatus1 = 0; |
| | 557 | |
| | 558 | /* NB: although the OA buffer will initially be allocated |
| | 559 | * zeroed via shmfs (and so this memset is redundant when |
| | 560 | * first allocating), we may re-init the OA buffer, either |
| | 561 | * when re-enabling a stream or in error/reset paths. |
| | 562 | * |
| | 563 | * The reason we clear the buffer for each re-init is for the |
| | 564 | * sanity check in gen7_append_oa_reports() that looks at the |
| | 565 | * report-id field to make sure it's non-zero which relies on |
| | 566 | * the assumption that new reports are being written to zeroed |
| | 567 | * memory... |
| | 568 | */ |
| | 569 | memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); |
| | 570 | |
| | 571 | /* Maybe make ->pollin per-stream state if we support multiple |
| | 572 | * concurrent streams in the future. |
| | 573 | */ |
| | 574 | dev_priv->perf.oa.pollin = false; |
| | 575 | } |
| | 576 | |
| | 577 | static int alloc_oa_buffer(struct drm_i915_private *dev_priv) |
| | 578 | { |
| | 579 | struct drm_i915_gem_object *bo; |
| | 580 | struct i915_vma *vma; |
| | 581 | int ret; |
| | 582 | |
| | 583 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) |
| | 584 | return -ENODEV; |
| | 585 | |
| | 586 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); |
| | 587 | if (ret) |
| | 588 | return ret; |
| | 589 | |
| | 590 | BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); |
| | 591 | BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); |
| | 592 | |
| | 593 | bo = i915_gem_object_create(&dev_priv->drm, OA_BUFFER_SIZE); |
| | 594 | if (IS_ERR(bo)) { |
| | 595 | DRM_ERROR("Failed to allocate OA buffer\n"); |
| | 596 | ret = PTR_ERR(bo); |
| | 597 | goto unlock; |
| | 598 | } |
| | 599 | |
| | 600 | ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); |
| | 601 | if (ret) |
| | 602 | goto err_unref; |
| | 603 | |
| | 604 | /* PreHSW required 512K alignment, HSW requires 16M */ |
| | 605 | vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); |
| | 606 | if (IS_ERR(vma)) { |
| | 607 | ret = PTR_ERR(vma); |
| | 608 | goto err_unref; |
| | 609 | } |
| | 610 | dev_priv->perf.oa.oa_buffer.vma = vma; |
| | 611 | |
| | 612 | dev_priv->perf.oa.oa_buffer.vaddr = |
| | 613 | i915_gem_object_pin_map(bo, I915_MAP_WB); |
| | 614 | if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { |
| | 615 | ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); |
| | 616 | goto err_unpin; |
| | 617 | } |
| | 618 | |
| | 619 | dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); |
| | 620 | |
| | 621 | DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", |
| | 622 | i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), |
| | 623 | dev_priv->perf.oa.oa_buffer.vaddr); |
| | 624 | |
| | 625 | goto unlock; |
| | 626 | |
| | 627 | err_unpin: |
| | 628 | __i915_vma_unpin(vma); |
| | 629 | |
| | 630 | err_unref: |
| | 631 | i915_gem_object_put(bo); |
| | 632 | |
| | 633 | dev_priv->perf.oa.oa_buffer.vaddr = NULL; |
| | 634 | dev_priv->perf.oa.oa_buffer.vma = NULL; |
| | 635 | |
| | 636 | unlock: |
| | 637 | mutex_unlock(&dev_priv->drm.struct_mutex); |
| | 638 | return ret; |
| | 639 | } |
| | 640 | |
| | 641 | static void config_oa_regs(struct drm_i915_private *dev_priv, |
| | 642 | const struct i915_oa_reg *regs, |
| | 643 | int n_regs) |
| | 644 | { |
| | 645 | int i; |
| | 646 | |
| | 647 | for (i = 0; i < n_regs; i++) { |
| | 648 | const struct i915_oa_reg *reg = regs + i; |
| | 649 | |
| | 650 | I915_WRITE(reg->addr, reg->value); |
| | 651 | } |
| | 652 | } |
| | 653 | |
| | 654 | static int hsw_enable_metric_set(struct drm_i915_private *dev_priv) |
| | 655 | { |
| | 656 | int ret = i915_oa_select_metric_set_hsw(dev_priv); |
| | 657 | |
| | 658 | if (ret) |
| | 659 | return ret; |
| | 660 | |
| | 661 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) | |
| | 662 | GT_NOA_ENABLE)); |
| | 663 | |
| | 664 | /* PRM: |
| | 665 | * |
| | 666 | * OA unit is using “crclk” for its functionality. When trunk |
| | 667 | * level clock gating takes place, OA clock would be gated, |
| | 668 | * unable to count the events from non-render clock domain. |
| | 669 | * Render clock gating must be disabled when OA is enabled to |
| | 670 | * count the events from non-render domain. Unit level clock |
| | 671 | * gating for RCS should also be disabled. |
| | 672 | */ |
| | 673 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & |
| | 674 | ~GEN7_DOP_CLOCK_GATE_ENABLE)); |
| | 675 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | |
| | 676 | GEN6_CSUNIT_CLOCK_GATE_DISABLE)); |
| | 677 | |
| | 678 | config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs, |
| | 679 | dev_priv->perf.oa.mux_regs_len); |
| | 680 | |
| | 681 | /* It apparently takes a fairly long time for a new MUX |
| | 682 | * configuration to be be applied after these register writes. |
| | 683 | * This delay duration was derived empirically based on the |
| | 684 | * render_basic config but hopefully it covers the maximum |
| | 685 | * configuration latency. |
| | 686 | * |
| | 687 | * As a fallback, the checks in _append_oa_reports() to skip |
| | 688 | * invalid OA reports do also seem to work to discard reports |
| | 689 | * generated before this config has completed - albeit not |
| | 690 | * silently. |
| | 691 | * |
| | 692 | * Unfortunately this is essentially a magic number, since we |
| | 693 | * don't currently know of a reliable mechanism for predicting |
| | 694 | * how long the MUX config will take to apply and besides |
| | 695 | * seeing invalid reports we don't know of a reliable way to |
| | 696 | * explicitly check that the MUX config has landed. |
| | 697 | * |
| | 698 | * It's even possible we've miss characterized the underlying |
| | 699 | * problem - it just seems like the simplest explanation why |
| | 700 | * a delay at this location would mitigate any invalid reports. |
| | 701 | */ |
| | 702 | usleep_range(15000, 20000); |
| | 703 | |
| | 704 | config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs, |
| | 705 | dev_priv->perf.oa.b_counter_regs_len); |
| | 706 | |
| | 707 | return 0; |
| | 708 | } |
| | 709 | |
| | 710 | static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) |
| | 711 | { |
| | 712 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & |
| | 713 | ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); |
| | 714 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | |
| | 715 | GEN7_DOP_CLOCK_GATE_ENABLE)); |
| | 716 | |
| | 717 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & |
| | 718 | ~GT_NOA_ENABLE)); |
| | 719 | } |
| | 720 | |
| | 721 | static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv) |
| | 722 | { |
| | 723 | assert_spin_locked(&dev_priv->perf.hook_lock); |
| | 724 | |
| | 725 | if (dev_priv->perf.oa.exclusive_stream->enabled) { |
| | 726 | struct i915_gem_context *ctx = |
| | 727 | dev_priv->perf.oa.exclusive_stream->ctx; |
| | 728 | u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; |
| | 729 | |
| | 730 | bool periodic = dev_priv->perf.oa.periodic; |
| | 731 | u32 period_exponent = dev_priv->perf.oa.period_exponent; |
| | 732 | u32 report_format = dev_priv->perf.oa.oa_buffer.format; |
| | 733 | |
| | 734 | I915_WRITE(GEN7_OACONTROL, |
| | 735 | (ctx_id & GEN7_OACONTROL_CTX_MASK) | |
| | 736 | (period_exponent << |
| | 737 | GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | |
| | 738 | (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | |
| | 739 | (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | |
| | 740 | (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | |
| | 741 | GEN7_OACONTROL_ENABLE); |
| | 742 | } else |
| | 743 | I915_WRITE(GEN7_OACONTROL, 0); |
| | 744 | } |
| | 745 | |
| | 746 | static void gen7_oa_enable(struct drm_i915_private *dev_priv) |
| | 747 | { |
| | 748 | unsigned long flags; |
| | 749 | |
| | 750 | /* Reset buf pointers so we don't forward reports from before now. |
| | 751 | * |
| | 752 | * Think carefully if considering trying to avoid this, since it |
| | 753 | * also ensures status flags and the buffer itself are cleared |
| | 754 | * in error paths, and we have checks for invalid reports based |
| | 755 | * on the assumption that certain fields are written to zeroed |
| | 756 | * memory which this helps maintains. |
| | 757 | */ |
| | 758 | gen7_init_oa_buffer(dev_priv); |
| | 759 | |
| | 760 | spin_lock_irqsave(&dev_priv->perf.hook_lock, flags); |
| | 761 | gen7_update_oacontrol_locked(dev_priv); |
| | 762 | spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags); |
| | 763 | } |
| | 764 | |
| | 765 | static void i915_oa_stream_enable(struct i915_perf_stream *stream) |
| | 766 | { |
| | 767 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 768 | |
| | 769 | dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| | 770 | |
| | 771 | if (dev_priv->perf.oa.periodic) |
| | 772 | hrtimer_start(&dev_priv->perf.oa.poll_check_timer, |
| | 773 | ns_to_ktime(POLL_PERIOD), |
| | 774 | HRTIMER_MODE_REL_PINNED); |
| | 775 | } |
| | 776 | |
| | 777 | static void gen7_oa_disable(struct drm_i915_private *dev_priv) |
| | 778 | { |
| | 779 | I915_WRITE(GEN7_OACONTROL, 0); |
| | 780 | } |
| | 781 | |
| | 782 | static void i915_oa_stream_disable(struct i915_perf_stream *stream) |
| | 783 | { |
| | 784 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 785 | |
| | 786 | dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| | 787 | |
| | 788 | if (dev_priv->perf.oa.periodic) |
| | 789 | hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); |
| | 790 | } |
| | 791 | |
| | 792 | static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) |
| | 793 | { |
| | 794 | return 1000000000ULL * (2ULL << exponent) / |
| | 795 | dev_priv->perf.oa.timestamp_frequency; |
| | 796 | } |
| | 797 | |
| | 798 | static const struct i915_perf_stream_ops i915_oa_stream_ops = { |
| | 799 | .destroy = i915_oa_stream_destroy, |
| | 800 | .enable = i915_oa_stream_enable, |
| | 801 | .disable = i915_oa_stream_disable, |
| | 802 | .wait_unlocked = i915_oa_wait_unlocked, |
| | 803 | .poll_wait = i915_oa_poll_wait, |
| | 804 | .read = i915_oa_read, |
| 36 | }; | 805 | }; |
| 37 | | 806 | |
| | 807 | static int i915_oa_stream_init(struct i915_perf_stream *stream, |
| | 808 | struct drm_i915_perf_open_param *param, |
| | 809 | struct perf_open_properties *props) |
| | 810 | { |
| | 811 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| | 812 | int format_size; |
| | 813 | int ret; |
| | 814 | |
| | 815 | if (!(props->sample_flags & SAMPLE_OA_REPORT)) { |
| | 816 | DRM_ERROR("Only OA report sampling supported\n"); |
| | 817 | return -EINVAL; |
| | 818 | } |
| | 819 | |
| | 820 | if (!dev_priv->perf.oa.ops.init_oa_buffer) { |
| | 821 | DRM_ERROR("OA unit not supported\n"); |
| | 822 | return -ENODEV; |
| | 823 | } |
| | 824 | |
| | 825 | /* To avoid the complexity of having to accurately filter |
| | 826 | * counter reports and marshal to the appropriate client |
| | 827 | * we currently only allow exclusive access |
| | 828 | */ |
| | 829 | if (dev_priv->perf.oa.exclusive_stream) { |
| | 830 | DRM_ERROR("OA unit already in use\n"); |
| | 831 | return -EBUSY; |
| | 832 | } |
| | 833 | |
| | 834 | if (!props->metrics_set) { |
| | 835 | DRM_ERROR("OA metric set not specified\n"); |
| | 836 | return -EINVAL; |
| | 837 | } |
| | 838 | |
| | 839 | if (!props->oa_format) { |
| | 840 | DRM_ERROR("OA report format not specified\n"); |
| | 841 | return -EINVAL; |
| | 842 | } |
| | 843 | |
| | 844 | stream->sample_size = sizeof(struct drm_i915_perf_record_header); |
| | 845 | |
| | 846 | format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; |
| | 847 | |
| | 848 | stream->sample_flags |= SAMPLE_OA_REPORT; |
| | 849 | stream->sample_size += format_size; |
| | 850 | |
| | 851 | dev_priv->perf.oa.oa_buffer.format_size = format_size; |
| | 852 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) |
| | 853 | return -EINVAL; |
| | 854 | |
| | 855 | dev_priv->perf.oa.oa_buffer.format = |
| | 856 | dev_priv->perf.oa.oa_formats[props->oa_format].format; |
| | 857 | |
| | 858 | dev_priv->perf.oa.metrics_set = props->metrics_set; |
| | 859 | |
| | 860 | dev_priv->perf.oa.periodic = props->oa_periodic; |
| | 861 | if (dev_priv->perf.oa.periodic) { |
| | 862 | u64 period_ns = oa_exponent_to_ns(dev_priv, |
| | 863 | props->oa_period_exponent); |
| | 864 | |
| | 865 | dev_priv->perf.oa.period_exponent = props->oa_period_exponent; |
| | 866 | |
| | 867 | /* See comment for OA_TAIL_MARGIN_NSEC for details |
| | 868 | * about this tail_margin... |
| | 869 | */ |
| | 870 | dev_priv->perf.oa.tail_margin = |
| | 871 | ((OA_TAIL_MARGIN_NSEC / period_ns) + 1) * format_size; |
| | 872 | } |
| | 873 | |
| | 874 | if (stream->ctx) { |
| | 875 | ret = oa_get_render_ctx_id(stream); |
| | 876 | if (ret) |
| | 877 | return ret; |
| | 878 | } |
| | 879 | |
| | 880 | ret = alloc_oa_buffer(dev_priv); |
| | 881 | if (ret) |
| | 882 | goto err_oa_buf_alloc; |
| | 883 | |
| | 884 | /* PRM - observability performance counters: |
| | 885 | * |
| | 886 | * OACONTROL, performance counter enable, note: |
| | 887 | * |
| | 888 | * "When this bit is set, in order to have coherent counts, |
| | 889 | * RC6 power state and trunk clock gating must be disabled. |
| | 890 | * This can be achieved by programming MMIO registers as |
| | 891 | * 0xA094=0 and 0xA090[31]=1" |
| | 892 | * |
| | 893 | * In our case we are expecting that taking pm + FORCEWAKE |
| | 894 | * references will effectively disable RC6. |
| | 895 | */ |
| | 896 | intel_runtime_pm_get(dev_priv); |
| | 897 | intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); |
| | 898 | |
| | 899 | ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv); |
| | 900 | if (ret) |
| | 901 | goto err_enable; |
| | 902 | |
| | 903 | stream->ops = &i915_oa_stream_ops; |
| | 904 | |
| | 905 | dev_priv->perf.oa.exclusive_stream = stream; |
| | 906 | |
| | 907 | return 0; |
| | 908 | |
| | 909 | err_enable: |
| | 910 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); |
| | 911 | intel_runtime_pm_put(dev_priv); |
| | 912 | free_oa_buffer(dev_priv); |
| | 913 | |
| | 914 | err_oa_buf_alloc: |
| | 915 | if (stream->ctx) |
| | 916 | oa_put_render_ctx_id(stream); |
| | 917 | |
| | 918 | return ret; |
| | 919 | } |
| | 920 | |
| 38 | static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, | 921 | static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, |
| 39 | struct file *file, | 922 | struct file *file, |
| 40 | char __user *buf, | 923 | char __user *buf, |
| @@ -78,8 +961,20 @@ static ssize_t i915_perf_read(struct file *file, · 37 unmodified lines |
| 78 | struct drm_i915_private *dev_priv = stream->dev_priv; | 961 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| 79 | ssize_t ret; | 962 | ssize_t ret; |
| 80 | | 963 | |
| | 964 | /* To ensure it's handled consistently we simply treat all reads of a |
| | 965 | * disabled stream as an error. In particular it might otherwise lead |
| | 966 | * to a deadlock for blocking file descriptors... |
| | 967 | */ |
| | 968 | if (!stream->enabled) |
| | 969 | return -EIO; |
| | 970 | |
| 81 | if (!(file->f_flags & O_NONBLOCK)) { | 971 | if (!(file->f_flags & O_NONBLOCK)) { |
| 82 | /* Allow false positives from stream->ops->wait_unlocked. | 972 | /* There's the small chance of false positives from |
| | 973 | * stream->ops->wait_unlocked. |
| | 974 | * |
| | 975 | * E.g. with single context filtering since we only wait until |
| | 976 | * oabuffer has >= 1 report we don't immediately know whether |
| | 977 | * any reports really belong to the current context |
| 83 | */ | 978 | */ |
| 84 | do { | 979 | do { |
| 85 | ret = stream->ops->wait_unlocked(stream); | 980 | ret = stream->ops->wait_unlocked(stream); |
| @@ -97,21 +992,51 @@ static ssize_t i915_perf_read(struct file *file, · 11 unmodified lines |
| 97 | mutex_unlock(&dev_priv->perf.lock); | 992 | mutex_unlock(&dev_priv->perf.lock); |
| 98 | } | 993 | } |
| 99 | | 994 | |
| | 995 | if (ret >= 0) { |
| | 996 | /* Maybe make ->pollin per-stream state if we support multiple |
| | 997 | * concurrent streams in the future. |
| | 998 | */ |
| | 999 | dev_priv->perf.oa.pollin = false; |
| | 1000 | } |
| | 1001 | |
| 100 | return ret; | 1002 | return ret; |
| 101 | } | 1003 | } |
| 102 | | 1004 | |
| 103 | static unsigned int i915_perf_poll_locked(struct i915_perf_stream *stream, | 1005 | static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) |
| | 1006 | { |
| | 1007 | struct drm_i915_private *dev_priv = |
| | 1008 | container_of(hrtimer, typeof(*dev_priv), |
| | 1009 | perf.oa.poll_check_timer); |
| | 1010 | |
| | 1011 | if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) { |
| | 1012 | dev_priv->perf.oa.pollin = true; |
| | 1013 | wake_up(&dev_priv->perf.oa.poll_wq); |
| | 1014 | } |
| | 1015 | |
| | 1016 | hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); |
| | 1017 | |
| | 1018 | return HRTIMER_RESTART; |
| | 1019 | } |
| | 1020 | |
| | 1021 | static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv, |
| | 1022 | struct i915_perf_stream *stream, |
| 104 | struct file *file, | 1023 | struct file *file, |
| 105 | poll_table *wait) | 1024 | poll_table *wait) |
| 106 | { | 1025 | { |
| 107 | unsigned int streams = 0; | 1026 | unsigned int events = 0; |
| 108 | | 1027 | |
| 109 | stream->ops->poll_wait(stream, file, wait); | 1028 | stream->ops->poll_wait(stream, file, wait); |
| 110 | | 1029 | |
| 111 | if (stream->ops->can_read(stream)) | 1030 | /* Note: we don't explicitly check whether there's something to read |
| 112 | streams |= POLLIN; | 1031 | * here since this path may be very hot depending on what else |
| | 1032 | * userspace is polling, or on the timeout in use. We rely solely on |
| | 1033 | * the hrtimer/oa_poll_check_timer_cb to notify us when there are |
| | 1034 | * samples to read. |
| | 1035 | */ |
| | 1036 | if (dev_priv->perf.oa.pollin) |
| | 1037 | events |= POLLIN; |
| 113 | | 1038 | |
| 114 | return streams; | 1039 | return events; |
| 115 | } | 1040 | } |
| 116 | | 1041 | |
| 117 | static unsigned int i915_perf_poll(struct file *file, poll_table *wait) | 1042 | static unsigned int i915_perf_poll(struct file *file, poll_table *wait) |
| @@ -121,7 +1046,7 @@ static unsigned int i915_perf_poll(struct file *file, poll_table *wait) · 3 unmodified lines |
| 121 | int ret; | 1046 | int ret; |
| 122 | | 1047 | |
| 123 | mutex_lock(&dev_priv->perf.lock); | 1048 | mutex_lock(&dev_priv->perf.lock); |
| 124 | ret = i915_perf_poll_locked(stream, file, wait); | 1049 | ret = i915_perf_poll_locked(dev_priv, stream, file, wait); |
| 125 | mutex_unlock(&dev_priv->perf.lock); | 1050 | mutex_unlock(&dev_priv->perf.lock); |
| 126 | | 1051 | |
| 127 | return ret; | 1052 | return ret; |
| @@ -285,18 +1210,21 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, · 157 unmodified lines |
| 285 | goto err_ctx; | 1210 | goto err_ctx; |
| 286 | } | 1211 | } |
| 287 | | 1212 | |
| 288 | stream->sample_flags = props->sample_flags; | | |
| 289 | stream->dev_priv = dev_priv; | 1213 | stream->dev_priv = dev_priv; |
| 290 | stream->ctx = specific_ctx; | 1214 | stream->ctx = specific_ctx; |
| 291 | | 1215 | |
| 292 | /* | 1216 | ret = i915_oa_stream_init(stream, param, props); |
| 293 | * TODO: support sampling something | 1217 | if (ret) |
| 294 | * | 1218 | goto err_alloc; |
| 295 | * For now this is as far as we can go. | 1219 | |
| | 1220 | /* we avoid simply assigning stream->sample_flags = props->sample_flags |
| | 1221 | * to have _stream_init check the combination of sample flags more |
| | 1222 | * thoroughly, but still this is the expected result at this point. |
| 296 | */ | 1223 | */ |
| 297 | DRM_ERROR("Unsupported i915 perf stream configuration\n"); | 1224 | if (WARN_ON(stream->sample_flags != props->sample_flags)) { |
| 298 | ret = -EINVAL; | 1225 | ret = -ENODEV; |
| 299 | goto err_alloc; | 1226 | goto err_alloc; |
| | 1227 | } |
| 300 | | 1228 | |
| 301 | list_add(&stream->link, &dev_priv->perf.streams); | 1229 | list_add(&stream->link, &dev_priv->perf.streams); |
| 302 | | 1230 | |
| @@ -382,6 +1310,56 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv, · 79 unmodified lines |
| 382 | props->single_context = 1; | 1310 | props->single_context = 1; |
| 383 | props->ctx_handle = value; | 1311 | props->ctx_handle = value; |
| 384 | break; | 1312 | break; |
| | 1313 | case DRM_I915_PERF_PROP_SAMPLE_OA: |
| | 1314 | props->sample_flags |= SAMPLE_OA_REPORT; |
| | 1315 | break; |
| | 1316 | case DRM_I915_PERF_PROP_OA_METRICS_SET: |
| | 1317 | if (value == 0 || |
| | 1318 | value > dev_priv->perf.oa.n_builtin_sets) { |
| | 1319 | DRM_ERROR("Unknown OA metric set ID"); |
| | 1320 | return -EINVAL; |
| | 1321 | } |
| | 1322 | props->metrics_set = value; |
| | 1323 | break; |
| | 1324 | case DRM_I915_PERF_PROP_OA_FORMAT: |
| | 1325 | if (value == 0 || value >= I915_OA_FORMAT_MAX) { |
| | 1326 | DRM_ERROR("Invalid OA report format\n"); |
| | 1327 | return -EINVAL; |
| | 1328 | } |
| | 1329 | if (!dev_priv->perf.oa.oa_formats[value].size) { |
| | 1330 | DRM_ERROR("Invalid OA report format\n"); |
| | 1331 | return -EINVAL; |
| | 1332 | } |
| | 1333 | props->oa_format = value; |
| | 1334 | break; |
| | 1335 | case DRM_I915_PERF_PROP_OA_EXPONENT: |
| | 1336 | if (value > OA_EXPONENT_MAX) { |
| | 1337 | DRM_ERROR("OA timer exponent too high (> %u)\n", |
| | 1338 | OA_EXPONENT_MAX); |
| | 1339 | return -EINVAL; |
| | 1340 | } |
| | 1341 | |
| | 1342 | /* NB: The exponent represents a period as follows: |
| | 1343 | * |
| | 1344 | * 80ns * 2^(period_exponent + 1) |
| | 1345 | * |
| | 1346 | * Theoretically we can program the OA unit to sample |
| | 1347 | * every 160ns but don't allow that by default unless |
| | 1348 | * root. |
| | 1349 | * |
| | 1350 | * Referring to perf's |
| | 1351 | * kernel.perf_event_max_sample_rate for a precedent |
| | 1352 | * (100000 by default); with an OA exponent of 6 we get |
| | 1353 | * a period of 10.240 microseconds -just under 100000Hz |
| | 1354 | */ |
| | 1355 | if (value < 6 && !capable(CAP_SYS_ADMIN)) { |
| | 1356 | DRM_ERROR("Minimum OA sampling exponent is 6 without root privileges\n"); |
| | 1357 | return -EACCES; |
| | 1358 | } |
| | 1359 | |
| | 1360 | props->oa_periodic = true; |
| | 1361 | props->oa_period_exponent = value; |
| | 1362 | break; |
| 385 | default: | 1363 | default: |
| 386 | MISSING_CASE(id); | 1364 | MISSING_CASE(id); |
| 387 | DRM_ERROR("Unknown i915 perf property ID"); | 1365 | DRM_ERROR("Unknown i915 perf property ID"); |
| @@ -432,8 +1410,33 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data, · 44 unmodified lines |
| 432 | | 1410 | |
| 433 | void i915_perf_init(struct drm_i915_private *dev_priv) | 1411 | void i915_perf_init(struct drm_i915_private *dev_priv) |
| 434 | { | 1412 | { |
| | 1413 | if (!IS_HASWELL(dev_priv)) |
| | 1414 | return; |
| | 1415 | |
| | 1416 | hrtimer_init(&dev_priv->perf.oa.poll_check_timer, |
| | 1417 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| | 1418 | dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; |
| | 1419 | init_waitqueue_head(&dev_priv->perf.oa.poll_wq); |
| | 1420 | |
| 435 | INIT_LIST_HEAD(&dev_priv->perf.streams); | 1421 | INIT_LIST_HEAD(&dev_priv->perf.streams); |
| 436 | mutex_init(&dev_priv->perf.lock); | 1422 | mutex_init(&dev_priv->perf.lock); |
| | 1423 | spin_lock_init(&dev_priv->perf.hook_lock); |
| | 1424 | |
| | 1425 | dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; |
| | 1426 | dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; |
| | 1427 | dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; |
| | 1428 | dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; |
| | 1429 | dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; |
| | 1430 | dev_priv->perf.oa.ops.read = gen7_oa_read; |
| | 1431 | dev_priv->perf.oa.ops.oa_buffer_is_empty = |
| | 1432 | gen7_oa_buffer_is_empty_fop_unlocked; |
| | 1433 | |
| | 1434 | dev_priv->perf.oa.timestamp_frequency = 12500000; |
| | 1435 | |
| | 1436 | dev_priv->perf.oa.oa_formats = hsw_oa_formats; |
| | 1437 | |
| | 1438 | dev_priv->perf.oa.n_builtin_sets = |
| | 1439 | i915_oa_n_builtin_metric_sets_hsw; |
| 437 | | 1440 | |
| 438 | dev_priv->perf.initialized = true; | 1441 | dev_priv->perf.initialized = true; |
| 439 | } | 1442 | } |
| @@ -443,7 +1446,6 @@ void i915_perf_fini(struct drm_i915_private *dev_priv) · 3 unmodified lines |
| 443 | if (!dev_priv->perf.initialized) | 1446 | if (!dev_priv->perf.initialized) |
| 444 | return; | 1447 | return; |
| 445 | | 1448 | |
| 446 | /* Currently nothing to clean up */ | 1449 | memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); |
| 447 | | | |
| 448 | dev_priv->perf.initialized = false; | 1450 | dev_priv->perf.initialized = false; |
| 449 | } | 1451 | } |