| @@ -25,16 +25,899 @@ · 24 unmodified lines |
| 25 | 25 | */ |
| 26 | 26 | |
| 27 | 27 | #include <linux/anon_inodes.h> |
| 28 | +#include <linux/sizes.h> |
| 28 | 29 | |
| 29 | 30 | #include "i915_drv.h" |
| 31 | +#include "i915_oa_hsw.h" |
| 32 | + |
| 33 | +/* HW requires this to be a power of two, between 128k and 16M, though driver |
| 34 | + * is currently generally designed assuming the largest 16M size is used such |
| 35 | + * that the overflow cases are unlikely in normal operation. |
| 36 | + */ |
| 37 | +#define OA_BUFFER_SIZE SZ_16M |
| 38 | + |
| 39 | +#define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) |
| 40 | + |
| 41 | +/* There's a HW race condition between OA unit tail pointer register updates and |
| 42 | + * writes to memory whereby the tail pointer can sometimes get ahead of what's |
| 43 | + * been written out to the OA buffer so far. |
| 44 | + * |
| 45 | + * Although this can be observed explicitly by checking for a zeroed report-id |
| 46 | + * field in tail reports, it seems preferable to account for this earlier e.g. |
| 47 | + * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles |
| 48 | + * in this situation. |
| 49 | + * |
| 50 | + * To give time for the most recent reports to land before they may be copied to |
| 51 | + * userspace, the driver operates as if the tail pointer effectively lags behind |
| 52 | + * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated |
| 53 | + * based on this constant in nanoseconds, the current OA sampling exponent |
| 54 | + * and current report size. |
| 55 | + * |
| 56 | + * There is also a fallback check while reading to simply skip over reports with |
| 57 | + * a zeroed report-id. |
| 58 | + */ |
| 59 | +#define OA_TAIL_MARGIN_NSEC 100000ULL |
| 60 | + |
| 61 | +/* frequency for checking whether the OA unit has written new reports to the |
| 62 | + * circular OA buffer... |
| 63 | + */ |
| 64 | +#define POLL_FREQUENCY 200 |
| 65 | +#define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) |
| 66 | + |
| 67 | +/* The maximum exponent the hardware accepts is 63 (essentially it selects one |
| 68 | + * of the 64bit timestamp bits to trigger reports from) but there's currently |
| 69 | + * no known use case for sampling as infrequently as once per 47 thousand years. |
| 70 | + * |
| 71 | + * Since the timestamps included in OA reports are only 32bits it seems |
| 72 | + * reasonable to limit the OA exponent where it's still possible to account for |
| 73 | + * overflow in OA report timestamps. |
| 74 | + */ |
| 75 | +#define OA_EXPONENT_MAX 31 |
| 76 | + |
| 77 | +#define INVALID_CTX_ID 0xffffffff |
| 78 | + |
| 79 | + |
| 80 | +/* XXX: beware if future OA HW adds new report formats that the current |
| 81 | + * code assumes all reports have a power-of-two size and ~(size - 1) can |
| 82 | + * be used as a mask to align the OA tail pointer. |
| 83 | + */ |
| 84 | +static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { |
| 85 | + [I915_OA_FORMAT_A13] = { 0, 64 }, |
| 86 | + [I915_OA_FORMAT_A29] = { 1, 128 }, |
| 87 | + [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, |
| 88 | + /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ |
| 89 | + [I915_OA_FORMAT_B4_C8] = { 4, 64 }, |
| 90 | + [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, |
| 91 | + [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, |
| 92 | + [I915_OA_FORMAT_C4_B8] = { 7, 64 }, |
| 93 | +}; |
| 94 | + |
| 95 | +#define SAMPLE_OA_REPORT (1<<0) |
| 30 | 96 | |
| 31 | 97 | struct perf_open_properties { |
| 32 | 98 | u32 sample_flags; |
| 33 | 99 | |
| 34 | 100 | u64 single_context:1; |
| 35 | 101 | u64 ctx_handle; |
| 102 | + |
| 103 | + /* OA sampling state */ |
| 104 | + int metrics_set; |
| 105 | + int oa_format; |
| 106 | + bool oa_periodic; |
| 107 | + int oa_period_exponent; |
| 108 | +}; |
| 109 | + |
| 110 | +/* NB: This is either called via fops or the poll check hrtimer (atomic ctx) |
| 111 | + * |
| 112 | + * It's safe to read OA config state here unlocked, assuming that this is only |
| 113 | + * called while the stream is enabled, while the global OA configuration can't |
| 114 | + * be modified. |
| 115 | + * |
| 116 | + * Note: we don't lock around the head/tail reads even though there's the slim |
| 117 | + * possibility of read() fop errors forcing a re-init of the OA buffer |
| 118 | + * pointers. A race here could result in a false positive !empty status which |
| 119 | + * is acceptable. |
| 120 | + */ |
| 121 | +static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv) |
| 122 | +{ |
| 123 | + int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| 124 | + u32 oastatus2 = I915_READ(GEN7_OASTATUS2); |
| 125 | + u32 oastatus1 = I915_READ(GEN7_OASTATUS1); |
| 126 | + u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| 127 | + u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| 128 | + |
| 129 | + return OA_TAKEN(tail, head) < |
| 130 | + dev_priv->perf.oa.tail_margin + report_size; |
| 131 | +} |
| 132 | + |
| 133 | +/** |
| 134 | + * Appends a status record to a userspace read() buffer. |
| 135 | + */ |
| 136 | +static int append_oa_status(struct i915_perf_stream *stream, |
| 137 | + char __user *buf, |
| 138 | + size_t count, |
| 139 | + size_t *offset, |
| 140 | + enum drm_i915_perf_record_type type) |
| 141 | +{ |
| 142 | + struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; |
| 143 | + |
| 144 | + if ((count - *offset) < header.size) |
| 145 | + return -ENOSPC; |
| 146 | + |
| 147 | + if (copy_to_user(buf + *offset, &header, sizeof(header))) |
| 148 | + return -EFAULT; |
| 149 | + |
| 150 | + (*offset) += header.size; |
| 151 | + |
| 152 | + return 0; |
| 153 | +} |
| 154 | + |
| 155 | +/** |
| 156 | + * Copies single OA report into userspace read() buffer. |
| 157 | + */ |
| 158 | +static int append_oa_sample(struct i915_perf_stream *stream, |
| 159 | + char __user *buf, |
| 160 | + size_t count, |
| 161 | + size_t *offset, |
| 162 | + const u8 *report) |
| 163 | +{ |
| 164 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 165 | + int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| 166 | + struct drm_i915_perf_record_header header; |
| 167 | + u32 sample_flags = stream->sample_flags; |
| 168 | + |
| 169 | + header.type = DRM_I915_PERF_RECORD_SAMPLE; |
| 170 | + header.pad = 0; |
| 171 | + header.size = stream->sample_size; |
| 172 | + |
| 173 | + if ((count - *offset) < header.size) |
| 174 | + return -ENOSPC; |
| 175 | + |
| 176 | + buf += *offset; |
| 177 | + if (copy_to_user(buf, &header, sizeof(header))) |
| 178 | + return -EFAULT; |
| 179 | + buf += sizeof(header); |
| 180 | + |
| 181 | + if (sample_flags & SAMPLE_OA_REPORT) { |
| 182 | + if (copy_to_user(buf, report, report_size)) |
| 183 | + return -EFAULT; |
| 184 | + } |
| 185 | + |
| 186 | + (*offset) += header.size; |
| 187 | + |
| 188 | + return 0; |
| 189 | +} |
| 190 | + |
| 191 | +/** |
| 192 | + * Copies all buffered OA reports into userspace read() buffer. |
| 193 | + * @stream: An i915-perf stream opened for OA metrics |
| 194 | + * @buf: destination buffer given by userspace |
| 195 | + * @count: the number of bytes userspace wants to read |
| 196 | + * @offset: (inout): the current position for writing into @buf |
| 197 | + * @head_ptr: (inout): the current oa buffer cpu read position |
| 198 | + * @tail: the current oa buffer gpu write position |
| 199 | + * |
| 200 | + * Returns 0 on success, negative error code on failure. |
| 201 | + * |
| 202 | + * Notably any error condition resulting in a short read (-ENOSPC or |
| 203 | + * -EFAULT) will be returned even though one or more records may |
| 204 | + * have been successfully copied. In this case it's up to the caller |
| 205 | + * to decide if the error should be squashed before returning to |
| 206 | + * userspace. |
| 207 | + * |
| 208 | + * Note: reports are consumed from the head, and appended to the |
| 209 | + * tail, so the head chases the tail?... If you think that's mad |
| 210 | + * and back-to-front you're not alone, but this follows the |
| 211 | + * Gen PRM naming convention. |
| 212 | + */ |
| 213 | +static int gen7_append_oa_reports(struct i915_perf_stream *stream, |
| 214 | + char __user *buf, |
| 215 | + size_t count, |
| 216 | + size_t *offset, |
| 217 | + u32 *head_ptr, |
| 218 | + u32 tail) |
| 219 | +{ |
| 220 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 221 | + int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| 222 | + u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; |
| 223 | + int tail_margin = dev_priv->perf.oa.tail_margin; |
| 224 | + u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); |
| 225 | + u32 mask = (OA_BUFFER_SIZE - 1); |
| 226 | + u32 head; |
| 227 | + u32 taken; |
| 228 | + int ret = 0; |
| 229 | + |
| 230 | + if (WARN_ON(!stream->enabled)) |
| 231 | + return -EIO; |
| 232 | + |
| 233 | + head = *head_ptr - gtt_offset; |
| 234 | + tail -= gtt_offset; |
| 235 | + |
| 236 | + /* The OA unit is expected to wrap the tail pointer according to the OA |
| 237 | + * buffer size and since we should never write a misaligned head |
| 238 | + * pointer we don't expect to read one back either... |
| 239 | + */ |
| 240 | + if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE || |
| 241 | + head % report_size) { |
| 242 | + DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n", |
| 243 | + head, tail); |
| 244 | + dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| 245 | + dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| 246 | + *head_ptr = I915_READ(GEN7_OASTATUS2) & |
| 247 | + GEN7_OASTATUS2_HEAD_MASK; |
| 248 | + return -EIO; |
| 249 | + } |
| 250 | + |
| 251 | + |
| 252 | + /* The tail pointer increases in 64 byte increments, not in report_size |
| 253 | + * steps... |
| 254 | + */ |
| 255 | + tail &= ~(report_size - 1); |
| 256 | + |
| 257 | + /* Move the tail pointer back by the current tail_margin to account for |
| 258 | + * the possibility that the latest reports may not have really landed |
| 259 | + * in memory yet... |
| 260 | + */ |
| 261 | + |
| 262 | + if (OA_TAKEN(tail, head) < report_size + tail_margin) |
| 263 | + return -EAGAIN; |
| 264 | + |
| 265 | + tail -= tail_margin; |
| 266 | + tail &= mask; |
| 267 | + |
| 268 | + for (/* none */; |
| 269 | + (taken = OA_TAKEN(tail, head)); |
| 270 | + head = (head + report_size) & mask) { |
| 271 | + u8 *report = oa_buf_base + head; |
| 272 | + u32 *report32 = (void *)report; |
| 273 | + |
| 274 | + /* All the report sizes factor neatly into the buffer |
| 275 | + * size so we never expect to see a report split |
| 276 | + * between the beginning and end of the buffer. |
| 277 | + * |
| 278 | + * Given the initial alignment check a misalignment |
| 279 | + * here would imply a driver bug that would result |
| 280 | + * in an overrun. |
| 281 | + */ |
| 282 | + if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { |
| 283 | + DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); |
| 284 | + break; |
| 285 | + } |
| 286 | + |
| 287 | + /* The report-ID field for periodic samples includes |
| 288 | + * some undocumented flags related to what triggered |
| 289 | + * the report and is never expected to be zero so we |
| 290 | + * can check that the report isn't invalid before |
| 291 | + * copying it to userspace... |
| 292 | + */ |
| 293 | + if (report32[0] == 0) { |
| 294 | + DRM_ERROR("Skipping spurious, invalid OA report\n"); |
| 295 | + continue; |
| 296 | + } |
| 297 | + |
| 298 | + ret = append_oa_sample(stream, buf, count, offset, report); |
| 299 | + if (ret) |
| 300 | + break; |
| 301 | + |
| 302 | + /* The above report-id field sanity check is based on |
| 303 | + * the assumption that the OA buffer is initially |
| 304 | + * zeroed and we reset the field after copying so the |
| 305 | + * check is still meaningful once old reports start |
| 306 | + * being overwritten. |
| 307 | + */ |
| 308 | + report32[0] = 0; |
| 309 | + } |
| 310 | + |
| 311 | + *head_ptr = gtt_offset + head; |
| 312 | + |
| 313 | + return ret; |
| 314 | +} |
| 315 | + |
| 316 | +static int gen7_oa_read(struct i915_perf_stream *stream, |
| 317 | + char __user *buf, |
| 318 | + size_t count, |
| 319 | + size_t *offset) |
| 320 | +{ |
| 321 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 322 | + int report_size = dev_priv->perf.oa.oa_buffer.format_size; |
| 323 | + u32 oastatus2; |
| 324 | + u32 oastatus1; |
| 325 | + u32 head; |
| 326 | + u32 tail; |
| 327 | + int ret; |
| 328 | + |
| 329 | + if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) |
| 330 | + return -EIO; |
| 331 | + |
| 332 | + oastatus2 = I915_READ(GEN7_OASTATUS2); |
| 333 | + oastatus1 = I915_READ(GEN7_OASTATUS1); |
| 334 | + |
| 335 | + head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| 336 | + tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| 337 | + |
| 338 | + /* XXX: On Haswell we don't have a safe way to clear oastatus1 |
| 339 | + * bits while the OA unit is enabled (while the tail pointer |
| 340 | + * may be updated asynchronously) so we ignore status bits |
| 341 | + * that have already been reported to userspace. |
| 342 | + */ |
| 343 | + oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; |
| 344 | + |
| 345 | + /* We treat OABUFFER_OVERFLOW as a significant error: |
| 346 | + * |
| 347 | + * - The status can be interpreted to mean that the buffer is |
| 348 | + * currently full (with a higher precedence than OA_TAKEN() |
| 349 | + * which will start to report a near-empty buffer after an |
| 350 | + * overflow) but it's awkward that we can't clear the status |
| 351 | + * on Haswell, so without a reset we won't be able to catch |
| 352 | + * the state again. |
| 353 | + * |
| 354 | + * - Since it also implies the HW has started overwriting old |
| 355 | + * reports it may also affect our sanity checks for invalid |
| 356 | + * reports when copying to userspace that assume new reports |
| 357 | + * are being written to cleared memory. |
| 358 | + * |
| 359 | + * - In the future we may want to introduce a flight recorder |
| 360 | + * mode where the driver will automatically maintain a safe |
| 361 | + * guard band between head/tail, avoiding this overflow |
| 362 | + * condition, but we avoid the added driver complexity for |
| 363 | + * now. |
| 364 | + */ |
| 365 | + if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { |
| 366 | + ret = append_oa_status(stream, buf, count, offset, |
| 367 | + DRM_I915_PERF_RECORD_OA_BUFFER_LOST); |
| 368 | + if (ret) |
| 369 | + return ret; |
| 370 | + |
| 371 | + DRM_ERROR("OA buffer overflow: force restart\n"); |
| 372 | + |
| 373 | + dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| 374 | + dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| 375 | + |
| 376 | + oastatus2 = I915_READ(GEN7_OASTATUS2); |
| 377 | + oastatus1 = I915_READ(GEN7_OASTATUS1); |
| 378 | + |
| 379 | + head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; |
| 380 | + tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; |
| 381 | + } |
| 382 | + |
| 383 | + if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { |
| 384 | + ret = append_oa_status(stream, buf, count, offset, |
| 385 | + DRM_I915_PERF_RECORD_OA_REPORT_LOST); |
| 386 | + if (ret) |
| 387 | + return ret; |
| 388 | + dev_priv->perf.oa.gen7_latched_oastatus1 |= |
| 389 | + GEN7_OASTATUS1_REPORT_LOST; |
| 390 | + } |
| 391 | + |
| 392 | + ret = gen7_append_oa_reports(stream, buf, count, offset, |
| 393 | + &head, tail); |
| 394 | + |
| 395 | + /* All the report sizes are a power of two and the |
| 396 | + * head should always be incremented by some multiple |
| 397 | + * of the report size. |
| 398 | + * |
| 399 | + * A warning here, but notably if we later read back a |
| 400 | + * misaligned pointer we will treat that as a bug since |
| 401 | + * it could lead to a buffer overrun. |
| 402 | + */ |
| 403 | + WARN_ONCE(head & (report_size - 1), |
| 404 | + "i915: Writing misaligned OA head pointer"); |
| 405 | + |
| 406 | + /* Note: we update the head pointer here even if an error |
| 407 | + * was returned since the error may represent a short read |
| 408 | + * where some some reports were successfully copied. |
| 409 | + */ |
| 410 | + I915_WRITE(GEN7_OASTATUS2, |
| 411 | + ((head & GEN7_OASTATUS2_HEAD_MASK) | |
| 412 | + OA_MEM_SELECT_GGTT)); |
| 413 | + |
| 414 | + return ret; |
| 415 | +} |
| 416 | + |
| 417 | +static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) |
| 418 | +{ |
| 419 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 420 | + |
| 421 | + /* We would wait indefinitely if periodic sampling is not enabled */ |
| 422 | + if (!dev_priv->perf.oa.periodic) |
| 423 | + return -EIO; |
| 424 | + |
| 425 | + /* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it |
| 426 | + * just performs mmio reads of the OA buffer head + tail pointers and |
| 427 | + * it's assumed we're handling some operation that implies the stream |
| 428 | + * can't be destroyed until completion (such as a read()) that ensures |
| 429 | + * the device + OA buffer can't disappear |
| 430 | + */ |
| 431 | + return wait_event_interruptible(dev_priv->perf.oa.poll_wq, |
| 432 | + !dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)); |
| 433 | +} |
| 434 | + |
| 435 | +static void i915_oa_poll_wait(struct i915_perf_stream *stream, |
| 436 | + struct file *file, |
| 437 | + poll_table *wait) |
| 438 | +{ |
| 439 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 440 | + |
| 441 | + poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); |
| 442 | +} |
| 443 | + |
| 444 | +static int i915_oa_read(struct i915_perf_stream *stream, |
| 445 | + char __user *buf, |
| 446 | + size_t count, |
| 447 | + size_t *offset) |
| 448 | +{ |
| 449 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 450 | + |
| 451 | + return dev_priv->perf.oa.ops.read(stream, buf, count, offset); |
| 452 | +} |
| 453 | + |
| 454 | +/* Determine the render context hw id, and ensure it remains fixed for the |
| 455 | + * lifetime of the stream. This ensures that we don't have to worry about |
| 456 | + * updating the context ID in OACONTROL on the fly. |
| 457 | + */ |
| 458 | +static int oa_get_render_ctx_id(struct i915_perf_stream *stream) |
| 459 | +{ |
| 460 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 461 | + struct i915_vma *vma; |
| 462 | + int ret; |
| 463 | + |
| 464 | + ret = i915_mutex_lock_interruptible(&dev_priv->drm); |
| 465 | + if (ret) |
| 466 | + return ret; |
| 467 | + |
| 468 | + /* As the ID is the gtt offset of the context's vma we pin |
| 469 | + * the vma to ensure the ID remains fixed. |
| 470 | + * |
| 471 | + * NB: implied RCS engine... |
| 472 | + */ |
| 473 | + vma = i915_gem_context_pin_legacy(stream->ctx, 0); |
| 474 | + if (IS_ERR(vma)) { |
| 475 | + ret = PTR_ERR(vma); |
| 476 | + goto unlock; |
| 477 | + } |
| 478 | + |
| 479 | + dev_priv->perf.oa.pinned_rcs_vma = vma; |
| 480 | + |
| 481 | + /* Explicitly track the ID (instead of calling i915_ggtt_offset() |
| 482 | + * on the fly) considering the difference with gen8+ and |
| 483 | + * execlists |
| 484 | + */ |
| 485 | + dev_priv->perf.oa.specific_ctx_id = i915_ggtt_offset(vma); |
| 486 | + |
| 487 | +unlock: |
| 488 | + mutex_unlock(&dev_priv->drm.struct_mutex); |
| 489 | + |
| 490 | + return ret; |
| 491 | +} |
| 492 | + |
| 493 | +static void oa_put_render_ctx_id(struct i915_perf_stream *stream) |
| 494 | +{ |
| 495 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 496 | + |
| 497 | + mutex_lock(&dev_priv->drm.struct_mutex); |
| 498 | + |
| 499 | + i915_vma_unpin(dev_priv->perf.oa.pinned_rcs_vma); |
| 500 | + dev_priv->perf.oa.pinned_rcs_vma = NULL; |
| 501 | + |
| 502 | + dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; |
| 503 | + |
| 504 | + mutex_unlock(&dev_priv->drm.struct_mutex); |
| 505 | +} |
| 506 | + |
| 507 | +static void |
| 508 | +free_oa_buffer(struct drm_i915_private *i915) |
| 509 | +{ |
| 510 | + mutex_lock(&i915->drm.struct_mutex); |
| 511 | + |
| 512 | + i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); |
| 513 | + i915_vma_unpin(i915->perf.oa.oa_buffer.vma); |
| 514 | + i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); |
| 515 | + |
| 516 | + i915->perf.oa.oa_buffer.vma = NULL; |
| 517 | + i915->perf.oa.oa_buffer.vaddr = NULL; |
| 518 | + |
| 519 | + mutex_unlock(&i915->drm.struct_mutex); |
| 520 | +} |
| 521 | + |
| 522 | +static void i915_oa_stream_destroy(struct i915_perf_stream *stream) |
| 523 | +{ |
| 524 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 525 | + |
| 526 | + BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); |
| 527 | + |
| 528 | + dev_priv->perf.oa.ops.disable_metric_set(dev_priv); |
| 529 | + |
| 530 | + free_oa_buffer(dev_priv); |
| 531 | + |
| 532 | + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); |
| 533 | + intel_runtime_pm_put(dev_priv); |
| 534 | + |
| 535 | + if (stream->ctx) |
| 536 | + oa_put_render_ctx_id(stream); |
| 537 | + |
| 538 | + dev_priv->perf.oa.exclusive_stream = NULL; |
| 539 | +} |
| 540 | + |
| 541 | +static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) |
| 542 | +{ |
| 543 | + u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); |
| 544 | + |
| 545 | + /* Pre-DevBDW: OABUFFER must be set with counters off, |
| 546 | + * before OASTATUS1, but after OASTATUS2 |
| 547 | + */ |
| 548 | + I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */ |
| 549 | + I915_WRITE(GEN7_OABUFFER, gtt_offset); |
| 550 | + I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ |
| 551 | + |
| 552 | + /* On Haswell we have to track which OASTATUS1 flags we've |
| 553 | + * already seen since they can't be cleared while periodic |
| 554 | + * sampling is enabled. |
| 555 | + */ |
| 556 | + dev_priv->perf.oa.gen7_latched_oastatus1 = 0; |
| 557 | + |
| 558 | + /* NB: although the OA buffer will initially be allocated |
| 559 | + * zeroed via shmfs (and so this memset is redundant when |
| 560 | + * first allocating), we may re-init the OA buffer, either |
| 561 | + * when re-enabling a stream or in error/reset paths. |
| 562 | + * |
| 563 | + * The reason we clear the buffer for each re-init is for the |
| 564 | + * sanity check in gen7_append_oa_reports() that looks at the |
| 565 | + * report-id field to make sure it's non-zero which relies on |
| 566 | + * the assumption that new reports are being written to zeroed |
| 567 | + * memory... |
| 568 | + */ |
| 569 | + memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); |
| 570 | + |
| 571 | + /* Maybe make ->pollin per-stream state if we support multiple |
| 572 | + * concurrent streams in the future. |
| 573 | + */ |
| 574 | + dev_priv->perf.oa.pollin = false; |
| 575 | +} |
| 576 | + |
| 577 | +static int alloc_oa_buffer(struct drm_i915_private *dev_priv) |
| 578 | +{ |
| 579 | + struct drm_i915_gem_object *bo; |
| 580 | + struct i915_vma *vma; |
| 581 | + int ret; |
| 582 | + |
| 583 | + if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) |
| 584 | + return -ENODEV; |
| 585 | + |
| 586 | + ret = i915_mutex_lock_interruptible(&dev_priv->drm); |
| 587 | + if (ret) |
| 588 | + return ret; |
| 589 | + |
| 590 | + BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); |
| 591 | + BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); |
| 592 | + |
| 593 | + bo = i915_gem_object_create(&dev_priv->drm, OA_BUFFER_SIZE); |
| 594 | + if (IS_ERR(bo)) { |
| 595 | + DRM_ERROR("Failed to allocate OA buffer\n"); |
| 596 | + ret = PTR_ERR(bo); |
| 597 | + goto unlock; |
| 598 | + } |
| 599 | + |
| 600 | + ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); |
| 601 | + if (ret) |
| 602 | + goto err_unref; |
| 603 | + |
| 604 | + /* PreHSW required 512K alignment, HSW requires 16M */ |
| 605 | + vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); |
| 606 | + if (IS_ERR(vma)) { |
| 607 | + ret = PTR_ERR(vma); |
| 608 | + goto err_unref; |
| 609 | + } |
| 610 | + dev_priv->perf.oa.oa_buffer.vma = vma; |
| 611 | + |
| 612 | + dev_priv->perf.oa.oa_buffer.vaddr = |
| 613 | + i915_gem_object_pin_map(bo, I915_MAP_WB); |
| 614 | + if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { |
| 615 | + ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); |
| 616 | + goto err_unpin; |
| 617 | + } |
| 618 | + |
| 619 | + dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); |
| 620 | + |
| 621 | + DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", |
| 622 | + i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), |
| 623 | + dev_priv->perf.oa.oa_buffer.vaddr); |
| 624 | + |
| 625 | + goto unlock; |
| 626 | + |
| 627 | +err_unpin: |
| 628 | + __i915_vma_unpin(vma); |
| 629 | + |
| 630 | +err_unref: |
| 631 | + i915_gem_object_put(bo); |
| 632 | + |
| 633 | + dev_priv->perf.oa.oa_buffer.vaddr = NULL; |
| 634 | + dev_priv->perf.oa.oa_buffer.vma = NULL; |
| 635 | + |
| 636 | +unlock: |
| 637 | + mutex_unlock(&dev_priv->drm.struct_mutex); |
| 638 | + return ret; |
| 639 | +} |
| 640 | + |
| 641 | +static void config_oa_regs(struct drm_i915_private *dev_priv, |
| 642 | + const struct i915_oa_reg *regs, |
| 643 | + int n_regs) |
| 644 | +{ |
| 645 | + int i; |
| 646 | + |
| 647 | + for (i = 0; i < n_regs; i++) { |
| 648 | + const struct i915_oa_reg *reg = regs + i; |
| 649 | + |
| 650 | + I915_WRITE(reg->addr, reg->value); |
| 651 | + } |
| 652 | +} |
| 653 | + |
| 654 | +static int hsw_enable_metric_set(struct drm_i915_private *dev_priv) |
| 655 | +{ |
| 656 | + int ret = i915_oa_select_metric_set_hsw(dev_priv); |
| 657 | + |
| 658 | + if (ret) |
| 659 | + return ret; |
| 660 | + |
| 661 | + I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) | |
| 662 | + GT_NOA_ENABLE)); |
| 663 | + |
| 664 | + /* PRM: |
| 665 | + * |
| 666 | + * OA unit is using “crclk” for its functionality. When trunk |
| 667 | + * level clock gating takes place, OA clock would be gated, |
| 668 | + * unable to count the events from non-render clock domain. |
| 669 | + * Render clock gating must be disabled when OA is enabled to |
| 670 | + * count the events from non-render domain. Unit level clock |
| 671 | + * gating for RCS should also be disabled. |
| 672 | + */ |
| 673 | + I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & |
| 674 | + ~GEN7_DOP_CLOCK_GATE_ENABLE)); |
| 675 | + I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | |
| 676 | + GEN6_CSUNIT_CLOCK_GATE_DISABLE)); |
| 677 | + |
| 678 | + config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs, |
| 679 | + dev_priv->perf.oa.mux_regs_len); |
| 680 | + |
| 681 | + /* It apparently takes a fairly long time for a new MUX |
| 682 | + * configuration to be be applied after these register writes. |
| 683 | + * This delay duration was derived empirically based on the |
| 684 | + * render_basic config but hopefully it covers the maximum |
| 685 | + * configuration latency. |
| 686 | + * |
| 687 | + * As a fallback, the checks in _append_oa_reports() to skip |
| 688 | + * invalid OA reports do also seem to work to discard reports |
| 689 | + * generated before this config has completed - albeit not |
| 690 | + * silently. |
| 691 | + * |
| 692 | + * Unfortunately this is essentially a magic number, since we |
| 693 | + * don't currently know of a reliable mechanism for predicting |
| 694 | + * how long the MUX config will take to apply and besides |
| 695 | + * seeing invalid reports we don't know of a reliable way to |
| 696 | + * explicitly check that the MUX config has landed. |
| 697 | + * |
| 698 | + * It's even possible we've miss characterized the underlying |
| 699 | + * problem - it just seems like the simplest explanation why |
| 700 | + * a delay at this location would mitigate any invalid reports. |
| 701 | + */ |
| 702 | + usleep_range(15000, 20000); |
| 703 | + |
| 704 | + config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs, |
| 705 | + dev_priv->perf.oa.b_counter_regs_len); |
| 706 | + |
| 707 | + return 0; |
| 708 | +} |
| 709 | + |
| 710 | +static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) |
| 711 | +{ |
| 712 | + I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & |
| 713 | + ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); |
| 714 | + I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | |
| 715 | + GEN7_DOP_CLOCK_GATE_ENABLE)); |
| 716 | + |
| 717 | + I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & |
| 718 | + ~GT_NOA_ENABLE)); |
| 719 | +} |
| 720 | + |
| 721 | +static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv) |
| 722 | +{ |
| 723 | + assert_spin_locked(&dev_priv->perf.hook_lock); |
| 724 | + |
| 725 | + if (dev_priv->perf.oa.exclusive_stream->enabled) { |
| 726 | + struct i915_gem_context *ctx = |
| 727 | + dev_priv->perf.oa.exclusive_stream->ctx; |
| 728 | + u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; |
| 729 | + |
| 730 | + bool periodic = dev_priv->perf.oa.periodic; |
| 731 | + u32 period_exponent = dev_priv->perf.oa.period_exponent; |
| 732 | + u32 report_format = dev_priv->perf.oa.oa_buffer.format; |
| 733 | + |
| 734 | + I915_WRITE(GEN7_OACONTROL, |
| 735 | + (ctx_id & GEN7_OACONTROL_CTX_MASK) | |
| 736 | + (period_exponent << |
| 737 | + GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | |
| 738 | + (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | |
| 739 | + (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | |
| 740 | + (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | |
| 741 | + GEN7_OACONTROL_ENABLE); |
| 742 | + } else |
| 743 | + I915_WRITE(GEN7_OACONTROL, 0); |
| 744 | +} |
| 745 | + |
| 746 | +static void gen7_oa_enable(struct drm_i915_private *dev_priv) |
| 747 | +{ |
| 748 | + unsigned long flags; |
| 749 | + |
| 750 | + /* Reset buf pointers so we don't forward reports from before now. |
| 751 | + * |
| 752 | + * Think carefully if considering trying to avoid this, since it |
| 753 | + * also ensures status flags and the buffer itself are cleared |
| 754 | + * in error paths, and we have checks for invalid reports based |
| 755 | + * on the assumption that certain fields are written to zeroed |
| 756 | + * memory which this helps maintains. |
| 757 | + */ |
| 758 | + gen7_init_oa_buffer(dev_priv); |
| 759 | + |
| 760 | + spin_lock_irqsave(&dev_priv->perf.hook_lock, flags); |
| 761 | + gen7_update_oacontrol_locked(dev_priv); |
| 762 | + spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags); |
| 763 | +} |
| 764 | + |
| 765 | +static void i915_oa_stream_enable(struct i915_perf_stream *stream) |
| 766 | +{ |
| 767 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 768 | + |
| 769 | + dev_priv->perf.oa.ops.oa_enable(dev_priv); |
| 770 | + |
| 771 | + if (dev_priv->perf.oa.periodic) |
| 772 | + hrtimer_start(&dev_priv->perf.oa.poll_check_timer, |
| 773 | + ns_to_ktime(POLL_PERIOD), |
| 774 | + HRTIMER_MODE_REL_PINNED); |
| 775 | +} |
| 776 | + |
| 777 | +static void gen7_oa_disable(struct drm_i915_private *dev_priv) |
| 778 | +{ |
| 779 | + I915_WRITE(GEN7_OACONTROL, 0); |
| 780 | +} |
| 781 | + |
| 782 | +static void i915_oa_stream_disable(struct i915_perf_stream *stream) |
| 783 | +{ |
| 784 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 785 | + |
| 786 | + dev_priv->perf.oa.ops.oa_disable(dev_priv); |
| 787 | + |
| 788 | + if (dev_priv->perf.oa.periodic) |
| 789 | + hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); |
| 790 | +} |
| 791 | + |
| 792 | +static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) |
| 793 | +{ |
| 794 | + return 1000000000ULL * (2ULL << exponent) / |
| 795 | + dev_priv->perf.oa.timestamp_frequency; |
| 796 | +} |
| 797 | + |
| 798 | +static const struct i915_perf_stream_ops i915_oa_stream_ops = { |
| 799 | + .destroy = i915_oa_stream_destroy, |
| 800 | + .enable = i915_oa_stream_enable, |
| 801 | + .disable = i915_oa_stream_disable, |
| 802 | + .wait_unlocked = i915_oa_wait_unlocked, |
| 803 | + .poll_wait = i915_oa_poll_wait, |
| 804 | + .read = i915_oa_read, |
| 36 | 805 | }; |
| 37 | 806 | |
| 807 | +static int i915_oa_stream_init(struct i915_perf_stream *stream, |
| 808 | + struct drm_i915_perf_open_param *param, |
| 809 | + struct perf_open_properties *props) |
| 810 | +{ |
| 811 | + struct drm_i915_private *dev_priv = stream->dev_priv; |
| 812 | + int format_size; |
| 813 | + int ret; |
| 814 | + |
| 815 | + if (!(props->sample_flags & SAMPLE_OA_REPORT)) { |
| 816 | + DRM_ERROR("Only OA report sampling supported\n"); |
| 817 | + return -EINVAL; |
| 818 | + } |
| 819 | + |
| 820 | + if (!dev_priv->perf.oa.ops.init_oa_buffer) { |
| 821 | + DRM_ERROR("OA unit not supported\n"); |
| 822 | + return -ENODEV; |
| 823 | + } |
| 824 | + |
| 825 | + /* To avoid the complexity of having to accurately filter |
| 826 | + * counter reports and marshal to the appropriate client |
| 827 | + * we currently only allow exclusive access |
| 828 | + */ |
| 829 | + if (dev_priv->perf.oa.exclusive_stream) { |
| 830 | + DRM_ERROR("OA unit already in use\n"); |
| 831 | + return -EBUSY; |
| 832 | + } |
| 833 | + |
| 834 | + if (!props->metrics_set) { |
| 835 | + DRM_ERROR("OA metric set not specified\n"); |
| 836 | + return -EINVAL; |
| 837 | + } |
| 838 | + |
| 839 | + if (!props->oa_format) { |
| 840 | + DRM_ERROR("OA report format not specified\n"); |
| 841 | + return -EINVAL; |
| 842 | + } |
| 843 | + |
| 844 | + stream->sample_size = sizeof(struct drm_i915_perf_record_header); |
| 845 | + |
| 846 | + format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; |
| 847 | + |
| 848 | + stream->sample_flags |= SAMPLE_OA_REPORT; |
| 849 | + stream->sample_size += format_size; |
| 850 | + |
| 851 | + dev_priv->perf.oa.oa_buffer.format_size = format_size; |
| 852 | + if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) |
| 853 | + return -EINVAL; |
| 854 | + |
| 855 | + dev_priv->perf.oa.oa_buffer.format = |
| 856 | + dev_priv->perf.oa.oa_formats[props->oa_format].format; |
| 857 | + |
| 858 | + dev_priv->perf.oa.metrics_set = props->metrics_set; |
| 859 | + |
| 860 | + dev_priv->perf.oa.periodic = props->oa_periodic; |
| 861 | + if (dev_priv->perf.oa.periodic) { |
| 862 | + u64 period_ns = oa_exponent_to_ns(dev_priv, |
| 863 | + props->oa_period_exponent); |
| 864 | + |
| 865 | + dev_priv->perf.oa.period_exponent = props->oa_period_exponent; |
| 866 | + |
| 867 | + /* See comment for OA_TAIL_MARGIN_NSEC for details |
| 868 | + * about this tail_margin... |
| 869 | + */ |
| 870 | + dev_priv->perf.oa.tail_margin = |
| 871 | + ((OA_TAIL_MARGIN_NSEC / period_ns) + 1) * format_size; |
| 872 | + } |
| 873 | + |
| 874 | + if (stream->ctx) { |
| 875 | + ret = oa_get_render_ctx_id(stream); |
| 876 | + if (ret) |
| 877 | + return ret; |
| 878 | + } |
| 879 | + |
| 880 | + ret = alloc_oa_buffer(dev_priv); |
| 881 | + if (ret) |
| 882 | + goto err_oa_buf_alloc; |
| 883 | + |
| 884 | + /* PRM - observability performance counters: |
| 885 | + * |
| 886 | + * OACONTROL, performance counter enable, note: |
| 887 | + * |
| 888 | + * "When this bit is set, in order to have coherent counts, |
| 889 | + * RC6 power state and trunk clock gating must be disabled. |
| 890 | + * This can be achieved by programming MMIO registers as |
| 891 | + * 0xA094=0 and 0xA090[31]=1" |
| 892 | + * |
| 893 | + * In our case we are expecting that taking pm + FORCEWAKE |
| 894 | + * references will effectively disable RC6. |
| 895 | + */ |
| 896 | + intel_runtime_pm_get(dev_priv); |
| 897 | + intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); |
| 898 | + |
| 899 | + ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv); |
| 900 | + if (ret) |
| 901 | + goto err_enable; |
| 902 | + |
| 903 | + stream->ops = &i915_oa_stream_ops; |
| 904 | + |
| 905 | + dev_priv->perf.oa.exclusive_stream = stream; |
| 906 | + |
| 907 | + return 0; |
| 908 | + |
| 909 | +err_enable: |
| 910 | + intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); |
| 911 | + intel_runtime_pm_put(dev_priv); |
| 912 | + free_oa_buffer(dev_priv); |
| 913 | + |
| 914 | +err_oa_buf_alloc: |
| 915 | + if (stream->ctx) |
| 916 | + oa_put_render_ctx_id(stream); |
| 917 | + |
| 918 | + return ret; |
| 919 | +} |
| 920 | + |
| 38 | 921 | static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, |
| 39 | 922 | struct file *file, |
| 40 | 923 | char __user *buf, |
| @@ -78,8 +961,20 @@ static ssize_t i915_perf_read(struct file *file, · 37 unmodified lines |
| 78 | 961 | struct drm_i915_private *dev_priv = stream->dev_priv; |
| 79 | 962 | ssize_t ret; |
| 80 | 963 | |
| 964 | + /* To ensure it's handled consistently we simply treat all reads of a |
| 965 | + * disabled stream as an error. In particular it might otherwise lead |
| 966 | + * to a deadlock for blocking file descriptors... |
| 967 | + */ |
| 968 | + if (!stream->enabled) |
| 969 | + return -EIO; |
| 970 | + |
| 81 | 971 | if (!(file->f_flags & O_NONBLOCK)) { |
| 82 | | − /* Allow false positives from stream->ops->wait_unlocked. |
| 972 | + /* There's the small chance of false positives from |
| 973 | + * stream->ops->wait_unlocked. |
| 974 | + * |
| 975 | + * E.g. with single context filtering since we only wait until |
| 976 | + * oabuffer has >= 1 report we don't immediately know whether |
| 977 | + * any reports really belong to the current context |
| 83 | 978 | */ |
| 84 | 979 | do { |
| 85 | 980 | ret = stream->ops->wait_unlocked(stream); |
| @@ -97,21 +992,51 @@ static ssize_t i915_perf_read(struct file *file, · 11 unmodified lines |
| 97 | 992 | mutex_unlock(&dev_priv->perf.lock); |
| 98 | 993 | } |
| 99 | 994 | |
| 995 | + if (ret >= 0) { |
| 996 | + /* Maybe make ->pollin per-stream state if we support multiple |
| 997 | + * concurrent streams in the future. |
| 998 | + */ |
| 999 | + dev_priv->perf.oa.pollin = false; |
| 1000 | + } |
| 1001 | + |
| 100 | 1002 | return ret; |
| 101 | 1003 | } |
| 102 | 1004 | |
| 103 | | −static unsigned int i915_perf_poll_locked(struct i915_perf_stream *stream, |
| 1005 | +static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) |
| 1006 | +{ |
| 1007 | + struct drm_i915_private *dev_priv = |
| 1008 | + container_of(hrtimer, typeof(*dev_priv), |
| 1009 | + perf.oa.poll_check_timer); |
| 1010 | + |
| 1011 | + if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) { |
| 1012 | + dev_priv->perf.oa.pollin = true; |
| 1013 | + wake_up(&dev_priv->perf.oa.poll_wq); |
| 1014 | + } |
| 1015 | + |
| 1016 | + hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); |
| 1017 | + |
| 1018 | + return HRTIMER_RESTART; |
| 1019 | +} |
| 1020 | + |
| 1021 | +static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv, |
| 1022 | + struct i915_perf_stream *stream, |
| 104 | 1023 | struct file *file, |
| 105 | 1024 | poll_table *wait) |
| 106 | 1025 | { |
| 107 | | − unsigned int streams = 0; |
| 1026 | + unsigned int events = 0; |
| 108 | 1027 | |
| 109 | 1028 | stream->ops->poll_wait(stream, file, wait); |
| 110 | 1029 | |
| 111 | | − if (stream->ops->can_read(stream)) |
| 112 | | − streams |= POLLIN; |
| 1030 | + /* Note: we don't explicitly check whether there's something to read |
| 1031 | + * here since this path may be very hot depending on what else |
| 1032 | + * userspace is polling, or on the timeout in use. We rely solely on |
| 1033 | + * the hrtimer/oa_poll_check_timer_cb to notify us when there are |
| 1034 | + * samples to read. |
| 1035 | + */ |
| 1036 | + if (dev_priv->perf.oa.pollin) |
| 1037 | + events |= POLLIN; |
| 113 | 1038 | |
| 114 | | − return streams; |
| 1039 | + return events; |
| 115 | 1040 | } |
| 116 | 1041 | |
| 117 | 1042 | static unsigned int i915_perf_poll(struct file *file, poll_table *wait) |
| @@ -121,7 +1046,7 @@ static unsigned int i915_perf_poll(struct file *file, poll_table *wait) · 3 unmodified lines |
| 121 | 1046 | int ret; |
| 122 | 1047 | |
| 123 | 1048 | mutex_lock(&dev_priv->perf.lock); |
| 124 | | − ret = i915_perf_poll_locked(stream, file, wait); |
| 1049 | + ret = i915_perf_poll_locked(dev_priv, stream, file, wait); |
| 125 | 1050 | mutex_unlock(&dev_priv->perf.lock); |
| 126 | 1051 | |
| 127 | 1052 | return ret; |
| @@ -285,18 +1210,21 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, · 157 unmodified lines |
| 285 | 1210 | goto err_ctx; |
| 286 | 1211 | } |
| 287 | 1212 | |
| 288 | | − stream->sample_flags = props->sample_flags; |
| 289 | 1213 | stream->dev_priv = dev_priv; |
| 290 | 1214 | stream->ctx = specific_ctx; |
| 291 | 1215 | |
| 292 | | − /* |
| 293 | | − * TODO: support sampling something |
| 294 | | − * |
| 295 | | − * For now this is as far as we can go. |
| 1216 | + ret = i915_oa_stream_init(stream, param, props); |
| 1217 | + if (ret) |
| 1218 | + goto err_alloc; |
| 1219 | + |
| 1220 | + /* we avoid simply assigning stream->sample_flags = props->sample_flags |
| 1221 | + * to have _stream_init check the combination of sample flags more |
| 1222 | + * thoroughly, but still this is the expected result at this point. |
| 296 | 1223 | */ |
| 297 | | − DRM_ERROR("Unsupported i915 perf stream configuration\n"); |
| 298 | | − ret = -EINVAL; |
| 299 | | − goto err_alloc; |
| 1224 | + if (WARN_ON(stream->sample_flags != props->sample_flags)) { |
| 1225 | + ret = -ENODEV; |
| 1226 | + goto err_alloc; |
| 1227 | + } |
| 300 | 1228 | |
| 301 | 1229 | list_add(&stream->link, &dev_priv->perf.streams); |
| 302 | 1230 | |
| @@ -382,6 +1310,56 @@ static int read_properties_unlocked(struct drm_i915_private *dev_priv, · 79 unmodified lines |
| 382 | 1310 | props->single_context = 1; |
| 383 | 1311 | props->ctx_handle = value; |
| 384 | 1312 | break; |
| 1313 | + case DRM_I915_PERF_PROP_SAMPLE_OA: |
| 1314 | + props->sample_flags |= SAMPLE_OA_REPORT; |
| 1315 | + break; |
| 1316 | + case DRM_I915_PERF_PROP_OA_METRICS_SET: |
| 1317 | + if (value == 0 || |
| 1318 | + value > dev_priv->perf.oa.n_builtin_sets) { |
| 1319 | + DRM_ERROR("Unknown OA metric set ID"); |
| 1320 | + return -EINVAL; |
| 1321 | + } |
| 1322 | + props->metrics_set = value; |
| 1323 | + break; |
| 1324 | + case DRM_I915_PERF_PROP_OA_FORMAT: |
| 1325 | + if (value == 0 || value >= I915_OA_FORMAT_MAX) { |
| 1326 | + DRM_ERROR("Invalid OA report format\n"); |
| 1327 | + return -EINVAL; |
| 1328 | + } |
| 1329 | + if (!dev_priv->perf.oa.oa_formats[value].size) { |
| 1330 | + DRM_ERROR("Invalid OA report format\n"); |
| 1331 | + return -EINVAL; |
| 1332 | + } |
| 1333 | + props->oa_format = value; |
| 1334 | + break; |
| 1335 | + case DRM_I915_PERF_PROP_OA_EXPONENT: |
| 1336 | + if (value > OA_EXPONENT_MAX) { |
| 1337 | + DRM_ERROR("OA timer exponent too high (> %u)\n", |
| 1338 | + OA_EXPONENT_MAX); |
| 1339 | + return -EINVAL; |
| 1340 | + } |
| 1341 | + |
| 1342 | + /* NB: The exponent represents a period as follows: |
| 1343 | + * |
| 1344 | + * 80ns * 2^(period_exponent + 1) |
| 1345 | + * |
| 1346 | + * Theoretically we can program the OA unit to sample |
| 1347 | + * every 160ns but don't allow that by default unless |
| 1348 | + * root. |
| 1349 | + * |
| 1350 | + * Referring to perf's |
| 1351 | + * kernel.perf_event_max_sample_rate for a precedent |
| 1352 | + * (100000 by default); with an OA exponent of 6 we get |
| 1353 | + * a period of 10.240 microseconds -just under 100000Hz |
| 1354 | + */ |
| 1355 | + if (value < 6 && !capable(CAP_SYS_ADMIN)) { |
| 1356 | + DRM_ERROR("Minimum OA sampling exponent is 6 without root privileges\n"); |
| 1357 | + return -EACCES; |
| 1358 | + } |
| 1359 | + |
| 1360 | + props->oa_periodic = true; |
| 1361 | + props->oa_period_exponent = value; |
| 1362 | + break; |
| 385 | 1363 | default: |
| 386 | 1364 | MISSING_CASE(id); |
| 387 | 1365 | DRM_ERROR("Unknown i915 perf property ID"); |
| @@ -432,8 +1410,33 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data, · 44 unmodified lines |
| 432 | 1410 | |
| 433 | 1411 | void i915_perf_init(struct drm_i915_private *dev_priv) |
| 434 | 1412 | { |
| 1413 | + if (!IS_HASWELL(dev_priv)) |
| 1414 | + return; |
| 1415 | + |
| 1416 | + hrtimer_init(&dev_priv->perf.oa.poll_check_timer, |
| 1417 | + CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 1418 | + dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; |
| 1419 | + init_waitqueue_head(&dev_priv->perf.oa.poll_wq); |
| 1420 | + |
| 435 | 1421 | INIT_LIST_HEAD(&dev_priv->perf.streams); |
| 436 | 1422 | mutex_init(&dev_priv->perf.lock); |
| 1423 | + spin_lock_init(&dev_priv->perf.hook_lock); |
| 1424 | + |
| 1425 | + dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; |
| 1426 | + dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; |
| 1427 | + dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; |
| 1428 | + dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; |
| 1429 | + dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; |
| 1430 | + dev_priv->perf.oa.ops.read = gen7_oa_read; |
| 1431 | + dev_priv->perf.oa.ops.oa_buffer_is_empty = |
| 1432 | + gen7_oa_buffer_is_empty_fop_unlocked; |
| 1433 | + |
| 1434 | + dev_priv->perf.oa.timestamp_frequency = 12500000; |
| 1435 | + |
| 1436 | + dev_priv->perf.oa.oa_formats = hsw_oa_formats; |
| 1437 | + |
| 1438 | + dev_priv->perf.oa.n_builtin_sets = |
| 1439 | + i915_oa_n_builtin_metric_sets_hsw; |
| 437 | 1440 | |
| 438 | 1441 | dev_priv->perf.initialized = true; |
| 439 | 1442 | } |
| @@ -443,7 +1446,6 @@ void i915_perf_fini(struct drm_i915_private *dev_priv) · 3 unmodified lines |
| 443 | 1446 | if (!dev_priv->perf.initialized) |
| 444 | 1447 | return; |
| 445 | 1448 | |
| 446 | | − /* Currently nothing to clean up */ |
| 447 | | − |
| 1449 | + memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); |
| 448 | 1450 | dev_priv->perf.initialized = false; |
| 449 | 1451 | } |