| @@ -22,6 +22,9 @@ · 21 unmodified lines |
| 22 | 22 | #include <asm/ptrace.h> |
| 23 | 23 | #include <asm/msr.h> |
| 24 | 24 | #include <asm/nmi.h> |
| 25 | +#include <asm/apic.h> |
| 26 | +#include <asm/processor.h> |
| 27 | +#include <asm/cpufeature.h> |
| 25 | 28 | |
| 26 | 29 | #include "op_x86_model.h" |
| 27 | 30 | #include "op_counter.h" |
| @@ -43,15 +46,13 @@ · 15 unmodified lines |
| 43 | 46 | |
| 44 | 47 | static unsigned long reset_value[NUM_VIRT_COUNTERS]; |
| 45 | 48 | |
| 46 | | −#ifdef CONFIG_OPROFILE_IBS |
| 47 | | − |
| 48 | 49 | /* IbsFetchCtl bits/masks */ |
| 49 | 50 | #define IBS_FETCH_RAND_EN (1ULL<<57) |
| 50 | 51 | #define IBS_FETCH_VAL (1ULL<<49) |
| 51 | 52 | #define IBS_FETCH_ENABLE (1ULL<<48) |
| 52 | 53 | #define IBS_FETCH_CNT_MASK 0xFFFF0000ULL |
| 53 | 54 | |
| 54 | | −/*IbsOpCtl bits */ |
| 55 | +/* IbsOpCtl bits */ |
| 55 | 56 | #define IBS_OP_CNT_CTL (1ULL<<19) |
| 56 | 57 | #define IBS_OP_VAL (1ULL<<18) |
| 57 | 58 | #define IBS_OP_ENABLE (1ULL<<17) |
| @@ -59,7 +60,7 @@ static unsigned long reset_value[NUM_VIRT_COUNTERS]; · 1 unmodified line |
| 59 | 60 | #define IBS_FETCH_SIZE 6 |
| 60 | 61 | #define IBS_OP_SIZE 12 |
| 61 | 62 | |
| 62 | | −static int has_ibs; /* AMD Family10h and later */ |
| 63 | +static u32 ibs_caps; |
| 63 | 64 | |
| 64 | 65 | struct op_ibs_config { |
| 65 | 66 | unsigned long op_enabled; |
| @@ -71,24 +72,52 @@ struct op_ibs_config { · 5 unmodified lines |
| 71 | 72 | }; |
| 72 | 73 | |
| 73 | 74 | static struct op_ibs_config ibs_config; |
| 75 | +static u64 ibs_op_ctl; |
| 74 | 76 | |
| 75 | | −#endif |
| 77 | +/* |
| 78 | + * IBS cpuid feature detection |
| 79 | + */ |
| 76 | 80 | |
| 77 | | −#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
| 81 | +#define IBS_CPUID_FEATURES 0x8000001b |
| 82 | + |
| 83 | +/* |
| 84 | + * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but |
| 85 | + * bit 0 is used to indicate the existence of IBS. |
| 86 | + */ |
| 87 | +#define IBS_CAPS_AVAIL (1LL<<0) |
| 88 | +#define IBS_CAPS_RDWROPCNT (1LL<<3) |
| 89 | +#define IBS_CAPS_OPCNT (1LL<<4) |
| 90 | + |
| 91 | +/* |
| 92 | + * IBS randomization macros |
| 93 | + */ |
| 94 | +#define IBS_RANDOM_BITS 12 |
| 95 | +#define IBS_RANDOM_MASK ((1ULL << IBS_RANDOM_BITS) - 1) |
| 96 | +#define IBS_RANDOM_MAXCNT_OFFSET (1ULL << (IBS_RANDOM_BITS - 5)) |
| 78 | 97 | |
| 79 | | −static void op_mux_fill_in_addresses(struct op_msrs * const msrs) |
| 98 | +static u32 get_ibs_caps(void) |
| 80 | 99 | { |
| 81 | | − int i; |
| 100 | + u32 ibs_caps; |
| 101 | + unsigned int max_level; |
| 82 | 102 | |
| 83 | | − for (i = 0; i < NUM_VIRT_COUNTERS; i++) { |
| 84 | | − int hw_counter = op_x86_virt_to_phys(i); |
| 85 | | − if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i)) |
| 86 | | − msrs->multiplex[i].addr = MSR_K7_PERFCTR0 + hw_counter; |
| 87 | | − else |
| 88 | | − msrs->multiplex[i].addr = 0; |
| 89 | | − } |
| 103 | + if (!boot_cpu_has(X86_FEATURE_IBS)) |
| 104 | + return 0; |
| 105 | + |
| 106 | + /* check IBS cpuid feature flags */ |
| 107 | + max_level = cpuid_eax(0x80000000); |
| 108 | + if (max_level < IBS_CPUID_FEATURES) |
| 109 | + return IBS_CAPS_AVAIL; |
| 110 | + |
| 111 | + ibs_caps = cpuid_eax(IBS_CPUID_FEATURES); |
| 112 | + if (!(ibs_caps & IBS_CAPS_AVAIL)) |
| 113 | + /* cpuid flags not valid */ |
| 114 | + return IBS_CAPS_AVAIL; |
| 115 | + |
| 116 | + return ibs_caps; |
| 90 | 117 | } |
| 91 | 118 | |
| 119 | +#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX |
| 120 | + |
| 92 | 121 | static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, |
| 93 | 122 | struct op_msrs const * const msrs) |
| 94 | 123 | { |
| @@ -98,7 +127,7 @@ static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, · 3 unmodified lines |
| 98 | 127 | /* enable active counters */ |
| 99 | 128 | for (i = 0; i < NUM_COUNTERS; ++i) { |
| 100 | 129 | int virt = op_x86_phys_to_virt(i); |
| 101 | | − if (!counter_config[virt].enabled) |
| 130 | + if (!reset_value[virt]) |
| 102 | 131 | continue; |
| 103 | 132 | rdmsrl(msrs->controls[i].addr, val); |
| 104 | 133 | val &= model->reserved; |
| @@ -107,10 +136,6 @@ static void op_mux_switch_ctrl(struct op_x86_model_spec const *model, · 2 unmodified lines |
| 107 | 136 | } |
| 108 | 137 | } |
| 109 | 138 | |
| 110 | | −#else |
| 111 | | − |
| 112 | | −static inline void op_mux_fill_in_addresses(struct op_msrs * const msrs) { } |
| 113 | | − |
| 114 | 139 | #endif |
| 115 | 140 | |
| 116 | 141 | /* functions for op_amd_spec */ |
| @@ -122,18 +147,12 @@ static void op_amd_fill_in_addresses(struct op_msrs * const msrs) · 5 unmodified lines |
| 122 | 147 | for (i = 0; i < NUM_COUNTERS; i++) { |
| 123 | 148 | if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i)) |
| 124 | 149 | msrs->counters[i].addr = MSR_K7_PERFCTR0 + i; |
| 125 | | − else |
| 126 | | − msrs->counters[i].addr = 0; |
| 127 | 150 | } |
| 128 | 151 | |
| 129 | 152 | for (i = 0; i < NUM_CONTROLS; i++) { |
| 130 | 153 | if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) |
| 131 | 154 | msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i; |
| 132 | | − else |
| 133 | | − msrs->controls[i].addr = 0; |
| 134 | 155 | } |
| 135 | | − |
| 136 | | − op_mux_fill_in_addresses(msrs); |
| 137 | 156 | } |
| 138 | 157 | |
| 139 | 158 | static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, |
| @@ -144,7 +163,8 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, · 4 unmodified lines |
| 144 | 163 | |
| 145 | 164 | /* setup reset_value */ |
| 146 | 165 | for (i = 0; i < NUM_VIRT_COUNTERS; ++i) { |
| 147 | | − if (counter_config[i].enabled) |
| 166 | + if (counter_config[i].enabled |
| 167 | + && msrs->counters[op_x86_virt_to_phys(i)].addr) |
| 148 | 168 | reset_value[i] = counter_config[i].count; |
| 149 | 169 | else |
| 150 | 170 | reset_value[i] = 0; |
| @@ -152,9 +172,18 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, · 1 unmodified line |
| 152 | 172 | |
| 153 | 173 | /* clear all counters */ |
| 154 | 174 | for (i = 0; i < NUM_CONTROLS; ++i) { |
| 155 | | − if (unlikely(!msrs->controls[i].addr)) |
| 175 | + if (unlikely(!msrs->controls[i].addr)) { |
| 176 | + if (counter_config[i].enabled && !smp_processor_id()) |
| 177 | + /* |
| 178 | + * counter is reserved, this is on all |
| 179 | + * cpus, so report only for cpu #0 |
| 180 | + */ |
| 181 | + op_x86_warn_reserved(i); |
| 156 | 182 | continue; |
| 183 | + } |
| 157 | 184 | rdmsrl(msrs->controls[i].addr, val); |
| 185 | + if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) |
| 186 | + op_x86_warn_in_use(i); |
| 158 | 187 | val &= model->reserved; |
| 159 | 188 | wrmsrl(msrs->controls[i].addr, val); |
| 160 | 189 | } |
| @@ -169,9 +198,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, · 8 unmodified lines |
| 169 | 198 | /* enable active counters */ |
| 170 | 199 | for (i = 0; i < NUM_COUNTERS; ++i) { |
| 171 | 200 | int virt = op_x86_phys_to_virt(i); |
| 172 | | − if (!counter_config[virt].enabled) |
| 173 | | − continue; |
| 174 | | − if (!msrs->counters[i].addr) |
| 201 | + if (!reset_value[virt]) |
| 175 | 202 | continue; |
| 176 | 203 | |
| 177 | 204 | /* setup counter registers */ |
| @@ -185,7 +212,60 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model, · 7 unmodified lines |
| 185 | 212 | } |
| 186 | 213 | } |
| 187 | 214 | |
| 188 | | −#ifdef CONFIG_OPROFILE_IBS |
| 215 | +/* |
| 216 | + * 16-bit Linear Feedback Shift Register (LFSR) |
| 217 | + * |
| 218 | + * 16 14 13 11 |
| 219 | + * Feedback polynomial = X + X + X + X + 1 |
| 220 | + */ |
| 221 | +static unsigned int lfsr_random(void) |
| 222 | +{ |
| 223 | + static unsigned int lfsr_value = 0xF00D; |
| 224 | + unsigned int bit; |
| 225 | + |
| 226 | + /* Compute next bit to shift in */ |
| 227 | + bit = ((lfsr_value >> 0) ^ |
| 228 | + (lfsr_value >> 2) ^ |
| 229 | + (lfsr_value >> 3) ^ |
| 230 | + (lfsr_value >> 5)) & 0x0001; |
| 231 | + |
| 232 | + /* Advance to next register value */ |
| 233 | + lfsr_value = (lfsr_value >> 1) | (bit << 15); |
| 234 | + |
| 235 | + return lfsr_value; |
| 236 | +} |
| 237 | + |
| 238 | +/* |
| 239 | + * IBS software randomization |
| 240 | + * |
| 241 | + * The IBS periodic op counter is randomized in software. The lower 12 |
| 242 | + * bits of the 20 bit counter are randomized. IbsOpCurCnt is |
| 243 | + * initialized with a 12 bit random value. |
| 244 | + */ |
| 245 | +static inline u64 op_amd_randomize_ibs_op(u64 val) |
| 246 | +{ |
| 247 | + unsigned int random = lfsr_random(); |
| 248 | + |
| 249 | + if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) |
| 250 | + /* |
| 251 | + * Work around if the hw can not write to IbsOpCurCnt |
| 252 | + * |
| 253 | + * Randomize the lower 8 bits of the 16 bit |
| 254 | + * IbsOpMaxCnt [15:0] value in the range of -128 to |
| 255 | + * +127 by adding/subtracting an offset to the |
| 256 | + * maximum count (IbsOpMaxCnt). |
| 257 | + * |
| 258 | + * To avoid over or underflows and protect upper bits |
| 259 | + * starting at bit 16, the initial value for |
| 260 | + * IbsOpMaxCnt must fit in the range from 0x0081 to |
| 261 | + * 0xff80. |
| 262 | + */ |
| 263 | + val += (s8)(random >> 4); |
| 264 | + else |
| 265 | + val |= (u64)(random & IBS_RANDOM_MASK) << 32; |
| 266 | + |
| 267 | + return val; |
| 268 | +} |
| 189 | 269 | |
| 190 | 270 | static inline void |
| 191 | 271 | op_amd_handle_ibs(struct pt_regs * const regs, |
| @@ -194,7 +274,7 @@ op_amd_handle_ibs(struct pt_regs * const regs, · 2 unmodified lines |
| 194 | 274 | u64 val, ctl; |
| 195 | 275 | struct op_entry entry; |
| 196 | 276 | |
| 197 | | − if (!has_ibs) |
| 277 | + if (!ibs_caps) |
| 198 | 278 | return; |
| 199 | 279 | |
| 200 | 280 | if (ibs_config.fetch_enabled) { |
| @@ -236,8 +316,7 @@ op_amd_handle_ibs(struct pt_regs * const regs, · 35 unmodified lines |
| 236 | 316 | oprofile_write_commit(&entry); |
| 237 | 317 | |
| 238 | 318 | /* reenable the IRQ */ |
| 239 | | − ctl &= ~IBS_OP_VAL & 0xFFFFFFFF; |
| 240 | | − ctl |= IBS_OP_ENABLE; |
| 319 | + ctl = op_amd_randomize_ibs_op(ibs_op_ctl); |
| 241 | 320 | wrmsrl(MSR_AMD64_IBSOPCTL, ctl); |
| 242 | 321 | } |
| 243 | 322 | } |
| @@ -246,41 +325,57 @@ op_amd_handle_ibs(struct pt_regs * const regs, · 2 unmodified lines |
| 246 | 325 | static inline void op_amd_start_ibs(void) |
| 247 | 326 | { |
| 248 | 327 | u64 val; |
| 249 | | − if (has_ibs && ibs_config.fetch_enabled) { |
| 328 | + |
| 329 | + if (!ibs_caps) |
| 330 | + return; |
| 331 | + |
| 332 | + if (ibs_config.fetch_enabled) { |
| 250 | 333 | val = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF; |
| 251 | 334 | val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0; |
| 252 | 335 | val |= IBS_FETCH_ENABLE; |
| 253 | 336 | wrmsrl(MSR_AMD64_IBSFETCHCTL, val); |
| 254 | 337 | } |
| 255 | 338 | |
| 256 | | − if (has_ibs && ibs_config.op_enabled) { |
| 257 | | − val = (ibs_config.max_cnt_op >> 4) & 0xFFFF; |
| 258 | | − val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0; |
| 259 | | − val |= IBS_OP_ENABLE; |
| 339 | + if (ibs_config.op_enabled) { |
| 340 | + ibs_op_ctl = ibs_config.max_cnt_op >> 4; |
| 341 | + if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) { |
| 342 | + /* |
| 343 | + * IbsOpCurCnt not supported. See |
| 344 | + * op_amd_randomize_ibs_op() for details. |
| 345 | + */ |
| 346 | + ibs_op_ctl = clamp(ibs_op_ctl, 0x0081ULL, 0xFF80ULL); |
| 347 | + } else { |
| 348 | + /* |
| 349 | + * The start value is randomized with a |
| 350 | + * positive offset, we need to compensate it |
| 351 | + * with the half of the randomized range. Also |
| 352 | + * avoid underflows. |
| 353 | + */ |
| 354 | + ibs_op_ctl = min(ibs_op_ctl + IBS_RANDOM_MAXCNT_OFFSET, |
| 355 | + 0xFFFFULL); |
| 356 | + } |
| 357 | + if (ibs_caps & IBS_CAPS_OPCNT && ibs_config.dispatched_ops) |
| 358 | + ibs_op_ctl |= IBS_OP_CNT_CTL; |
| 359 | + ibs_op_ctl |= IBS_OP_ENABLE; |
| 360 | + val = op_amd_randomize_ibs_op(ibs_op_ctl); |
| 260 | 361 | wrmsrl(MSR_AMD64_IBSOPCTL, val); |
| 261 | 362 | } |
| 262 | 363 | } |
| 263 | 364 | |
| 264 | 365 | static void op_amd_stop_ibs(void) |
| 265 | 366 | { |
| 266 | | − if (has_ibs && ibs_config.fetch_enabled) |
| 367 | + if (!ibs_caps) |
| 368 | + return; |
| 369 | + |
| 370 | + if (ibs_config.fetch_enabled) |
| 267 | 371 | /* clear max count and enable */ |
| 268 | 372 | wrmsrl(MSR_AMD64_IBSFETCHCTL, 0); |
| 269 | 373 | |
| 270 | | − if (has_ibs && ibs_config.op_enabled) |
| 374 | + if (ibs_config.op_enabled) |
| 271 | 375 | /* clear max count and enable */ |
| 272 | 376 | wrmsrl(MSR_AMD64_IBSOPCTL, 0); |
| 273 | 377 | } |
| 274 | 378 | |
| 275 | | −#else |
| 276 | | − |
| 277 | | −static inline void op_amd_handle_ibs(struct pt_regs * const regs, |
| 278 | | − struct op_msrs const * const msrs) { } |
| 279 | | −static inline void op_amd_start_ibs(void) { } |
| 280 | | −static inline void op_amd_stop_ibs(void) { } |
| 281 | | − |
| 282 | | −#endif |
| 283 | | − |
| 284 | 379 | static int op_amd_check_ctrs(struct pt_regs * const regs, |
| 285 | 380 | struct op_msrs const * const msrs) |
| 286 | 381 | { |
| @@ -355,8 +450,6 @@ static void op_amd_shutdown(struct op_msrs const * const msrs) · 68 unmodified lines |
| 355 | 450 | } |
| 356 | 451 | } |
| 357 | 452 | |
| 358 | | −#ifdef CONFIG_OPROFILE_IBS |
| 359 | | − |
| 360 | 453 | static u8 ibs_eilvt_off; |
| 361 | 454 | |
| 362 | 455 | static inline void apic_init_ibs_nmi_per_cpu(void *arg) |
| @@ -405,45 +498,36 @@ static int init_ibs_nmi(void) · 42 unmodified lines |
| 405 | 498 | return 1; |
| 406 | 499 | } |
| 407 | 500 | |
| 408 | | −#ifdef CONFIG_NUMA |
| 409 | | − /* Sanity check */ |
| 410 | | − /* Works only for 64bit with proper numa implementation. */ |
| 411 | | − if (nodes != num_possible_nodes()) { |
| 412 | | − printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, " |
| 413 | | − "found: %d, expected %d", |
| 414 | | − nodes, num_possible_nodes()); |
| 415 | | − return 1; |
| 416 | | − } |
| 417 | | −#endif |
| 418 | 501 | return 0; |
| 419 | 502 | } |
| 420 | 503 | |
| 421 | 504 | /* uninitialize the APIC for the IBS interrupts if needed */ |
| 422 | 505 | static void clear_ibs_nmi(void) |
| 423 | 506 | { |
| 424 | | − if (has_ibs) |
| 507 | + if (ibs_caps) |
| 425 | 508 | on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1); |
| 426 | 509 | } |
| 427 | 510 | |
| 428 | 511 | /* initialize the APIC for the IBS interrupts if available */ |
| 429 | 512 | static void ibs_init(void) |
| 430 | 513 | { |
| 431 | | − has_ibs = boot_cpu_has(X86_FEATURE_IBS); |
| 514 | + ibs_caps = get_ibs_caps(); |
| 432 | 515 | |
| 433 | | − if (!has_ibs) |
| 516 | + if (!ibs_caps) |
| 434 | 517 | return; |
| 435 | 518 | |
| 436 | 519 | if (init_ibs_nmi()) { |
| 437 | | − has_ibs = 0; |
| 520 | + ibs_caps = 0; |
| 438 | 521 | return; |
| 439 | 522 | } |
| 440 | 523 | |
| 441 | | − printk(KERN_INFO "oprofile: AMD IBS detected\n"); |
| 524 | + printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", |
| 525 | + (unsigned)ibs_caps); |
| 442 | 526 | } |
| 443 | 527 | |
| 444 | 528 | static void ibs_exit(void) |
| 445 | 529 | { |
| 446 | | − if (!has_ibs) |
| 530 | + if (!ibs_caps) |
| 447 | 531 | return; |
| 448 | 532 | |
| 449 | 533 | clear_ibs_nmi(); |
| @@ -463,7 +547,7 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root) · 13 unmodified lines |
| 463 | 547 | if (ret) |
| 464 | 548 | return ret; |
| 465 | 549 | |
| 466 | | − if (!has_ibs) |
| 550 | + if (!ibs_caps) |
| 467 | 551 | return ret; |
| 468 | 552 | |
| 469 | 553 | /* model specific files */ |
| @@ -473,7 +557,7 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root) · 3 unmodified lines |
| 473 | 557 | ibs_config.fetch_enabled = 0; |
| 474 | 558 | ibs_config.max_cnt_op = 250000; |
| 475 | 559 | ibs_config.op_enabled = 0; |
| 476 | | − ibs_config.dispatched_ops = 1; |
| 560 | + ibs_config.dispatched_ops = 0; |
| 477 | 561 | |
| 478 | 562 | dir = oprofilefs_mkdir(sb, root, "ibs_fetch"); |
| 479 | 563 | oprofilefs_create_ulong(sb, dir, "enable", |
| @@ -488,8 +572,9 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root) · 8 unmodified lines |
| 488 | 572 | &ibs_config.op_enabled); |
| 489 | 573 | oprofilefs_create_ulong(sb, dir, "max_count", |
| 490 | 574 | &ibs_config.max_cnt_op); |
| 491 | | − oprofilefs_create_ulong(sb, dir, "dispatched_ops", |
| 492 | | − &ibs_config.dispatched_ops); |
| 575 | + if (ibs_caps & IBS_CAPS_OPCNT) |
| 576 | + oprofilefs_create_ulong(sb, dir, "dispatched_ops", |
| 577 | + &ibs_config.dispatched_ops); |
| 493 | 578 | |
| 494 | 579 | return 0; |
| 495 | 580 | } |
| @@ -507,19 +592,6 @@ static void op_amd_exit(void) · 11 unmodified lines |
| 507 | 592 | ibs_exit(); |
| 508 | 593 | } |
| 509 | 594 | |
| 510 | | −#else |
| 511 | | − |
| 512 | | −/* no IBS support */ |
| 513 | | − |
| 514 | | −static int op_amd_init(struct oprofile_operations *ops) |
| 515 | | −{ |
| 516 | | − return 0; |
| 517 | | −} |
| 518 | | − |
| 519 | | −static void op_amd_exit(void) {} |
| 520 | | − |
| 521 | | −#endif /* CONFIG_OPROFILE_IBS */ |
| 522 | | − |
| 523 | 595 | struct op_x86_model_spec op_amd_spec = { |
| 524 | 596 | .num_counters = NUM_COUNTERS, |
| 525 | 597 | .num_controls = NUM_CONTROLS, |