perf_counter.c 27 KB
Newer Older
1 2 3 4 5
/*
 * Performance counter x86 architecture code
 *
 *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
 *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6
 *  Copyright(C) 2009 Jaswinder Singh Rajput
7 8 9 10 11 12 13 14 15
 *
 *  For licencing details see kernel-base/COPYING
 */

#include <linux/perf_counter.h>
#include <linux/capability.h>
#include <linux/notifier.h>
#include <linux/hardirq.h>
#include <linux/kprobes.h>
16
#include <linux/module.h>
17 18
#include <linux/kdebug.h>
#include <linux/sched.h>
19
#include <linux/uaccess.h>
20 21

#include <asm/apic.h>
22
#include <asm/stacktrace.h>
Peter Zijlstra's avatar
Peter Zijlstra committed
23
#include <asm/nmi.h>
24 25 26 27 28 29

static bool perf_counters_initialized __read_mostly;

/*
 * Number of (generic) HW counters:
 */
30 31
static int nr_counters_generic __read_mostly;
static u64 perf_counter_mask __read_mostly;
32
static u64 counter_value_mask __read_mostly;
33
static int counter_value_bits __read_mostly;
34

35
static int nr_counters_fixed __read_mostly;
36

37
struct cpu_hw_counters {
38 39
	struct perf_counter	*counters[X86_PMC_IDX_MAX];
	unsigned long		used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
40
	unsigned long		interrupts;
41
	u64			throttle_ctrl;
Peter Zijlstra's avatar
Peter Zijlstra committed
42
	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
43
	int			enabled;
44 45 46
};

/*
47
 * struct pmc_x86_ops - performance counter x86 ops
48
 */
49
struct pmc_x86_ops {
50
	u64		(*save_disable_all)(void);
51 52 53 54 55
	void		(*restore_all)(u64);
	u64		(*get_status)(u64);
	void		(*ack_status)(u64);
	void		(*enable)(int, u64);
	void		(*disable)(int, u64);
56 57
	unsigned	eventsel;
	unsigned	perfctr;
58 59
	u64		(*event_map)(int);
	u64		(*raw_event)(u64);
60
	int		max_events;
61 62
};

63
static struct pmc_x86_ops *pmc_ops __read_mostly;
64

65 66 67
static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
	.enabled = 1,
};
68

69 70
static __read_mostly int intel_perfmon_version;

71 72 73
/*
 * Intel PerfMon v3. Used on Core2 and later.
 */
74
static const u64 intel_perfmon_event_map[] =
75
{
76
  [PERF_COUNT_CPU_CYCLES]		= 0x003c,
77 78 79 80 81
  [PERF_COUNT_INSTRUCTIONS]		= 0x00c0,
  [PERF_COUNT_CACHE_REFERENCES]		= 0x4f2e,
  [PERF_COUNT_CACHE_MISSES]		= 0x412e,
  [PERF_COUNT_BRANCH_INSTRUCTIONS]	= 0x00c4,
  [PERF_COUNT_BRANCH_MISSES]		= 0x00c5,
82
  [PERF_COUNT_BUS_CYCLES]		= 0x013c,
83 84
};

85
static u64 pmc_intel_event_map(int event)
86 87 88
{
	return intel_perfmon_event_map[event];
}
89

90 91
static u64 pmc_intel_raw_event(u64 event)
{
92 93 94
#define CORE_EVNTSEL_EVENT_MASK		0x000000FFULL
#define CORE_EVNTSEL_UNIT_MASK		0x0000FF00ULL
#define CORE_EVNTSEL_COUNTER_MASK	0xFF000000ULL
95 96 97 98 99 100 101 102 103

#define CORE_EVNTSEL_MASK 		\
	(CORE_EVNTSEL_EVENT_MASK |	\
	 CORE_EVNTSEL_UNIT_MASK  |	\
	 CORE_EVNTSEL_COUNTER_MASK)

	return event & CORE_EVNTSEL_MASK;
}

104 105 106
/*
 * AMD Performance Monitor K7 and later.
 */
107
static const u64 amd_perfmon_event_map[] =
108 109 110 111 112 113 114 115 116
{
  [PERF_COUNT_CPU_CYCLES]		= 0x0076,
  [PERF_COUNT_INSTRUCTIONS]		= 0x00c0,
  [PERF_COUNT_CACHE_REFERENCES]		= 0x0080,
  [PERF_COUNT_CACHE_MISSES]		= 0x0081,
  [PERF_COUNT_BRANCH_INSTRUCTIONS]	= 0x00c4,
  [PERF_COUNT_BRANCH_MISSES]		= 0x00c5,
};

117
static u64 pmc_amd_event_map(int event)
118 119 120 121
{
	return amd_perfmon_event_map[event];
}

122 123
static u64 pmc_amd_raw_event(u64 event)
{
124 125 126
#define K7_EVNTSEL_EVENT_MASK	0x7000000FFULL
#define K7_EVNTSEL_UNIT_MASK	0x00000FF00ULL
#define K7_EVNTSEL_COUNTER_MASK	0x0FF000000ULL
127 128 129 130 131 132 133 134 135

#define K7_EVNTSEL_MASK			\
	(K7_EVNTSEL_EVENT_MASK |	\
	 K7_EVNTSEL_UNIT_MASK  |	\
	 K7_EVNTSEL_COUNTER_MASK)

	return event & K7_EVNTSEL_MASK;
}

136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
/*
 * Propagate counter elapsed time into the generic counter.
 * Can only be executed on the CPU where the counter is active.
 * Returns the delta events processed.
 */
static void
x86_perf_counter_update(struct perf_counter *counter,
			struct hw_perf_counter *hwc, int idx)
{
	u64 prev_raw_count, new_raw_count, delta;

	/*
	 * Careful: an NMI might modify the previous counter value.
	 *
	 * Our tactic to handle this is to first atomically read and
	 * exchange a new raw count - then add that new-prev delta
	 * count to the generic counter atomically:
	 */
again:
	prev_raw_count = atomic64_read(&hwc->prev_count);
	rdmsrl(hwc->counter_base + idx, new_raw_count);

	if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
					new_raw_count) != prev_raw_count)
		goto again;

	/*
	 * Now we have the new raw value and have updated the prev
	 * timestamp already. We can now calculate the elapsed delta
	 * (counter-)time and add that to the generic counter.
	 *
	 * Careful, not all hw sign-extends above the physical width
	 * of the count, so we do that by clipping the delta to 32 bits:
	 */
	delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);

	atomic64_add(delta, &counter->count);
	atomic64_sub(delta, &hwc->period_left);
}

Peter Zijlstra's avatar
Peter Zijlstra committed
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
static atomic_t num_counters;
static DEFINE_MUTEX(pmc_reserve_mutex);

static bool reserve_pmc_hardware(void)
{
	int i;

	if (nmi_watchdog == NMI_LOCAL_APIC)
		disable_lapic_nmi_watchdog();

	for (i = 0; i < nr_counters_generic; i++) {
		if (!reserve_perfctr_nmi(pmc_ops->perfctr + i))
			goto perfctr_fail;
	}

	for (i = 0; i < nr_counters_generic; i++) {
		if (!reserve_evntsel_nmi(pmc_ops->eventsel + i))
			goto eventsel_fail;
	}

	return true;

eventsel_fail:
	for (i--; i >= 0; i--)
		release_evntsel_nmi(pmc_ops->eventsel + i);

	i = nr_counters_generic;

perfctr_fail:
	for (i--; i >= 0; i--)
		release_perfctr_nmi(pmc_ops->perfctr + i);

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();

	return false;
}

static void release_pmc_hardware(void)
{
	int i;

	for (i = 0; i < nr_counters_generic; i++) {
		release_perfctr_nmi(pmc_ops->perfctr + i);
		release_evntsel_nmi(pmc_ops->eventsel + i);
	}

	if (nmi_watchdog == NMI_LOCAL_APIC)
		enable_lapic_nmi_watchdog();
}

static void hw_perf_counter_destroy(struct perf_counter *counter)
{
	if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
		release_pmc_hardware();
		mutex_unlock(&pmc_reserve_mutex);
	}
}

235 236 237
/*
 * Setup the hardware configuration for a given hw_event_type
 */
Ingo Molnar's avatar
Ingo Molnar committed
238
static int __hw_perf_counter_init(struct perf_counter *counter)
239
{
240
	struct perf_counter_hw_event *hw_event = &counter->hw_event;
241
	struct hw_perf_counter *hwc = &counter->hw;
Peter Zijlstra's avatar
Peter Zijlstra committed
242
	int err;
243 244 245 246

	if (unlikely(!perf_counters_initialized))
		return -EINVAL;

Peter Zijlstra's avatar
Peter Zijlstra committed
247 248 249 250 251 252 253 254 255 256 257 258
	err = 0;
	if (atomic_inc_not_zero(&num_counters)) {
		mutex_lock(&pmc_reserve_mutex);
		if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
			err = -EBUSY;
		else
			atomic_inc(&num_counters);
		mutex_unlock(&pmc_reserve_mutex);
	}
	if (err)
		return err;

259
	/*
260
	 * Generate PMC IRQs:
261 262
	 * (keep 'enabled' bit clear for now)
	 */
263
	hwc->config = ARCH_PERFMON_EVENTSEL_INT;
264 265

	/*
266
	 * Count user and OS events unless requested not to.
267
	 */
268 269 270
	if (!hw_event->exclude_user)
		hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
	if (!hw_event->exclude_kernel)
271
		hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
272 273 274 275 276 277 278

	/*
	 * If privileged enough, allow NMI events:
	 */
	hwc->nmi = 0;
	if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
		hwc->nmi = 1;
279

280
	hwc->irq_period		= hw_event->irq_period;
281 282 283 284 285
	/*
	 * Intel PMCs cannot be accessed sanely above 32 bit width,
	 * so we install an artificial 1<<31 period regardless of
	 * the generic counter period:
	 */
286 287 288
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
			hwc->irq_period = 0x7FFFFFFF;
289

290
	atomic64_set(&hwc->period_left, hwc->irq_period);
291 292

	/*
293
	 * Raw event type provide the config in the event structure
294
	 */
295 296
	if (perf_event_raw(hw_event)) {
		hwc->config |= pmc_ops->raw_event(perf_event_config(hw_event));
297
	} else {
298
		if (perf_event_id(hw_event) >= pmc_ops->max_events)
299 300 301 302
			return -EINVAL;
		/*
		 * The generic map:
		 */
303
		hwc->config |= pmc_ops->event_map(perf_event_id(hw_event));
304 305
	}

Peter Zijlstra's avatar
Peter Zijlstra committed
306 307
	counter->destroy = hw_perf_counter_destroy;

308 309 310
	return 0;
}

311
static u64 pmc_intel_save_disable_all(void)
312 313 314 315
{
	u64 ctrl;

	rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
316
	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
317

318
	return ctrl;
319
}
320

321 322
static u64 pmc_amd_save_disable_all(void)
{
323 324 325 326 327
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	int enabled, idx;

	enabled = cpuc->enabled;
	cpuc->enabled = 0;
328 329 330 331
	/*
	 * ensure we write the disable before we start disabling the
	 * counters proper, so that pcm_amd_enable() does the right thing.
	 */
332
	barrier();
333 334

	for (idx = 0; idx < nr_counters_generic; idx++) {
335 336
		u64 val;

337
		rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
338 339 340 341
		if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) {
			val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
			wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
		}
342 343
	}

344
	return enabled;
345 346
}

347 348 349 350 351 352 353
u64 hw_perf_save_disable(void)
{
	if (unlikely(!perf_counters_initialized))
		return 0;

	return pmc_ops->save_disable_all();
}
354 355 356
/*
 * Exported because of ACPI idle
 */
357
EXPORT_SYMBOL_GPL(hw_perf_save_disable);
358

359 360 361 362 363
static void pmc_intel_restore_all(u64 ctrl)
{
	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
}

364 365
static void pmc_amd_restore_all(u64 ctrl)
{
366
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
367 368
	int idx;

369 370 371 372 373
	cpuc->enabled = ctrl;
	barrier();
	if (!ctrl)
		return;

374
	for (idx = 0; idx < nr_counters_generic; idx++) {
Peter Zijlstra's avatar
Peter Zijlstra committed
375
		if (test_bit(idx, cpuc->active_mask)) {
376 377
			u64 val;

378 379 380 381 382 383 384
			rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
			val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
			wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
		}
	}
}

385 386
void hw_perf_restore(u64 ctrl)
{
387 388 389
	if (unlikely(!perf_counters_initialized))
		return;

390
	pmc_ops->restore_all(ctrl);
391
}
392 393 394
/*
 * Exported because of ACPI idle
 */
395 396
EXPORT_SYMBOL_GPL(hw_perf_restore);

397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
static u64 pmc_intel_get_status(u64 mask)
{
	u64 status;

	rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);

	return status;
}

static u64 pmc_amd_get_status(u64 mask)
{
	u64 status = 0;
	int idx;

	for (idx = 0; idx < nr_counters_generic; idx++) {
		s64 val;

		if (!(mask & (1 << idx)))
			continue;

		rdmsrl(MSR_K7_PERFCTR0 + idx, val);
		val <<= (64 - counter_value_bits);
		if (val >= 0)
			status |= (1 << idx);
	}

	return status;
}

static u64 hw_perf_get_status(u64 mask)
{
	if (unlikely(!perf_counters_initialized))
		return 0;

	return pmc_ops->get_status(mask);
}

static void pmc_intel_ack_status(u64 ack)
{
	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
}

static void pmc_amd_ack_status(u64 ack)
{
}

static void hw_perf_ack_status(u64 ack)
{
	if (unlikely(!perf_counters_initialized))
		return;

	pmc_ops->ack_status(ack);
}

static void pmc_intel_enable(int idx, u64 config)
{
	wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
			config | ARCH_PERFMON_EVENTSEL0_ENABLE);
}

static void pmc_amd_enable(int idx, u64 config)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);

Peter Zijlstra's avatar
Peter Zijlstra committed
461
	set_bit(idx, cpuc->active_mask);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
	if (cpuc->enabled)
		config |= ARCH_PERFMON_EVENTSEL0_ENABLE;

	wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
}

static void hw_perf_enable(int idx, u64 config)
{
	if (unlikely(!perf_counters_initialized))
		return;

	pmc_ops->enable(idx, config);
}

static void pmc_intel_disable(int idx, u64 config)
{
	wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
}

static void pmc_amd_disable(int idx, u64 config)
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);

Peter Zijlstra's avatar
Peter Zijlstra committed
485
	clear_bit(idx, cpuc->active_mask);
486 487 488 489 490 491 492 493 494 495 496 497
	wrmsrl(MSR_K7_EVNTSEL0 + idx, config);

}

static void hw_perf_disable(int idx, u64 config)
{
	if (unlikely(!perf_counters_initialized))
		return;

	pmc_ops->disable(idx, config);
}

498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
static inline void
__pmc_fixed_disable(struct perf_counter *counter,
		    struct hw_perf_counter *hwc, unsigned int __idx)
{
	int idx = __idx - X86_PMC_IDX_FIXED;
	u64 ctrl_val, mask;
	int err;

	mask = 0xfULL << (idx * 4);

	rdmsrl(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	err = checking_wrmsrl(hwc->config_base, ctrl_val);
}

513
static inline void
514
__pmc_generic_disable(struct perf_counter *counter,
515
			   struct hw_perf_counter *hwc, unsigned int idx)
516
{
517
	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
518 519
		__pmc_fixed_disable(counter, hwc, idx);
	else
520
		hw_perf_disable(idx, hwc->config);
521 522
}

523
static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
524

525 526 527 528 529 530 531
/*
 * Set the next IRQ period, based on the hwc->period_left value.
 * To be called with the counter disabled in hw:
 */
static void
__hw_perf_counter_set_period(struct perf_counter *counter,
			     struct hw_perf_counter *hwc, int idx)
532
{
533
	s64 left = atomic64_read(&hwc->period_left);
534
	s64 period = hwc->irq_period;
535
	int err;
536 537 538 539 540 541 542 543 544 545 546 547 548

	/*
	 * If we are way outside a reasoable range then just skip forward:
	 */
	if (unlikely(left <= -period)) {
		left = period;
		atomic64_set(&hwc->period_left, left);
	}

	if (unlikely(left <= 0)) {
		left += period;
		atomic64_set(&hwc->period_left, left);
	}
549

550 551 552 553 554 555
	per_cpu(prev_left[idx], smp_processor_id()) = left;

	/*
	 * The hw counter starts counting from this counter offset,
	 * mark it to be able to extra future deltas:
	 */
556
	atomic64_set(&hwc->prev_count, (u64)-left);
557

558 559 560 561 562 563 564 565 566 567 568 569 570
	err = checking_wrmsrl(hwc->counter_base + idx,
			     (u64)(-left) & counter_value_mask);
}

static inline void
__pmc_fixed_enable(struct perf_counter *counter,
		   struct hw_perf_counter *hwc, unsigned int __idx)
{
	int idx = __idx - X86_PMC_IDX_FIXED;
	u64 ctrl_val, bits, mask;
	int err;

	/*
571 572 573
	 * Enable IRQ generation (0x8),
	 * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
	 * if requested:
574
	 */
575 576 577
	bits = 0x8ULL;
	if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
		bits |= 0x2;
578 579 580 581 582 583 584 585 586
	if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
		bits |= 0x1;
	bits <<= (idx * 4);
	mask = 0xfULL << (idx * 4);

	rdmsrl(hwc->config_base, ctrl_val);
	ctrl_val &= ~mask;
	ctrl_val |= bits;
	err = checking_wrmsrl(hwc->config_base, ctrl_val);
587 588
}

589
static void
590
__pmc_generic_enable(struct perf_counter *counter,
591
			  struct hw_perf_counter *hwc, int idx)
592
{
593
	if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
594 595
		__pmc_fixed_enable(counter, hwc, idx);
	else
596
		hw_perf_enable(idx, hwc->config);
597 598
}

599 600
static int
fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
601
{
602 603
	unsigned int event;

604 605 606
	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
		return -1;

607 608 609 610 611
	if (unlikely(hwc->nmi))
		return -1;

	event = hwc->config & ARCH_PERFMON_EVENT_MASK;

612
	if (unlikely(event == pmc_ops->event_map(PERF_COUNT_INSTRUCTIONS)))
613
		return X86_PMC_IDX_FIXED_INSTRUCTIONS;
614
	if (unlikely(event == pmc_ops->event_map(PERF_COUNT_CPU_CYCLES)))
615
		return X86_PMC_IDX_FIXED_CPU_CYCLES;
616
	if (unlikely(event == pmc_ops->event_map(PERF_COUNT_BUS_CYCLES)))
617 618
		return X86_PMC_IDX_FIXED_BUS_CYCLES;

619 620 621
	return -1;
}

622 623 624
/*
 * Find a PMC slot for the freshly enabled / scheduled in counter:
 */
625
static int pmc_generic_enable(struct perf_counter *counter)
626 627 628
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
629
	int idx;
630

631 632 633 634 635 636 637 638
	idx = fixed_mode_idx(counter, hwc);
	if (idx >= 0) {
		/*
		 * Try to get the fixed counter, if that is already taken
		 * then try to get a generic counter:
		 */
		if (test_and_set_bit(idx, cpuc->used))
			goto try_generic;
639

640 641 642 643 644 645 646
		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
		/*
		 * We set it so that counter_base + idx in wrmsr/rdmsr maps to
		 * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
		 */
		hwc->counter_base =
			MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
647
		hwc->idx = idx;
648 649 650 651 652 653 654 655 656 657 658 659
	} else {
		idx = hwc->idx;
		/* Try to get the previous generic counter again */
		if (test_and_set_bit(idx, cpuc->used)) {
try_generic:
			idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
			if (idx == nr_counters_generic)
				return -EAGAIN;

			set_bit(idx, cpuc->used);
			hwc->idx = idx;
		}
660 661
		hwc->config_base  = pmc_ops->eventsel;
		hwc->counter_base = pmc_ops->perfctr;
662 663 664 665
	}

	perf_counters_lapic_init(hwc->nmi);

666
	__pmc_generic_disable(counter, hwc, idx);
667

668
	cpuc->counters[idx] = counter;
669 670 671 672
	/*
	 * Make it visible before enabling the hw:
	 */
	smp_wmb();
673

674
	__hw_perf_counter_set_period(counter, hwc, idx);
675
	__pmc_generic_enable(counter, hwc, idx);
676 677

	return 0;
678 679 680 681
}

void perf_counter_print_debug(void)
{
682
	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
683
	struct cpu_hw_counters *cpuc;
684 685
	int cpu, idx;

686
	if (!nr_counters_generic)
687
		return;
688 689 690 691

	local_irq_disable();

	cpu = smp_processor_id();
692
	cpuc = &per_cpu(cpu_hw_counters, cpu);
693

694
	if (intel_perfmon_version >= 2) {
695 696 697 698 699 700 701 702 703 704
		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);

		pr_info("\n");
		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
705
	}
706
	pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
707

708
	for (idx = 0; idx < nr_counters_generic; idx++) {
709 710
		rdmsrl(pmc_ops->eventsel + idx, pmc_ctrl);
		rdmsrl(pmc_ops->perfctr  + idx, pmc_count);
711

712
		prev_left = per_cpu(prev_left[idx], cpu);
713

714
		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
715
			cpu, idx, pmc_ctrl);
716
		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
717
			cpu, idx, pmc_count);
718
		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
719
			cpu, idx, prev_left);
720
	}
721 722 723
	for (idx = 0; idx < nr_counters_fixed; idx++) {
		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);

724
		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
725 726
			cpu, idx, pmc_count);
	}
727 728 729
	local_irq_enable();
}

730
static void pmc_generic_disable(struct perf_counter *counter)
731 732 733 734 735
{
	struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
	struct hw_perf_counter *hwc = &counter->hw;
	unsigned int idx = hwc->idx;

736
	__pmc_generic_disable(counter, hwc, idx);
737 738

	clear_bit(idx, cpuc->used);
739
	cpuc->counters[idx] = NULL;
740 741 742 743 744
	/*
	 * Make sure the cleared pointer becomes visible before we
	 * (potentially) free the counter:
	 */
	smp_wmb();
745

746 747 748 749 750
	/*
	 * Drain the remaining delta count out of a counter
	 * that we are disabling:
	 */
	x86_perf_counter_update(counter, hwc, idx);
751 752
}

753
/*
754 755
 * Save and restart an expired counter. Called by NMI contexts,
 * so it has to be careful about preempting normal counter ops:
756
 */
757 758 759 760 761
static void perf_save_and_restart(struct perf_counter *counter)
{
	struct hw_perf_counter *hwc = &counter->hw;
	int idx = hwc->idx;

762 763
	x86_perf_counter_update(counter, hwc, idx);
	__hw_perf_counter_set_period(counter, hwc, idx);
764

765
	if (counter->state == PERF_COUNTER_STATE_ACTIVE)
766
		__pmc_generic_enable(counter, hwc, idx);
767 768
}

769 770 771
/*
 * Maximum interrupt frequency of 100KHz per CPU
 */
772
#define PERFMON_MAX_INTERRUPTS (100000/HZ)
773

774 775 776 777
/*
 * This handler is triggered by the local APIC, so the APIC IRQ handling
 * rules apply:
 */
778
static int __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
779 780
{
	int bit, cpu = smp_processor_id();
781
	u64 ack, status;
782
	struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
783
	int ret = 0;
784

785
	cpuc->throttle_ctrl = hw_perf_save_disable();
786

787
	status = hw_perf_get_status(cpuc->throttle_ctrl);
788 789 790
	if (!status)
		goto out;

791
	ret = 1;
792
again:
793
	inc_irq_stat(apic_perf_irqs);
794
	ack = status;
795
	for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
796
		struct perf_counter *counter = cpuc->counters[bit];
797 798 799 800 801 802

		clear_bit(bit, (unsigned long *) &status);
		if (!counter)
			continue;

		perf_save_and_restart(counter);
803
		perf_counter_output(counter, nmi, regs);
804 805
	}

806
	hw_perf_ack_status(ack);
807 808 809 810

	/*
	 * Repeat if there is more work to be done:
	 */
811
	status = hw_perf_get_status(cpuc->throttle_ctrl);
812 813
	if (status)
		goto again;
814
out:
815
	/*
816
	 * Restore - do not reenable when global enable is off or throttled:
817
	 */
818
	if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
819 820 821
		hw_perf_restore(cpuc->throttle_ctrl);

	return ret;
822 823 824 825 826 827 828 829 830 831 832 833
}

void perf_counter_unthrottle(void)
{
	struct cpu_hw_counters *cpuc;

	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
		return;

	if (unlikely(!perf_counters_initialized))
		return;

834
	cpuc = &__get_cpu_var(cpu_hw_counters);
835
	if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
836
		if (printk_ratelimit())
837
			printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
838
		hw_perf_restore(cpuc->throttle_ctrl);
839
	}
840
	cpuc->interrupts = 0;
841 842 843 844 845 846
}

void smp_perf_counter_interrupt(struct pt_regs *regs)
{
	irq_enter();
	apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
847
	ack_APIC_irq();
848 849 850 851
	__smp_perf_counter_interrupt(regs, 0);
	irq_exit();
}

852
void perf_counters_lapic_init(int nmi)
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
{
	u32 apic_val;

	if (!perf_counters_initialized)
		return;
	/*
	 * Enable the performance counter vector in the APIC LVT:
	 */
	apic_val = apic_read(APIC_LVTERR);

	apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
	if (nmi)
		apic_write(APIC_LVTPC, APIC_DM_NMI);
	else
		apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
	apic_write(APIC_LVTERR, apic_val);
}

static int __kprobes
perf_counter_nmi_handler(struct notifier_block *self,
			 unsigned long cmd, void *__args)
{
	struct die_args *args = __args;
	struct pt_regs *regs;
877 878 879 880 881 882
	int ret;

	switch (cmd) {
	case DIE_NMI:
	case DIE_NMI_IPI:
		break;
883

884
	default:
885
		return NOTIFY_DONE;
886
	}
887 888 889 890

	regs = args->regs;

	apic_write(APIC_LVTPC, APIC_DM_NMI);
891
	ret = __smp_perf_counter_interrupt(regs, 1);
892

893
	return ret ? NOTIFY_STOP : NOTIFY_OK;
894 895 896
}

static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
897 898 899
	.notifier_call		= perf_counter_nmi_handler,
	.next			= NULL,
	.priority		= 1
900 901
};

902 903 904
static struct pmc_x86_ops pmc_intel_ops = {
	.save_disable_all	= pmc_intel_save_disable_all,
	.restore_all		= pmc_intel_restore_all,
905 906 907 908
	.get_status		= pmc_intel_get_status,
	.ack_status		= pmc_intel_ack_status,
	.enable			= pmc_intel_enable,
	.disable		= pmc_intel_disable,
909 910 911
	.eventsel		= MSR_ARCH_PERFMON_EVENTSEL0,
	.perfctr		= MSR_ARCH_PERFMON_PERFCTR0,
	.event_map		= pmc_intel_event_map,
912
	.raw_event		= pmc_intel_raw_event,
913 914 915
	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
};

916 917 918
static struct pmc_x86_ops pmc_amd_ops = {
	.save_disable_all	= pmc_amd_save_disable_all,
	.restore_all		= pmc_amd_restore_all,
919 920 921 922
	.get_status		= pmc_amd_get_status,
	.ack_status		= pmc_amd_ack_status,
	.enable			= pmc_amd_enable,
	.disable		= pmc_amd_disable,
923 924 925
	.eventsel		= MSR_K7_EVNTSEL0,
	.perfctr		= MSR_K7_PERFCTR0,
	.event_map		= pmc_amd_event_map,
926
	.raw_event		= pmc_amd_raw_event,
927 928 929
	.max_events		= ARRAY_SIZE(amd_perfmon_event_map),
};

930
static struct pmc_x86_ops *pmc_intel_init(void)
931
{
932
	union cpuid10_edx edx;
933
	union cpuid10_eax eax;
934
	unsigned int unused;
935
	unsigned int ebx;
936 937 938 939 940

	/*
	 * Check whether the Architectural PerfMon supports
	 * Branch Misses Retired Event or not.
	 */
941
	cpuid(10, &eax.full, &ebx, &unused, &edx.full);
942
	if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
943
		return NULL;
944

945 946 947 948
	intel_perfmon_version = eax.split.version_id;
	if (intel_perfmon_version < 2)
		return NULL;

949
	pr_info("Intel Performance Monitoring support detected.\n");
950
	pr_info("... version:         %d\n", intel_perfmon_version);
951 952
	pr_info("... bit width:       %d\n", eax.split.bit_width);
	pr_info("... mask length:     %d\n", eax.split.mask_length);
953

954
	nr_counters_generic = eax.split.num_counters;
955 956 957 958 959 960
	nr_counters_fixed = edx.split.num_counters_fixed;
	counter_value_mask = (1ULL << eax.split.bit_width) - 1;

	return &pmc_intel_ops;
}

961 962 963 964
static struct pmc_x86_ops *pmc_amd_init(void)
{
	nr_counters_generic = 4;
	nr_counters_fixed = 0;
965 966
	counter_value_mask = 0x0000FFFFFFFFFFFFULL;
	counter_value_bits = 48;
967

968
	pr_info("AMD Performance Monitoring support detected.\n");
969 970 971 972

	return &pmc_amd_ops;
}

973 974 975 976 977 978 979 980 981
void __init init_hw_perf_counters(void)
{
	if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
		return;

	switch (boot_cpu_data.x86_vendor) {
	case X86_VENDOR_INTEL:
		pmc_ops = pmc_intel_init();
		break;
982 983 984
	case X86_VENDOR_AMD:
		pmc_ops = pmc_amd_init();
		break;
985 986 987 988
	}
	if (!pmc_ops)
		return;

989
	pr_info("... num counters:    %d\n", nr_counters_generic);
990 991
	if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
		nr_counters_generic = X86_PMC_MAX_GENERIC;
992
		WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
993
			nr_counters_generic, X86_PMC_MAX_GENERIC);
994
	}
995 996
	perf_counter_mask = (1 << nr_counters_generic) - 1;
	perf_max_counters = nr_counters_generic;
997

998
	pr_info("... value mask:      %016Lx\n", counter_value_mask);
999

1000 1001
	if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
		nr_counters_fixed = X86_PMC_MAX_FIXED;
1002
		WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1003
			nr_counters_fixed, X86_PMC_MAX_FIXED);
1004
	}