Commit 1da53e02 authored by Stephane Eranian's avatar Stephane Eranian Committed by Ingo Molnar

perf_events, x86: Improve x86 event scheduling

This patch improves event scheduling by maximizing the use of PMU
registers regardless of the order in which events are created in a group.

The algorithm takes into account the list of counter constraints for each
event. It assigns events to counters from the most constrained, i.e.,
works on only one counter, to the least constrained, i.e., works on any
counter.

Intel Fixed counter events and the BTS special event are also handled via
this algorithm which is designed to be fairly generic.

The patch also updates the validation of an event to use the scheduling
algorithm. This will cause early failure in perf_event_open().

The 2nd version of this patch follows the model used by PPC, by running
the scheduling algorithm and the actual assignment separately. Actual
assignment takes place in hw_perf_enable() whereas scheduling is
implemented in hw_perf_group_sched_in() and x86_pmu_enable().
Signed-off-by: default avatarStephane Eranian <eranian@google.com>
[ fixup whitespace and style nits as well as adding is_x86_event() ]
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
LKML-Reference: <4b5430c6.0f975e0a.1bf9.ffff85fe@mx.google.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent e0e53db6
......@@ -26,7 +26,14 @@
/*
* Includes eventsel and unit mask as well:
*/
#define ARCH_PERFMON_EVENT_MASK 0xffff
#define INTEL_ARCH_EVTSEL_MASK 0x000000FFULL
#define INTEL_ARCH_UNIT_MASK 0x0000FF00ULL
#define INTEL_ARCH_EDGE_MASK 0x00040000ULL
#define INTEL_ARCH_INV_MASK 0x00800000ULL
#define INTEL_ARCH_CNT_MASK 0xFF000000ULL
#define INTEL_ARCH_EVENT_MASK (INTEL_ARCH_UNIT_MASK|INTEL_ARCH_EVTSEL_MASK)
/*
* filter mask to validate fixed counter events.
......@@ -37,7 +44,12 @@
* The other filters are supported by fixed counters.
* The any-thread option is supported starting with v3.
*/
#define ARCH_PERFMON_EVENT_FILTER_MASK 0xff840000
#define INTEL_ARCH_FIXED_MASK \
(INTEL_ARCH_CNT_MASK| \
INTEL_ARCH_INV_MASK| \
INTEL_ARCH_EDGE_MASK|\
INTEL_ARCH_UNIT_MASK|\
INTEL_ARCH_EVENT_MASK)
#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c
#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)
......
......@@ -7,6 +7,7 @@
* Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
* Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
* Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
* Copyright (C) 2009 Google, Inc., Stephane Eranian
*
* For licencing details see kernel-base/COPYING
*/
......@@ -68,26 +69,37 @@ struct debug_store {
u64 pebs_event_reset[MAX_PEBS_EVENTS];
};
#define BITS_TO_U64(nr) DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(u64))
struct event_constraint {
u64 idxmsk[BITS_TO_U64(X86_PMC_IDX_MAX)];
int code;
int cmask;
};
struct cpu_hw_events {
struct perf_event *events[X86_PMC_IDX_MAX];
unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
unsigned long interrupts;
int enabled;
struct debug_store *ds;
};
struct event_constraint {
unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
int code;
int n_events;
int n_added;
int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
};
#define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
#define EVENT_CONSTRAINT_END { .code = 0, .idxmsk[0] = 0 }
#define EVENT_CONSTRAINT(c, n, m) { \
.code = (c), \
.cmask = (m), \
.idxmsk[0] = (n) }
#define for_each_event_constraint(e, c) \
for ((e) = (c); (e)->idxmsk[0]; (e)++)
#define EVENT_CONSTRAINT_END \
{ .code = 0, .cmask = 0, .idxmsk[0] = 0 }
#define for_each_event_constraint(e, c) \
for ((e) = (c); (e)->cmask; (e)++)
/*
* struct x86_pmu - generic x86 pmu
......@@ -114,8 +126,9 @@ struct x86_pmu {
u64 intel_ctrl;
void (*enable_bts)(u64 config);
void (*disable_bts)(void);
int (*get_event_idx)(struct cpu_hw_events *cpuc,
struct hw_perf_event *hwc);
void (*get_event_constraints)(struct cpu_hw_events *cpuc, struct perf_event *event, u64 *idxmsk);
void (*put_event_constraints)(struct cpu_hw_events *cpuc, struct perf_event *event);
const struct event_constraint *event_constraints;
};
static struct x86_pmu x86_pmu __read_mostly;
......@@ -124,7 +137,8 @@ static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
.enabled = 1,
};
static const struct event_constraint *event_constraints;
static int x86_perf_event_set_period(struct perf_event *event,
struct hw_perf_event *hwc, int idx);
/*
* Not sure about some of these
......@@ -171,14 +185,14 @@ static u64 p6_pmu_raw_event(u64 hw_event)
return hw_event & P6_EVNTSEL_MASK;
}
static const struct event_constraint intel_p6_event_constraints[] =
static struct event_constraint intel_p6_event_constraints[] =
{
EVENT_CONSTRAINT(0xc1, 0x1), /* FLOPS */
EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
EVENT_CONSTRAINT(0x11, 0x1), /* FP_ASSIST */
EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
EVENT_CONSTRAINT(0xc1, 0x1, INTEL_ARCH_EVENT_MASK), /* FLOPS */
EVENT_CONSTRAINT(0x10, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_COMP_OPS_EXE */
EVENT_CONSTRAINT(0x11, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_ASSIST */
EVENT_CONSTRAINT(0x12, 0x2, INTEL_ARCH_EVENT_MASK), /* MUL */
EVENT_CONSTRAINT(0x13, 0x2, INTEL_ARCH_EVENT_MASK), /* DIV */
EVENT_CONSTRAINT(0x14, 0x1, INTEL_ARCH_EVENT_MASK), /* CYCLES_DIV_BUSY */
EVENT_CONSTRAINT_END
};
......@@ -196,32 +210,43 @@ static const u64 intel_perfmon_event_map[] =
[PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
};
static const struct event_constraint intel_core_event_constraints[] =
{
EVENT_CONSTRAINT(0x10, 0x1), /* FP_COMP_OPS_EXE */
EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
EVENT_CONSTRAINT(0x13, 0x2), /* DIV */
EVENT_CONSTRAINT(0x14, 0x1), /* CYCLES_DIV_BUSY */
EVENT_CONSTRAINT(0x18, 0x1), /* IDLE_DURING_DIV */
EVENT_CONSTRAINT(0x19, 0x2), /* DELAYED_BYPASS */
EVENT_CONSTRAINT(0xa1, 0x1), /* RS_UOPS_DISPATCH_CYCLES */
EVENT_CONSTRAINT(0xcb, 0x1), /* MEM_LOAD_RETIRED */
static struct event_constraint intel_core_event_constraints[] =
{
EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
EVENT_CONSTRAINT(0x10, 0x1, INTEL_ARCH_EVENT_MASK), /* FP_COMP_OPS_EXE */
EVENT_CONSTRAINT(0x11, 0x2, INTEL_ARCH_EVENT_MASK), /* FP_ASSIST */
EVENT_CONSTRAINT(0x12, 0x2, INTEL_ARCH_EVENT_MASK), /* MUL */
EVENT_CONSTRAINT(0x13, 0x2, INTEL_ARCH_EVENT_MASK), /* DIV */
EVENT_CONSTRAINT(0x14, 0x1, INTEL_ARCH_EVENT_MASK), /* CYCLES_DIV_BUSY */
EVENT_CONSTRAINT(0x18, 0x1, INTEL_ARCH_EVENT_MASK), /* IDLE_DURING_DIV */
EVENT_CONSTRAINT(0x19, 0x2, INTEL_ARCH_EVENT_MASK), /* DELAYED_BYPASS */
EVENT_CONSTRAINT(0xa1, 0x1, INTEL_ARCH_EVENT_MASK), /* RS_UOPS_DISPATCH_CYCLES */
EVENT_CONSTRAINT(0xcb, 0x1, INTEL_ARCH_EVENT_MASK), /* MEM_LOAD_RETIRED */
EVENT_CONSTRAINT_END
};
static const struct event_constraint intel_nehalem_event_constraints[] =
{
EVENT_CONSTRAINT(0x40, 0x3), /* L1D_CACHE_LD */
EVENT_CONSTRAINT(0x41, 0x3), /* L1D_CACHE_ST */
EVENT_CONSTRAINT(0x42, 0x3), /* L1D_CACHE_LOCK */
EVENT_CONSTRAINT(0x43, 0x3), /* L1D_ALL_REF */
EVENT_CONSTRAINT(0x4e, 0x3), /* L1D_PREFETCH */
EVENT_CONSTRAINT(0x4c, 0x3), /* LOAD_HIT_PRE */
EVENT_CONSTRAINT(0x51, 0x3), /* L1D */
EVENT_CONSTRAINT(0x52, 0x3), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
EVENT_CONSTRAINT(0x53, 0x3), /* L1D_CACHE_LOCK_FB_HIT */
EVENT_CONSTRAINT(0xc5, 0x3), /* CACHE_LOCK_CYCLES */
static struct event_constraint intel_nehalem_event_constraints[] =
{
EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
EVENT_CONSTRAINT(0x40, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LD */
EVENT_CONSTRAINT(0x41, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_ST */
EVENT_CONSTRAINT(0x42, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LOCK */
EVENT_CONSTRAINT(0x43, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_ALL_REF */
EVENT_CONSTRAINT(0x4e, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_PREFETCH */
EVENT_CONSTRAINT(0x4c, 0x3, INTEL_ARCH_EVENT_MASK), /* LOAD_HIT_PRE */
EVENT_CONSTRAINT(0x51, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D */
EVENT_CONSTRAINT(0x52, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
EVENT_CONSTRAINT(0x53, 0x3, INTEL_ARCH_EVENT_MASK), /* L1D_CACHE_LOCK_FB_HIT */
EVENT_CONSTRAINT(0xc5, 0x3, INTEL_ARCH_EVENT_MASK), /* CACHE_LOCK_CYCLES */
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_gen_event_constraints[] =
{
EVENT_CONSTRAINT(0xc0, (0x3|(1ULL<<32)), INTEL_ARCH_FIXED_MASK), /* INSTRUCTIONS_RETIRED */
EVENT_CONSTRAINT(0x3c, (0x3|(1ULL<<33)), INTEL_ARCH_FIXED_MASK), /* UNHALTED_CORE_CYCLES */
EVENT_CONSTRAINT_END
};
......@@ -527,11 +552,11 @@ static u64 intel_pmu_raw_event(u64 hw_event)
#define CORE_EVNTSEL_REG_MASK 0xFF000000ULL
#define CORE_EVNTSEL_MASK \
(CORE_EVNTSEL_EVENT_MASK | \
CORE_EVNTSEL_UNIT_MASK | \
CORE_EVNTSEL_EDGE_MASK | \
CORE_EVNTSEL_INV_MASK | \
CORE_EVNTSEL_REG_MASK)
(INTEL_ARCH_EVTSEL_MASK | \
INTEL_ARCH_UNIT_MASK | \
INTEL_ARCH_EDGE_MASK | \
INTEL_ARCH_INV_MASK | \
INTEL_ARCH_CNT_MASK)
return hw_event & CORE_EVNTSEL_MASK;
}
......@@ -1120,9 +1145,15 @@ static void amd_pmu_disable_all(void)
void hw_perf_disable(void)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
if (!x86_pmu_initialized())
return;
return x86_pmu.disable_all();
if (cpuc->enabled)
cpuc->n_added = 0;
x86_pmu.disable_all();
}
static void p6_pmu_enable_all(void)
......@@ -1189,10 +1220,237 @@ static void amd_pmu_enable_all(void)
}
}
static const struct pmu pmu;
static inline int is_x86_event(struct perf_event *event)
{
return event->pmu == &pmu;
}
static int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
int i, j , w, num;
int weight, wmax;
unsigned long *c;
u64 constraints[X86_PMC_IDX_MAX][BITS_TO_LONGS(X86_PMC_IDX_MAX)];
unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
struct hw_perf_event *hwc;
bitmap_zero(used_mask, X86_PMC_IDX_MAX);
for (i = 0; i < n; i++) {
x86_pmu.get_event_constraints(cpuc,
cpuc->event_list[i],
constraints[i]);
}
/*
* weight = number of possible counters
*
* 1 = most constrained, only works on one counter
* wmax = least constrained, works on any counter
*
* assign events to counters starting with most
* constrained events.
*/
wmax = x86_pmu.num_events;
/*
* when fixed event counters are present,
* wmax is incremented by 1 to account
* for one more choice
*/
if (x86_pmu.num_events_fixed)
wmax++;
num = n;
for (w = 1; num && w <= wmax; w++) {
/* for each event */
for (i = 0; i < n; i++) {
c = (unsigned long *)constraints[i];
hwc = &cpuc->event_list[i]->hw;
weight = bitmap_weight(c, X86_PMC_IDX_MAX);
if (weight != w)
continue;
/*
* try to reuse previous assignment
*
* This is possible despite the fact that
* events or events order may have changed.
*
* What matters is the level of constraints
* of an event and this is constant for now.
*
* This is possible also because we always
* scan from most to least constrained. Thus,
* if a counter can be reused, it means no,
* more constrained events, needed it. And
* next events will either compete for it
* (which cannot be solved anyway) or they
* have fewer constraints, and they can use
* another counter.
*/
j = hwc->idx;
if (j != -1 && !test_bit(j, used_mask))
goto skip;
for_each_bit(j, c, X86_PMC_IDX_MAX) {
if (!test_bit(j, used_mask))
break;
}
if (j == X86_PMC_IDX_MAX)
break;
skip:
set_bit(j, used_mask);
#if 0
pr_debug("CPU%d config=0x%llx idx=%d assign=%c\n",
smp_processor_id(),
hwc->config,
j,
assign ? 'y' : 'n');
#endif
if (assign)
assign[i] = j;
num--;
}
}
/*
* scheduling failed or is just a simulation,
* free resources if necessary
*/
if (!assign || num) {
for (i = 0; i < n; i++) {
if (x86_pmu.put_event_constraints)
x86_pmu.put_event_constraints(cpuc, cpuc->event_list[i]);
}
}
return num ? -ENOSPC : 0;
}
/*
* dogrp: true if must collect siblings events (group)
* returns total number of events and error code
*/
static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
{
struct perf_event *event;
int n, max_count;
max_count = x86_pmu.num_events + x86_pmu.num_events_fixed;
/* current number of events already accepted */
n = cpuc->n_events;
if (is_x86_event(leader)) {
if (n >= max_count)
return -ENOSPC;
cpuc->event_list[n] = leader;
n++;
}
if (!dogrp)
return n;
list_for_each_entry(event, &leader->sibling_list, group_entry) {
if (!is_x86_event(event) ||
event->state == PERF_EVENT_STATE_OFF)
continue;
if (n >= max_count)
return -ENOSPC;
cpuc->event_list[n] = event;
n++;
}
return n;
}
static inline void x86_assign_hw_event(struct perf_event *event,
struct hw_perf_event *hwc, int idx)
{
hwc->idx = idx;
if (hwc->idx == X86_PMC_IDX_FIXED_BTS) {
hwc->config_base = 0;
hwc->event_base = 0;
} else if (hwc->idx >= X86_PMC_IDX_FIXED) {
hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
/*
* We set it so that event_base + idx in wrmsr/rdmsr maps to
* MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
*/
hwc->event_base =
MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
} else {
hwc->config_base = x86_pmu.eventsel;
hwc->event_base = x86_pmu.perfctr;
}
}
void hw_perf_enable(void)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct perf_event *event;
struct hw_perf_event *hwc;
int i;
if (!x86_pmu_initialized())
return;
if (cpuc->n_added) {
/*
* apply assignment obtained either from
* hw_perf_group_sched_in() or x86_pmu_enable()
*
* step1: save events moving to new counters
* step2: reprogram moved events into new counters
*/
for (i = 0; i < cpuc->n_events; i++) {
event = cpuc->event_list[i];
hwc = &event->hw;
if (hwc->idx == -1 || hwc->idx == cpuc->assign[i])
continue;
x86_pmu.disable(hwc, hwc->idx);
clear_bit(hwc->idx, cpuc->active_mask);
barrier();
cpuc->events[hwc->idx] = NULL;
x86_perf_event_update(event, hwc, hwc->idx);
hwc->idx = -1;
}
for (i = 0; i < cpuc->n_events; i++) {
event = cpuc->event_list[i];
hwc = &event->hw;
if (hwc->idx == -1) {
x86_assign_hw_event(event, hwc, cpuc->assign[i]);
x86_perf_event_set_period(event, hwc, hwc->idx);
}
/*
* need to mark as active because x86_pmu_disable()
* clear active_mask and eventsp[] yet it preserves
* idx
*/
set_bit(hwc->idx, cpuc->active_mask);
cpuc->events[hwc->idx] = event;
x86_pmu.enable(hwc, hwc->idx);
perf_event_update_userpage(event);
}
cpuc->n_added = 0;
perf_events_lapic_init();
}
x86_pmu.enable_all();
}
......@@ -1391,148 +1649,43 @@ static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
x86_pmu_enable_event(hwc, idx);
}
static int fixed_mode_idx(struct hw_perf_event *hwc)
{
unsigned int hw_event;
hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
if (unlikely((hw_event ==
x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
(hwc->sample_period == 1)))
return X86_PMC_IDX_FIXED_BTS;
if (!x86_pmu.num_events_fixed)
return -1;
/*
* fixed counters do not take all possible filters
*/
if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
return -1;
if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
return X86_PMC_IDX_FIXED_INSTRUCTIONS;
if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
return X86_PMC_IDX_FIXED_CPU_CYCLES;
if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
return X86_PMC_IDX_FIXED_BUS_CYCLES;
return -1;
}
/*
* generic counter allocator: get next free counter
*/
static int
gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
{
int idx;
idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
return idx == x86_pmu.num_events ? -1 : idx;
}
/*
* intel-specific counter allocator: check event constraints
*/
static int
intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
{
const struct event_constraint *event_constraint;
int i, code;
if (!event_constraints)
goto skip;
code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
for_each_event_constraint(event_constraint, event_constraints) {
if (code == event_constraint->code) {
for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
if (!test_and_set_bit(i, cpuc->used_mask))
return i;
}
return -1;
}
}
skip:
return gen_get_event_idx(cpuc, hwc);
}
static int
x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
{
int idx;
idx = fixed_mode_idx(hwc);
if (idx == X86_PMC_IDX_FIXED_BTS) {
/* BTS is already occupied. */
if (test_and_set_bit(idx, cpuc->used_mask))
return -EAGAIN;
hwc->config_base = 0;
hwc->event_base = 0;
hwc->idx = idx;
} else if (idx >= 0) {
/*
* Try to get the fixed event, if that is already taken
* then try to get a generic event:
*/
if (test_and_set_bit(idx, cpuc->used_mask))
goto try_generic;
hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
/*
* We set it so that event_base + idx in wrmsr/rdmsr maps to
* MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
*/
hwc->event_base =
MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
hwc->idx = idx;
} else {
idx = hwc->idx;
/* Try to get the previous generic event again */
if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
try_generic:
idx = x86_pmu.get_event_idx(cpuc, hwc);
if (idx == -1)
return -EAGAIN;
set_bit(idx, cpuc->used_mask);
hwc->idx = idx;
}
hwc->config_base = x86_pmu.eventsel;
hwc->event_base = x86_pmu.perfctr;
}
return idx;
}
/*
* Find a PMC slot for the freshly enabled / scheduled in event:
* activate a single event
*
* The event is added to the group of enabled events
* but only if it can be scehduled with existing events.
*
* Called with PMU disabled. If successful and return value 1,
* then guaranteed to call perf_enable() and hw_perf_enable()
*/
static int x86_pmu_enable(struct perf_event *event)
{
struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
struct hw_perf_event *hwc = &event->hw;
int idx;
struct hw_perf_event *hwc;
int assign[X86_PMC_IDX_MAX];
int n, n0, ret;
idx = x86_schedule_event(cpuc, hwc);
if (idx < 0)
return idx;
hwc = &event->hw;
perf_events_lapic_init();
n0 = cpuc->n_events;
n = collect_events(cpuc, event, false);
if (n < 0)