perf_event.h 39.3 KB
Newer Older
1
/*
2
 * Performance events:
3
 *
Ingo Molnar's avatar
Ingo Molnar committed
4
 *    Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5 6
 *    Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
 *    Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
7
 *
8
 * Data type definitions, declarations, prototypes.
9
 *
Ingo Molnar's avatar
Ingo Molnar committed
10
 *    Started by: Thomas Gleixner and Ingo Molnar
11
 *
12
 * For licencing details see kernel-base/COPYING
13
 */
14 15
#ifndef _LINUX_PERF_EVENT_H
#define _LINUX_PERF_EVENT_H
16

17
#include <uapi/linux/perf_event.h>
18

19
/*
20
 * Kernel-internal data types and definitions:
21 22
 */

23 24
#ifdef CONFIG_PERF_EVENTS
# include <asm/perf_event.h>
25
# include <asm/local64.h>
26 27
#endif

28
struct perf_guest_info_callbacks {
29 30 31
	int				(*is_in_guest)(void);
	int				(*is_user_mode)(void);
	unsigned long			(*get_guest_ip)(void);
32 33
};

34 35 36 37
#ifdef CONFIG_HAVE_HW_BREAKPOINT
#include <asm/hw_breakpoint.h>
#endif

38 39 40 41 42
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/rculist.h>
#include <linux/rcupdate.h>
#include <linux/spinlock.h>
43
#include <linux/hrtimer.h>
Peter Zijlstra's avatar
Peter Zijlstra committed
44
#include <linux/fs.h>
45
#include <linux/pid_namespace.h>
46
#include <linux/workqueue.h>
47
#include <linux/ftrace.h>
48
#include <linux/cpu.h>
49
#include <linux/irq_work.h>
50
#include <linux/static_key.h>
51
#include <linux/jump_label_ratelimit.h>
Arun Sharma's avatar
Arun Sharma committed
52
#include <linux/atomic.h>
53
#include <linux/sysfs.h>
54
#include <linux/perf_regs.h>
55
#include <linux/workqueue.h>
56
#include <linux/cgroup.h>
57
#include <asm/local.h>
58

59 60
struct perf_callchain_entry {
	__u64				nr;
61
	__u64				ip[0]; /* /proc/sys/kernel/perf_event_max_stack */
62 63
};

64 65 66
struct perf_callchain_entry_ctx {
	struct perf_callchain_entry *entry;
	u32			    max_stack;
67
	u32			    nr;
68 69
	short			    contexts;
	bool			    contexts_maxed;
70 71
};

72
typedef unsigned long (*perf_copy_f)(void *dst, const void *src,
73
				     unsigned long off, unsigned long len);
74 75 76 77 78 79 80 81 82 83 84

struct perf_raw_frag {
	union {
		struct perf_raw_frag	*next;
		unsigned long		pad;
	};
	perf_copy_f			copy;
	void				*data;
	u32				size;
} __packed;

85
struct perf_raw_record {
86
	struct perf_raw_frag		frag;
87
	u32				size;
88 89
};

90 91 92 93 94 95 96 97 98
/*
 * branch stack layout:
 *  nr: number of taken branches stored in entries[]
 *
 * Note that nr can vary from sample to sample
 * branches (to, from) are stored from most recent
 * to least recent, i.e., entries[0] contains the most
 * recent branch.
 */
99 100 101 102 103
struct perf_branch_stack {
	__u64				nr;
	struct perf_branch_entry	entries[0];
};

104 105
struct task_struct;

106 107 108 109 110 111 112 113 114 115
/*
 * extra PMU register associated with an event
 */
struct hw_perf_event_extra {
	u64		config;	/* register value */
	unsigned int	reg;	/* register address or index */
	int		alloc;	/* extra register already allocated */
	int		idx;	/* index in shared_regs->regs[] */
};

116
/**
117
 * struct hw_perf_event - performance event hardware details:
118
 */
119 120
struct hw_perf_event {
#ifdef CONFIG_PERF_EVENTS
121 122
	union {
		struct { /* hardware */
Ingo Molnar's avatar
Ingo Molnar committed
123
			u64		config;
124
			u64		last_tag;
Ingo Molnar's avatar
Ingo Molnar committed
125
			unsigned long	config_base;
126
			unsigned long	event_base;
127
			int		event_base_rdpmc;
Ingo Molnar's avatar
Ingo Molnar committed
128
			int		idx;
129
			int		last_cpu;
130
			int		flags;
131

132
			struct hw_perf_event_extra extra_reg;
133
			struct hw_perf_event_extra branch_reg;
134
		};
135
		struct { /* software */
Ingo Molnar's avatar
Ingo Molnar committed
136
			struct hrtimer	hrtimer;
137
		};
138 139 140 141
		struct { /* tracepoint */
			/* for tp_event->class */
			struct list_head	tp_list;
		};
142 143
		struct { /* intel_cqm */
			int			cqm_state;
144
			u32			cqm_rmid;
145
			int			is_group_event;
146 147 148 149
			struct list_head	cqm_events_entry;
			struct list_head	cqm_groups_entry;
			struct list_head	cqm_group_entry;
		};
150 151 152
		struct { /* itrace */
			int			itrace_started;
		};
153 154 155 156
		struct { /* amd_power */
			u64	pwr_acc;
			u64	ptsc;
		};
157
#ifdef CONFIG_HAVE_HW_BREAKPOINT
158
		struct { /* breakpoint */
159 160 161 162 163
			/*
			 * Crufty hack to avoid the chicken and egg
			 * problem hw_breakpoint has with context
			 * creation and event initalization.
			 */
164 165
			struct arch_hw_breakpoint	info;
			struct list_head		bp_list;
166
		};
167
#endif
168 169 170 171 172 173 174
		struct { /* amd_iommu */
			u8	iommu_bank;
			u8	iommu_cntr;
			u16	padding;
			u64	conf;
			u64	conf1;
		};
175
	};
176 177 178 179
	/*
	 * If the event is a per task event, this will point to the task in
	 * question. See the comment in perf_event_alloc().
	 */
180
	struct task_struct		*target;
181

182 183 184 185 186 187 188 189 190
	/*
	 * PMU would store hardware filter configuration
	 * here.
	 */
	void				*addr_filters;

	/* Last sync'ed generation of filters */
	unsigned long			addr_filters_gen;

191 192 193 194 195 196 197
/*
 * hw_perf_event::state flags; used to track the PERF_EF_* state.
 */
#define PERF_HES_STOPPED	0x01 /* the counter is stopped */
#define PERF_HES_UPTODATE	0x02 /* event->count up-to-date */
#define PERF_HES_ARCH		0x04

Peter Zijlstra's avatar
Peter Zijlstra committed
198
	int				state;
199 200 201 202 203

	/*
	 * The last observed hardware counter value, updated with a
	 * local64_cmpxchg() such that pmu::read() can be called nested.
	 */
204
	local64_t			prev_count;
205 206 207 208

	/*
	 * The period to start the next sample with.
	 */
209
	u64				sample_period;
210 211 212 213

	/*
	 * The period we started this sample with.
	 */
214
	u64				last_period;
215 216 217 218 219 220

	/*
	 * However much is left of the current period; note that this is
	 * a full 64bit value and allows for generation of periods longer
	 * than hardware might allow.
	 */
221
	local64_t			period_left;
222 223 224 225 226

	/*
	 * State for throttling the event, see __perf_event_overflow() and
	 * perf_adjust_freq_unthr_context().
	 */
227
	u64                             interrupts_seq;
228
	u64				interrupts;
229

230 231 232 233
	/*
	 * State for freq target events, see __perf_event_overflow() and
	 * perf_adjust_freq_unthr_context().
	 */
234 235
	u64				freq_time_stamp;
	u64				freq_count_stamp;
236
#endif
237 238
};

239
struct perf_event;
Ingo Molnar's avatar
Ingo Molnar committed
240

241 242 243
/*
 * Common implementation detail of pmu::{start,commit,cancel}_txn
 */
244
#define PERF_PMU_TXN_ADD  0x1		/* txn to add/schedule event on PMU */
245
#define PERF_PMU_TXN_READ 0x2		/* txn to read event group from PMU */
246

247 248 249 250
/**
 * pmu::capabilities flags
 */
#define PERF_PMU_CAP_NO_INTERRUPT		0x01
251
#define PERF_PMU_CAP_NO_NMI			0x02
252
#define PERF_PMU_CAP_AUX_NO_SG			0x04
253
#define PERF_PMU_CAP_AUX_SW_DOUBLEBUF		0x08
254
#define PERF_PMU_CAP_EXCLUSIVE			0x10
255
#define PERF_PMU_CAP_ITRACE			0x20
256
#define PERF_PMU_CAP_HETEROGENEOUS_CPUS		0x40
257

Ingo Molnar's avatar
Ingo Molnar committed
258
/**
259
 * struct pmu - generic performance monitoring unit
Ingo Molnar's avatar
Ingo Molnar committed
260
 */
261
struct pmu {
262 263
	struct list_head		entry;

264
	struct module			*module;
Peter Zijlstra's avatar
Peter Zijlstra committed
265
	struct device			*dev;
266
	const struct attribute_group	**attr_groups;
267
	const char			*name;
Peter Zijlstra's avatar
Peter Zijlstra committed
268 269
	int				type;

270 271 272 273 274
	/*
	 * various common per-pmu feature flags
	 */
	int				capabilities;

275 276
	int * __percpu			pmu_disable_count;
	struct perf_cpu_context * __percpu pmu_cpu_context;
277
	atomic_t			exclusive_cnt; /* < 0: cpu; > 0: tsk */
Peter Zijlstra's avatar
Peter Zijlstra committed
278
	int				task_ctx_nr;
279
	int				hrtimer_interval_ms;
280

281 282 283
	/* number of address filters this PMU can do */
	unsigned int			nr_addr_filters;

284
	/*
Peter Zijlstra's avatar
Peter Zijlstra committed
285 286
	 * Fully disable/enable this PMU, can be used to protect from the PMI
	 * as well as for lazy/batch writing of the MSRs.
287
	 */
Peter Zijlstra's avatar
Peter Zijlstra committed
288 289
	void (*pmu_enable)		(struct pmu *pmu); /* optional */
	void (*pmu_disable)		(struct pmu *pmu); /* optional */
290

291
	/*
Peter Zijlstra's avatar
Peter Zijlstra committed
292
	 * Try and initialize the event for this PMU.
293 294 295 296 297 298 299 300 301 302 303 304 305
	 *
	 * Returns:
	 *  -ENOENT	-- @event is not for this PMU
	 *
	 *  -ENODEV	-- @event is for this PMU but PMU not present
	 *  -EBUSY	-- @event is for this PMU but PMU temporarily unavailable
	 *  -EINVAL	-- @event is for this PMU but @event is not valid
	 *  -EOPNOTSUPP -- @event is for this PMU, @event is valid, but not supported
	 *  -EACCESS	-- @event is for this PMU, @event is valid, but no privilidges
	 *
	 *  0		-- @event is for this PMU and valid
	 *
	 * Other error return values are allowed.
306
	 */
307 308
	int (*event_init)		(struct perf_event *event);

309 310 311 312
	/*
	 * Notification that the event was mapped or unmapped.  Called
	 * in the context of the mapping task.
	 */
313 314
	void (*event_mapped)		(struct perf_event *event, struct mm_struct *mm); /* optional */
	void (*event_unmapped)		(struct perf_event *event, struct mm_struct *mm); /* optional */
315

316 317 318 319
	/*
	 * Flags for ->add()/->del()/ ->start()/->stop(). There are
	 * matching hw_perf_event::state flags.
	 */
Peter Zijlstra's avatar
Peter Zijlstra committed
320 321 322 323
#define PERF_EF_START	0x01		/* start the counter when adding    */
#define PERF_EF_RELOAD	0x02		/* reload the counter when starting */
#define PERF_EF_UPDATE	0x04		/* update the counter when stopping */

324
	/*
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
	 * Adds/Removes a counter to/from the PMU, can be done inside a
	 * transaction, see the ->*_txn() methods.
	 *
	 * The add/del callbacks will reserve all hardware resources required
	 * to service the event, this includes any counter constraint
	 * scheduling etc.
	 *
	 * Called with IRQs disabled and the PMU disabled on the CPU the event
	 * is on.
	 *
	 * ->add() called without PERF_EF_START should result in the same state
	 *  as ->add() followed by ->stop().
	 *
	 * ->del() must always PERF_EF_UPDATE stop an event. If it calls
	 *  ->stop() that must deal with already being stopped without
	 *  PERF_EF_UPDATE.
Peter Zijlstra's avatar
Peter Zijlstra committed
341 342 343 344 345
	 */
	int  (*add)			(struct perf_event *event, int flags);
	void (*del)			(struct perf_event *event, int flags);

	/*
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
	 * Starts/Stops a counter present on the PMU.
	 *
	 * The PMI handler should stop the counter when perf_event_overflow()
	 * returns !0. ->start() will be used to continue.
	 *
	 * Also used to change the sample period.
	 *
	 * Called with IRQs disabled and the PMU disabled on the CPU the event
	 * is on -- will be called from NMI context with the PMU generates
	 * NMIs.
	 *
	 * ->stop() with PERF_EF_UPDATE will read the counter and update
	 *  period/count values like ->read() would.
	 *
	 * ->start() with PERF_EF_RELOAD will reprogram the the counter
	 *  value, must be preceded by a ->stop() with PERF_EF_UPDATE.
Peter Zijlstra's avatar
Peter Zijlstra committed
362 363 364 365 366 367
	 */
	void (*start)			(struct perf_event *event, int flags);
	void (*stop)			(struct perf_event *event, int flags);

	/*
	 * Updates the counter value of the event.
368 369 370
	 *
	 * For sampling capable PMUs this will also update the software period
	 * hw_perf_event::period_left field.
Peter Zijlstra's avatar
Peter Zijlstra committed
371
	 */
372
	void (*read)			(struct perf_event *event);
373 374

	/*
375 376 377
	 * Group events scheduling is treated as a transaction, add
	 * group events as a whole and perform one schedulability test.
	 * If the test fails, roll back the whole group
Peter Zijlstra's avatar
Peter Zijlstra committed
378 379
	 *
	 * Start the transaction, after this ->add() doesn't need to
380
	 * do schedulability tests.
381 382
	 *
	 * Optional.
383
	 */
384
	void (*start_txn)		(struct pmu *pmu, unsigned int txn_flags);
385
	/*
Peter Zijlstra's avatar
Peter Zijlstra committed
386
	 * If ->start_txn() disabled the ->add() schedulability test
387 388 389
	 * then ->commit_txn() is required to perform one. On success
	 * the transaction is closed. On error the transaction is kept
	 * open until ->cancel_txn() is called.
390 391
	 *
	 * Optional.
392
	 */
393
	int  (*commit_txn)		(struct pmu *pmu);
394
	/*
Peter Zijlstra's avatar
Peter Zijlstra committed
395
	 * Will cancel the transaction, assumes ->del() is called
Lucas De Marchi's avatar
Lucas De Marchi committed
396
	 * for each successful ->add() during the transaction.
397 398
	 *
	 * Optional.
399
	 */
400
	void (*cancel_txn)		(struct pmu *pmu);
401 402 403 404 405 406

	/*
	 * Will return the value for perf_event_mmap_page::index for this event,
	 * if no implementation is provided it will default to: event->hw.idx + 1.
	 */
	int (*event_idx)		(struct perf_event *event); /*optional */
407

408 409 410 411 412
	/*
	 * context-switches callback
	 */
	void (*sched_task)		(struct perf_event_context *ctx,
					bool sched_in);
413 414 415 416
	/*
	 * PMU specific data size
	 */
	size_t				task_ctx_size;
417

418 419 420 421 422

	/*
	 * Return the count value for a counter.
	 */
	u64 (*count)			(struct perf_event *event); /*optional*/
423 424 425 426 427 428 429 430 431 432 433 434

	/*
	 * Set up pmu-private data structures for an AUX area
	 */
	void *(*setup_aux)		(int cpu, void **pages,
					 int nr_pages, bool overwrite);
					/* optional */

	/*
	 * Free pmu-private AUX data structures
	 */
	void (*free_aux)		(void *aux); /* optional */
435

436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
	/*
	 * Validate address range filters: make sure the HW supports the
	 * requested configuration and number of filters; return 0 if the
	 * supplied filters are valid, -errno otherwise.
	 *
	 * Runs in the context of the ioctl()ing process and is not serialized
	 * with the rest of the PMU callbacks.
	 */
	int (*addr_filters_validate)	(struct list_head *filters);
					/* optional */

	/*
	 * Synchronize address range filter configuration:
	 * translate hw-agnostic filters into hardware configuration in
	 * event::hw::addr_filters.
	 *
	 * Runs as a part of filter sync sequence that is done in ->start()
	 * callback by calling perf_event_addr_filters_sync().
	 *
	 * May (and should) traverse event::addr_filters::list, for which its
	 * caller provides necessary serialization.
	 */
	void (*addr_filters_sync)	(struct perf_event *event);
					/* optional */

461 462 463 464
	/*
	 * Filter events for PMU-specific reasons.
	 */
	int (*filter_match)		(struct perf_event *event); /* optional */
Ingo Molnar's avatar
Ingo Molnar committed
465 466
};

467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
/**
 * struct perf_addr_filter - address range filter definition
 * @entry:	event's filter list linkage
 * @inode:	object file's inode for file-based filters
 * @offset:	filter range offset
 * @size:	filter range size
 * @range:	1: range, 0: address
 * @filter:	1: filter/start, 0: stop
 *
 * This is a hardware-agnostic filter configuration as specified by the user.
 */
struct perf_addr_filter {
	struct list_head	entry;
	struct inode		*inode;
	unsigned long		offset;
	unsigned long		size;
	unsigned int		range	: 1,
				filter	: 1;
};

/**
 * struct perf_addr_filters_head - container for address range filters
 * @list:	list of filters for this event
 * @lock:	spinlock that serializes accesses to the @list and event's
 *		(and its children's) filter generations.
492
 * @nr_file_filters:	number of file-based filters
493 494 495 496 497 498 499
 *
 * A child event will use parent's @list (and therefore @lock), so they are
 * bundled together; see perf_event_addr_filters().
 */
struct perf_addr_filters_head {
	struct list_head	list;
	raw_spinlock_t		lock;
500
	unsigned int		nr_file_filters;
501 502
};

503
/**
504
 * enum perf_event_active_state - the states of a event
505
 */
506
enum perf_event_active_state {
Peter Zijlstra's avatar
Peter Zijlstra committed
507
	PERF_EVENT_STATE_DEAD		= -4,
508
	PERF_EVENT_STATE_EXIT		= -3,
509
	PERF_EVENT_STATE_ERROR		= -2,
510 511
	PERF_EVENT_STATE_OFF		= -1,
	PERF_EVENT_STATE_INACTIVE	=  0,
512
	PERF_EVENT_STATE_ACTIVE		=  1,
513 514
};

515
struct file;
516 517
struct perf_sample_data;

518
typedef void (*perf_overflow_handler_t)(struct perf_event *,
519 520 521
					struct perf_sample_data *,
					struct pt_regs *regs);

522 523 524 525
/*
 * Event capabilities. For event_caps and groups caps.
 *
 * PERF_EV_CAP_SOFTWARE: Is a software event.
526 527
 * PERF_EV_CAP_READ_ACTIVE_PKG: A CPU event (or cgroup event) that can be read
 * from any CPU in the package where it is active.
528 529
 */
#define PERF_EV_CAP_SOFTWARE		BIT(0)
530
#define PERF_EV_CAP_READ_ACTIVE_PKG	BIT(1)
531

532 533
#define SWEVENT_HLIST_BITS		8
#define SWEVENT_HLIST_SIZE		(1 << SWEVENT_HLIST_BITS)
534 535

struct swevent_hlist {
536 537
	struct hlist_head		heads[SWEVENT_HLIST_SIZE];
	struct rcu_head			rcu_head;
538 539
};

540 541
#define PERF_ATTACH_CONTEXT	0x01
#define PERF_ATTACH_GROUP	0x02
542
#define PERF_ATTACH_TASK	0x04
543
#define PERF_ATTACH_TASK_DATA	0x08
544

545
struct perf_cgroup;
546 547
struct ring_buffer;

548 549 550 551 552
struct pmu_event_list {
	raw_spinlock_t		lock;
	struct list_head	list;
};

553
/**
554
 * struct perf_event - performance event kernel representation:
555
 */
556 557
struct perf_event {
#ifdef CONFIG_PERF_EVENTS
558 559 560 561 562
	/*
	 * entry onto perf_event_context::event_list;
	 *   modifications require ctx->lock
	 *   RCU safe iterations.
	 */
563
	struct list_head		event_entry;
564 565 566 567 568 569 570 571 572 573

	/*
	 * XXX: group_entry and sibling_list should be mutually exclusive;
	 * either you're a sibling on a group, or you're the group leader.
	 * Rework the code to always use the same list element.
	 *
	 * Locked for modification by both ctx->mutex and ctx->lock; holding
	 * either sufficies for read.
	 */
	struct list_head		group_entry;
574
	struct list_head		sibling_list;
575 576 577 578 579 580 581 582

	/*
	 * We need storage to track the entries in perf_pmu_migrate_context; we
	 * cannot use the event_entry because of RCU and we want to keep the
	 * group in tact which avoids us using the other two entries.
	 */
	struct list_head		migrate_entry;

583 584
	struct hlist_node		hlist_entry;
	struct list_head		active_entry;
585
	int				nr_siblings;
586 587 588 589 590 591

	/* Not serialized. Only written during event initialization. */
	int				event_caps;
	/* The cumulative AND of all event_caps for events in this group. */
	int				group_caps;

592
	struct perf_event		*group_leader;
Peter Zijlstra's avatar
Peter Zijlstra committed
593
	struct pmu			*pmu;
594
	void				*pmu_private;
595

596
	enum perf_event_active_state	state;
597
	unsigned int			attach_state;
598
	local64_t			count;
599
	atomic64_t			child_count;
600

601
	/*
602
	 * These are the total time in nanoseconds that the event
603
	 * has been enabled (i.e. eligible to run, and the task has
604
	 * been scheduled in, if this is a per-task event)
605 606 607
	 * and running (scheduled onto the CPU), respectively.
	 *
	 * They are computed from tstamp_enabled, tstamp_running and
608
	 * tstamp_stopped when the event is in INACTIVE or ACTIVE state.
609 610 611 612 613 614
	 */
	u64				total_time_enabled;
	u64				total_time_running;

	/*
	 * These are timestamps used for computing total_time_enabled
615
	 * and total_time_running when the event is in INACTIVE or
616 617
	 * ACTIVE state, measured in nanoseconds from an arbitrary point
	 * in time.
618 619
	 * tstamp_enabled: the notional time when the event was enabled
	 * tstamp_running: the notional time when the event was scheduled on
620
	 * tstamp_stopped: in INACTIVE state, the notional time when the
621
	 *	event was scheduled off.
622 623 624 625 626
	 */
	u64				tstamp_enabled;
	u64				tstamp_running;
	u64				tstamp_stopped;

627 628 629 630 631 632 633 634 635 636
	/*
	 * timestamp shadows the actual context timing but it can
	 * be safely used in NMI interrupt context. It reflects the
	 * context time as it was when the event was last scheduled in.
	 *
	 * ctx_time already accounts for ctx->timestamp. Therefore to
	 * compute ctx_time for a sample, simply add perf_clock().
	 */
	u64				shadow_ctx_time;

637
	struct perf_event_attr		attr;
638
	u16				header_size;
639
	u16				id_header_size;
640
	u16				read_size;
641
	struct hw_perf_event		hw;
642

643
	struct perf_event_context	*ctx;
644
	atomic_long_t			refcount;
645

646 647
	/*
	 * These accumulate total time (in nanoseconds) that children
648
	 * events have been enabled and running, respectively.
649 650 651 652
	 */
	atomic64_t			child_total_time_enabled;
	atomic64_t			child_total_time_running;

653
	/*
654
	 * Protect attach/detach and child_list:
655
	 */
656 657
	struct mutex			child_mutex;
	struct list_head		child_list;
658
	struct perf_event		*parent;
659 660 661 662

	int				oncpu;
	int				cpu;

663 664 665
	struct list_head		owner_entry;
	struct task_struct		*owner;

666 667 668
	/* mmap bits */
	struct mutex			mmap_mutex;
	atomic_t			mmap_count;
Peter Zijlstra's avatar
Peter Zijlstra committed
669

670
	struct ring_buffer		*rb;
671
	struct list_head		rb_entry;
672 673
	unsigned long			rcu_batches;
	int				rcu_pending;
674

675
	/* poll related */
676
	wait_queue_head_t		waitq;
Peter Zijlstra's avatar
Peter Zijlstra committed
677
	struct fasync_struct		*fasync;
678 679 680

	/* delayed work for NMIs and such */
	int				pending_wakeup;
681
	int				pending_kill;
682
	int				pending_disable;
683
	struct irq_work			pending;
684

685 686
	atomic_t			event_limit;

687 688 689 690 691 692
	/* address range filters */
	struct perf_addr_filters_head	addr_filters;
	/* vma address array for file-based filders */
	unsigned long			*addr_filters_offs;
	unsigned long			addr_filters_gen;

693
	void (*destroy)(struct perf_event *);
694
	struct rcu_head			rcu_head;
695 696

	struct pid_namespace		*ns;
697
	u64				id;
Li Zefan's avatar
Li Zefan committed
698

699
	u64				(*clock)(void);
700
	perf_overflow_handler_t		overflow_handler;
701
	void				*overflow_handler_context;
702 703 704 705
#ifdef CONFIG_BPF_SYSCALL
	perf_overflow_handler_t		orig_overflow_handler;
	struct bpf_prog			*prog;
#endif
706

707
#ifdef CONFIG_EVENT_TRACING
708
	struct trace_event_call		*tp_event;
Li Zefan's avatar
Li Zefan committed
709
	struct event_filter		*filter;
710 711 712
#ifdef CONFIG_FUNCTION_TRACER
	struct ftrace_ops               ftrace_ops;
#endif
713
#endif
Li Zefan's avatar
Li Zefan committed
714

Stephane Eranian's avatar
Stephane Eranian committed
715 716 717 718 719
#ifdef CONFIG_CGROUP_PERF
	struct perf_cgroup		*cgrp; /* cgroup event is attach to */
	int				cgrp_defer_enabled;
#endif

720
	struct list_head		sb_list;
Li Zefan's avatar
Li Zefan committed
721
#endif /* CONFIG_PERF_EVENTS */
722 723 724
};

/**
725
 * struct perf_event_context - event context structure
726
 *
727
 * Used as a container for task events and CPU events as well:
728
 */
729
struct perf_event_context {
730
	struct pmu			*pmu;
731
	/*
732
	 * Protect the states of the events in the list,
733
	 * nr_active, and the list:
734
	 */
735
	raw_spinlock_t			lock;
736
	/*
737
	 * Protect the list of events.  Locking either mutex or lock
738 739 740
	 * is sufficient to ensure the list doesn't change; to change
	 * the list you need to lock both the mutex and the spinlock.
	 */
Ingo Molnar's avatar
Ingo Molnar committed
741
	struct mutex			mutex;
742

743
	struct list_head		active_ctx_list;
744 745
	struct list_head		pinned_groups;
	struct list_head		flexible_groups;
Ingo Molnar's avatar
Ingo Molnar committed
746
	struct list_head		event_list;
747
	int				nr_events;
Ingo Molnar's avatar
Ingo Molnar committed
748 749
	int				nr_active;
	int				is_active;
750
	int				nr_stat;
751
	int				nr_freq;
752
	int				rotate_disable;
Ingo Molnar's avatar
Ingo Molnar committed
753 754
	atomic_t			refcount;
	struct task_struct		*task;
755 756

	/*
757
	 * Context clock, runs when context enabled.
758
	 */
Ingo Molnar's avatar
Ingo Molnar committed
759 760
	u64				time;
	u64				timestamp;
761 762 763 764 765

	/*
	 * These fields let us detect when two contexts have both
	 * been cloned (inherited) from a common ancestor.
	 */
766
	struct perf_event_context	*parent_ctx;
Ingo Molnar's avatar
Ingo Molnar committed
767 768 769
	u64				parent_gen;
	u64				generation;
	int				pin_count;
770
#ifdef CONFIG_CGROUP_PERF
771
	int				nr_cgroups;	 /* cgroup evts */
772
#endif
773
	void				*task_ctx_data; /* pmu specific data */
774
	struct rcu_head			rcu_head;
775 776
};

777 778
/*
 * Number of contexts where an event can trigger:
779
 *	task, softirq, hardirq, nmi.
780 781 782
 */
#define PERF_NR_CONTEXTS	4

783
/**
784
 * struct perf_event_cpu_context - per cpu event context structure
785 786
 */
struct perf_cpu_context {
787 788
	struct perf_event_context	ctx;
	struct perf_event_context	*task_ctx;
789
	int				active_oncpu;
790
	int				exclusive;
791 792

	raw_spinlock_t			hrtimer_lock;
793 794
	struct hrtimer			hrtimer;
	ktime_t				hrtimer_interval;
795 796
	unsigned int			hrtimer_active;

797
#ifdef CONFIG_CGROUP_PERF
Stephane Eranian's avatar
Stephane Eranian committed
798
	struct perf_cgroup		*cgrp;
799
	struct list_head		cgrp_cpuctx_entry;
800
#endif
801 802 803

	struct list_head		sched_cb_entry;
	int				sched_cb_usage;
804 805

	int				online;
806 807
};

808
struct perf_output_handle {
809
	struct perf_event		*event;
810
	struct ring_buffer		*rb;
811
	unsigned long			wakeup;
812
	unsigned long			size;
813
	u64				aux_flags;
814 815 816 817
	union {
		void			*addr;
		unsigned long		head;
	};
818
	int				page;
819 820
};

821 822 823 824 825
struct bpf_perf_event_data_kern {
	struct pt_regs *regs;
	struct perf_sample_data *data;
};

826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
#ifdef CONFIG_CGROUP_PERF

/*
 * perf_cgroup_info keeps track of time_enabled for a cgroup.
 * This is a per-cpu dynamically allocated data structure.
 */
struct perf_cgroup_info {
	u64				time;
	u64				timestamp;
};

struct perf_cgroup {
	struct cgroup_subsys_state	css;
	struct perf_cgroup_info	__percpu *info;
};

/*
 * Must ensure cgroup is pinned (css_get) before calling
 * this function. In other words, we cannot call this function
 * if there is no cgroup event for the current CPU context.
 */
static inline struct perf_cgroup *
848
perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx)
849
{
850 851 852
	return container_of(task_css_check(task, perf_event_cgrp_id,
					   ctx ? lockdep_is_held(&ctx->lock)
					       : true),
853 854 855 856
			    struct perf_cgroup, css);
}
#endif /* CONFIG_CGROUP_PERF */

857
#ifdef CONFIG_PERF_EVENTS
858

859 860 861
extern void *perf_aux_output_begin(struct perf_output_handle *handle,
				   struct perf_event *event);
extern void perf_aux_output_end(struct perf_output_handle *handle,
862
				unsigned long size);
863 864 865
extern int perf_aux_output_skip(struct perf_output_handle *handle,
				unsigned long size);
extern void *perf_get_aux(struct perf_output_handle *handle);
866
extern void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags);
867

868
extern int perf_pmu_register(struct pmu *pmu, const char *name, int type);
869
extern void perf_pmu_unregister(struct pmu *pmu);
Ingo Molnar's avatar
Ingo Molnar committed
870

871
extern int perf_num_counters(void);
872
extern const char *perf_pmu_name(void);
873 874 875 876
extern void __perf_event_task_sched_in(struct task_struct *prev,
				       struct task_struct *task);
extern void __perf_event_task_sched_out(struct task_struct *prev,
					struct task_struct *next);