kprobes.c 40.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 *  Kernel Probes (KProbes)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2002, 2004
 *
 * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
 *		Probes initial implementation ( includes contributions from
 *		Rusty Russell).
 * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
 *		interface to access function arguments.
25
26
 * 2004-Oct	Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> adapted for x86_64 from i386.
Linus Torvalds's avatar
Linus Torvalds committed
27
28
 * 2005-Mar	Roland McGrath <roland@redhat.com>
 *		Fixed to handle %rip-relative addressing mode correctly.
29
30
31
32
33
34
35
 * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
 *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
 *		<prasanna@in.ibm.com> added function-return probes.
 * 2005-May	Rusty Lynch <rusty.lynch@intel.com>
 * 		Added function return probes functionality
 * 2006-Feb	Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
 * 		kprobe-booster and kretprobe-booster for i386.
36
37
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
 * 		and kretprobe-booster for x86-64
38
39
40
 * 2007-Dec	Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
 * 		<arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
 * 		unified x86 kprobes code.
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
 */

#include <linux/kprobes.h>
#include <linux/ptrace.h>
#include <linux/string.h>
#include <linux/slab.h>
47
#include <linux/hardirq.h>
Linus Torvalds's avatar
Linus Torvalds committed
48
#include <linux/preempt.h>
49
#include <linux/module.h>
50
#include <linux/kdebug.h>
51
#include <linux/kallsyms.h>
52
#include <linux/ftrace.h>
53

54
55
#include <asm/cacheflush.h>
#include <asm/desc.h>
Linus Torvalds's avatar
Linus Torvalds committed
56
#include <asm/pgtable.h>
57
#include <asm/uaccess.h>
58
#include <asm/alternative.h>
59
#include <asm/insn.h>
60
#include <asm/debugreg.h>
Linus Torvalds's avatar
Linus Torvalds committed
61
62
63

void jprobe_return_end(void);

64
65
DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
Linus Torvalds's avatar
Linus Torvalds committed
66

67
#define stack_addr(regs) ((unsigned long *)kernel_stack_pointer(regs))
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
	(((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
	  (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
	  (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
	  (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
	 << (row % 32))
	/*
	 * Undefined/reserved opcodes, conditional jump, Opcode Extension
	 * Groups, and some special opcodes can not boost.
	 */
static const u32 twobyte_is_boostable[256 / 32] = {
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
	/*      ----------------------------------------------          */
	W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
	W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 10 */
	W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
	W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
	W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
	W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
	W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
	W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
	W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
	W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
	W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
	W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
	W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
	W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
	W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
	W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0)   /* f0 */
	/*      -----------------------------------------------         */
	/*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f          */
};
#undef W

103
104
105
106
107
108
109
struct kretprobe_blackpoint kretprobe_blacklist[] = {
	{"__switch_to", }, /* This function switches only current task, but
			      doesn't switch kernel stack.*/
	{NULL, NULL}	/* Terminator */
};
const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);

110
static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
111
{
112
113
	struct __arch_relative_insn {
		u8 op;
114
		s32 raddr;
115
116
117
118
119
120
121
122
123
124
125
	} __attribute__((packed)) *insn;

	insn = (struct __arch_relative_insn *)from;
	insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
	insn->op = op;
}

/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
static void __kprobes synthesize_reljump(void *from, void *to)
{
	__synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
126
127
}

128
129
130
131
132
133
134
135
136
137
138
139
140
/*
 * Check for the REX prefix which can only exist on X86_64
 * X86_32 always returns 0
 */
static int __kprobes is_REX_prefix(kprobe_opcode_t *insn)
{
#ifdef CONFIG_X86_64
	if ((*insn & 0xf0) == 0x40)
		return 1;
#endif
	return 0;
}

141
/*
142
143
 * Returns non-zero if opcode is boostable.
 * RIP relative instructions are adjusted at copying time in 64 bits mode
144
 */
145
static int __kprobes can_boost(kprobe_opcode_t *opcodes)
146
147
148
149
{
	kprobe_opcode_t opcode;
	kprobe_opcode_t *orig_opcodes = opcodes;

150
	if (search_exception_tables((unsigned long)opcodes))
151
152
		return 0;	/* Page fault may occur on this address. */

153
154
155
156
157
158
159
160
161
retry:
	if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
		return 0;
	opcode = *(opcodes++);

	/* 2nd-byte opcode */
	if (opcode == 0x0f) {
		if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
			return 0;
162
163
		return test_bit(*opcodes,
				(unsigned long *)twobyte_is_boostable);
164
165
166
	}

	switch (opcode & 0xf0) {
167
#ifdef CONFIG_X86_64
168
169
	case 0x40:
		goto retry; /* REX prefix is boostable */
170
#endif
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
	case 0x60:
		if (0x63 < opcode && opcode < 0x67)
			goto retry; /* prefixes */
		/* can't boost Address-size override and bound */
		return (opcode != 0x62 && opcode != 0x67);
	case 0x70:
		return 0; /* can't boost conditional jump */
	case 0xc0:
		/* can't boost software-interruptions */
		return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
	case 0xd0:
		/* can boost AA* and XLAT */
		return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
	case 0xe0:
		/* can boost in/out and absolute jmps */
		return ((opcode & 0x04) || opcode == 0xea);
	case 0xf0:
		if ((opcode & 0x0c) == 0 && opcode != 0xf1)
			goto retry; /* lock/rep(ne) prefix */
		/* clear and set flags are boostable */
		return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
	default:
		/* segment override prefixes are boostable */
		if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
			goto retry; /* prefixes */
		/* CS override prefix and call are not boostable */
		return (opcode != 0x2e && opcode != 0x9a);
	}
}

201
202
203
204
205
206
207
208
209
210
211
/* Recover the probed instruction at addr for further analysis. */
static int recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
{
	struct kprobe *kp;
	kp = get_kprobe((void *)addr);
	if (!kp)
		return -EINVAL;

	/*
	 *  Basically, kp->ainsn.insn has an original instruction.
	 *  However, RIP-relative instruction can not do single-stepping
212
	 *  at different place, __copy_instruction() tweaks the displacement of
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
	 *  that instruction. In that case, we can't recover the instruction
	 *  from the kp->ainsn.insn.
	 *
	 *  On the other hand, kp->opcode has a copy of the first byte of
	 *  the probed instruction, which is overwritten by int3. And
	 *  the instruction at kp->addr is not modified by kprobes except
	 *  for the first byte, we can recover the original instruction
	 *  from it and kp->opcode.
	 */
	memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
	buf[0] = kp->opcode;
	return 0;
}

/* Dummy buffers for kallsyms_lookup */
static char __dummy_buf[KSYM_NAME_LEN];

/* Check if paddr is at an instruction boundary */
static int __kprobes can_probe(unsigned long paddr)
{
	int ret;
	unsigned long addr, offset = 0;
	struct insn insn;
	kprobe_opcode_t buf[MAX_INSN_SIZE];

	if (!kallsyms_lookup(paddr, NULL, &offset, NULL, __dummy_buf))
		return 0;

	/* Decode instructions */
	addr = paddr - offset;
	while (addr < paddr) {
		kernel_insn_init(&insn, (void *)addr);
		insn_get_opcode(&insn);

		/*
		 * Check if the instruction has been modified by another
		 * kprobe, in which case we replace the breakpoint by the
		 * original instruction in our buffer.
		 */
		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
			ret = recover_probed_instruction(buf, addr);
			if (ret)
				/*
				 * Another debugging subsystem might insert
				 * this breakpoint. In that case, we can't
				 * recover it.
				 */
				return 0;
			kernel_insn_init(&insn, buf);
		}
		insn_get_length(&insn);
		addr += insn.length;
	}

	return (addr == paddr);
}

Linus Torvalds's avatar
Linus Torvalds committed
270
/*
271
 * Returns non-zero if opcode modifies the interrupt flag.
Linus Torvalds's avatar
Linus Torvalds committed
272
 */
273
static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278
279
280
281
{
	switch (*insn) {
	case 0xfa:		/* cli */
	case 0xfb:		/* sti */
	case 0xcf:		/* iret/iretd */
	case 0x9d:		/* popf/popfd */
		return 1;
	}
282

283
	/*
284
	 * on X86_64, 0x40-0x4f are REX prefixes so we need to look
285
286
	 * at the next byte instead.. but of course not recurse infinitely
	 */
287
	if (is_REX_prefix(insn))
288
		return is_IF_modifier(++insn);
289

Linus Torvalds's avatar
Linus Torvalds committed
290
291
292
293
	return 0;
}

/*
294
295
 * Copy an instruction and adjust the displacement if the instruction
 * uses the %rip-relative addressing mode.
296
 * If it does, Return the address of the 32-bit displacement word.
Linus Torvalds's avatar
Linus Torvalds committed
297
 * If not, return null.
298
 * Only applicable to 64-bit x86.
Linus Torvalds's avatar
Linus Torvalds committed
299
 */
300
static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover)
Linus Torvalds's avatar
Linus Torvalds committed
301
{
302
	struct insn insn;
303
304
	int ret;
	kprobe_opcode_t buf[MAX_INSN_SIZE];
Linus Torvalds's avatar
Linus Torvalds committed
305

306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
	kernel_insn_init(&insn, src);
	if (recover) {
		insn_get_opcode(&insn);
		if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
			ret = recover_probed_instruction(buf,
							 (unsigned long)src);
			if (ret)
				return 0;
			kernel_insn_init(&insn, buf);
		}
	}
	insn_get_length(&insn);
	memcpy(dest, insn.kaddr, insn.length);

#ifdef CONFIG_X86_64
321
322
323
	if (insn_rip_relative(&insn)) {
		s64 newdisp;
		u8 *disp;
324
		kernel_insn_init(&insn, dest);
325
326
327
328
329
330
331
332
333
334
335
336
337
		insn_get_displacement(&insn);
		/*
		 * The copied instruction uses the %rip-relative addressing
		 * mode.  Adjust the displacement for the difference between
		 * the original location of this instruction and the location
		 * of the copy that will actually be run.  The tricky bit here
		 * is making sure that the sign extension happens correctly in
		 * this calculation, since we need a signed 32-bit result to
		 * be sign-extended to 64 bits when it's added to the %rip
		 * value and yield the same 64-bit result that the sign-
		 * extension of the original signed 32-bit displacement would
		 * have given.
		 */
338
339
		newdisp = (u8 *) src + (s64) insn.displacement.value -
			  (u8 *) dest;
340
		BUG_ON((s64) (s32) newdisp != newdisp); /* Sanity check.  */
341
		disp = (u8 *) dest + insn_offset_displacement(&insn);
342
		*(s32 *) disp = (s32) newdisp;
Linus Torvalds's avatar
Linus Torvalds committed
343
	}
344
#endif
345
	return insn.length;
346
}
Linus Torvalds's avatar
Linus Torvalds committed
347

348
static void __kprobes arch_copy_kprobe(struct kprobe *p)
Linus Torvalds's avatar
Linus Torvalds committed
349
{
350
351
352
353
354
	/*
	 * Copy an instruction without recovering int3, because it will be
	 * put by another subsystem.
	 */
	__copy_instruction(p->ainsn.insn, p->addr, 0);
355

356
	if (can_boost(p->addr))
357
		p->ainsn.boostable = 0;
358
	else
359
		p->ainsn.boostable = -1;
360

361
	p->opcode = *p->addr;
Linus Torvalds's avatar
Linus Torvalds committed
362
363
}

364
365
int __kprobes arch_prepare_kprobe(struct kprobe *p)
{
366
367
368
	if (alternatives_text_reserved(p->addr, p->addr))
		return -EINVAL;

369
370
	if (!can_probe((unsigned long)p->addr))
		return -EILSEQ;
371
372
373
374
375
376
377
378
	/* insn: must be on special executable page on x86. */
	p->ainsn.insn = get_insn_slot();
	if (!p->ainsn.insn)
		return -ENOMEM;
	arch_copy_kprobe(p);
	return 0;
}

379
void __kprobes arch_arm_kprobe(struct kprobe *p)
Linus Torvalds's avatar
Linus Torvalds committed
380
{
381
	text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
Linus Torvalds's avatar
Linus Torvalds committed
382
383
}

384
void __kprobes arch_disarm_kprobe(struct kprobe *p)
Linus Torvalds's avatar
Linus Torvalds committed
385
{
386
	text_poke(p->addr, &p->opcode, 1);
387
388
}

389
void __kprobes arch_remove_kprobe(struct kprobe *p)
390
{
391
392
393
394
	if (p->ainsn.insn) {
		free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
		p->ainsn.insn = NULL;
	}
Linus Torvalds's avatar
Linus Torvalds committed
395
396
}

397
static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
398
{
399
400
	kcb->prev_kprobe.kp = kprobe_running();
	kcb->prev_kprobe.status = kcb->kprobe_status;
401
402
	kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
	kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
403
404
}

405
static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
406
{
407
408
	__get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
	kcb->kprobe_status = kcb->prev_kprobe.status;
409
410
	kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
	kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
411
412
}

413
static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
414
				struct kprobe_ctlblk *kcb)
415
{
416
	__get_cpu_var(current_kprobe) = p;
417
	kcb->kprobe_saved_flags = kcb->kprobe_old_flags
418
		= (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
419
	if (is_IF_modifier(p->ainsn.insn))
420
		kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
421
422
}

423
static void __kprobes clear_btf(void)
Roland McGrath's avatar
Roland McGrath committed
424
{
Peter Zijlstra's avatar
Peter Zijlstra committed
425
426
427
428
429
430
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl &= ~DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
Roland McGrath's avatar
Roland McGrath committed
431
432
}

433
static void __kprobes restore_btf(void)
Roland McGrath's avatar
Roland McGrath committed
434
{
Peter Zijlstra's avatar
Peter Zijlstra committed
435
436
437
438
439
440
	if (test_thread_flag(TIF_BLOCKSTEP)) {
		unsigned long debugctl = get_debugctlmsr();

		debugctl |= DEBUGCTLMSR_BTF;
		update_debugctlmsr(debugctl);
	}
Roland McGrath's avatar
Roland McGrath committed
441
442
}

443
void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
444
				      struct pt_regs *regs)
445
{
446
	unsigned long *sara = stack_addr(regs);
447

448
	ri->ret_addr = (kprobe_opcode_t *) *sara;
449

450
451
	/* Replace the return addr with trampoline addr */
	*sara = (unsigned long) &kretprobe_trampoline;
452
}
453

454
455
456
457
458
459
460
461
#ifdef CONFIG_OPTPROBES
static int  __kprobes setup_detour_execution(struct kprobe *p,
					     struct pt_regs *regs,
					     int reenter);
#else
#define setup_detour_execution(p, regs, reenter) (0)
#endif

462
static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
463
				       struct kprobe_ctlblk *kcb, int reenter)
464
{
465
466
467
	if (setup_detour_execution(p, regs, reenter))
		return;

468
#if !defined(CONFIG_PREEMPT)
469
470
	if (p->ainsn.boostable == 1 && !p->post_handler) {
		/* Boost up -- we can execute copied instructions directly */
471
472
473
474
475
476
477
		if (!reenter)
			reset_current_kprobe();
		/*
		 * Reentering boosted probe doesn't reset current_kprobe,
		 * nor set current_kprobe, because it doesn't use single
		 * stepping.
		 */
478
479
480
481
482
		regs->ip = (unsigned long)p->ainsn.insn;
		preempt_enable_no_resched();
		return;
	}
#endif
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
	if (reenter) {
		save_previous_kprobe(kcb);
		set_current_kprobe(p, regs, kcb);
		kcb->kprobe_status = KPROBE_REENTER;
	} else
		kcb->kprobe_status = KPROBE_HIT_SS;
	/* Prepare real single stepping */
	clear_btf();
	regs->flags |= X86_EFLAGS_TF;
	regs->flags &= ~X86_EFLAGS_IF;
	/* single step inline if the instruction is an int3 */
	if (p->opcode == BREAKPOINT_INSTRUCTION)
		regs->ip = (unsigned long)p->addr;
	else
		regs->ip = (unsigned long)p->ainsn.insn;
498
499
}

500
501
502
503
504
/*
 * We have reentered the kprobe_handler(), since another probe was hit while
 * within the handler. We save the original kprobes variables and just single
 * step on the instruction of the new probe without calling any user handlers.
 */
505
506
static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
				    struct kprobe_ctlblk *kcb)
507
{
508
509
510
	switch (kcb->kprobe_status) {
	case KPROBE_HIT_SSDONE:
	case KPROBE_HIT_ACTIVE:
511
		kprobes_inc_nmissed_count(p);
512
		setup_singlestep(p, regs, kcb, 1);
513
514
		break;
	case KPROBE_HIT_SS:
515
516
517
518
519
520
521
522
523
524
		/* A probe has been hit in the codepath leading up to, or just
		 * after, single-stepping of a probed instruction. This entire
		 * codepath should strictly reside in .kprobes.text section.
		 * Raise a BUG or we'll continue in an endless reentering loop
		 * and eventually a stack overflow.
		 */
		printk(KERN_WARNING "Unrecoverable kprobe detected at %p.\n",
		       p->addr);
		dump_kprobe(p);
		BUG();
525
526
527
	default:
		/* impossible cases */
		WARN_ON(1);
528
		return 0;
529
	}
530

531
	return 1;
532
}
533

534
535
/*
 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
536
 * remain disabled throughout this function.
537
538
 */
static int __kprobes kprobe_handler(struct pt_regs *regs)
Linus Torvalds's avatar
Linus Torvalds committed
539
{
540
	kprobe_opcode_t *addr;
541
	struct kprobe *p;
542
543
	struct kprobe_ctlblk *kcb;

544
	addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
545
546
	/*
	 * We don't want to be preempted for the entire
547
548
549
	 * duration of kprobe processing. We conditionally
	 * re-enable preemption at the end of this function,
	 * and also in reenter_kprobe() and setup_singlestep().
550
551
	 */
	preempt_disable();
Linus Torvalds's avatar
Linus Torvalds committed
552

553
	kcb = get_kprobe_ctlblk();
554
	p = get_kprobe(addr);
555

556
557
	if (p) {
		if (kprobe_running()) {
558
559
			if (reenter_kprobe(p, regs, kcb))
				return 1;
Linus Torvalds's avatar
Linus Torvalds committed
560
		} else {
561
562
			set_current_kprobe(p, regs, kcb);
			kcb->kprobe_status = KPROBE_HIT_ACTIVE;
563

Linus Torvalds's avatar
Linus Torvalds committed
564
			/*
565
566
567
568
569
570
			 * If we have no pre-handler or it returned 0, we
			 * continue with normal processing.  If we have a
			 * pre-handler and it returned non-zero, it prepped
			 * for calling the break_handler below on re-entry
			 * for jprobe processing, so get out doing nothing
			 * more here.
Linus Torvalds's avatar
Linus Torvalds committed
571
			 */
572
			if (!p->pre_handler || !p->pre_handler(p, regs))
573
				setup_singlestep(p, regs, kcb, 0);
574
			return 1;
575
		}
576
577
578
579
580
581
582
583
584
585
586
587
588
	} else if (*addr != BREAKPOINT_INSTRUCTION) {
		/*
		 * The breakpoint instruction was removed right
		 * after we hit it.  Another cpu has removed
		 * either a probepoint or a debugger breakpoint
		 * at this address.  In either case, no further
		 * handling of this interrupt is appropriate.
		 * Back up over the (now missing) int3 and run
		 * the original instruction.
		 */
		regs->ip = (unsigned long)addr;
		preempt_enable_no_resched();
		return 1;
589
590
591
	} else if (kprobe_running()) {
		p = __get_cpu_var(current_kprobe);
		if (p->break_handler && p->break_handler(p, regs)) {
592
			setup_singlestep(p, regs, kcb, 0);
593
			return 1;
Linus Torvalds's avatar
Linus Torvalds committed
594
		}
595
	} /* else: not a kprobe fault; let the kernel handle it */
Linus Torvalds's avatar
Linus Torvalds committed
596

597
	preempt_enable_no_resched();
598
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
599
600
}

601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
#ifdef CONFIG_X86_64
#define SAVE_REGS_STRING		\
	/* Skip cs, ip, orig_ax. */	\
	"	subq $24, %rsp\n"	\
	"	pushq %rdi\n"		\
	"	pushq %rsi\n"		\
	"	pushq %rdx\n"		\
	"	pushq %rcx\n"		\
	"	pushq %rax\n"		\
	"	pushq %r8\n"		\
	"	pushq %r9\n"		\
	"	pushq %r10\n"		\
	"	pushq %r11\n"		\
	"	pushq %rbx\n"		\
	"	pushq %rbp\n"		\
	"	pushq %r12\n"		\
	"	pushq %r13\n"		\
	"	pushq %r14\n"		\
	"	pushq %r15\n"
#define RESTORE_REGS_STRING		\
	"	popq %r15\n"		\
	"	popq %r14\n"		\
	"	popq %r13\n"		\
	"	popq %r12\n"		\
	"	popq %rbp\n"		\
	"	popq %rbx\n"		\
	"	popq %r11\n"		\
	"	popq %r10\n"		\
	"	popq %r9\n"		\
	"	popq %r8\n"		\
	"	popq %rax\n"		\
	"	popq %rcx\n"		\
	"	popq %rdx\n"		\
	"	popq %rsi\n"		\
	"	popq %rdi\n"		\
	/* Skip orig_ax, ip, cs */	\
	"	addq $24, %rsp\n"
#else
#define SAVE_REGS_STRING		\
	/* Skip cs, ip, orig_ax and gs. */	\
	"	subl $16, %esp\n"	\
	"	pushl %fs\n"		\
	"	pushl %es\n"		\
644
	"	pushl %ds\n"		\
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
	"	pushl %eax\n"		\
	"	pushl %ebp\n"		\
	"	pushl %edi\n"		\
	"	pushl %esi\n"		\
	"	pushl %edx\n"		\
	"	pushl %ecx\n"		\
	"	pushl %ebx\n"
#define RESTORE_REGS_STRING		\
	"	popl %ebx\n"		\
	"	popl %ecx\n"		\
	"	popl %edx\n"		\
	"	popl %esi\n"		\
	"	popl %edi\n"		\
	"	popl %ebp\n"		\
	"	popl %eax\n"		\
	/* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\
	"	addl $24, %esp\n"
#endif

664
/*
665
666
 * When a retprobed function returns, this code saves registers and
 * calls trampoline_handler() runs, which calls the kretprobe's handler.
667
 */
668
static void __used __kprobes kretprobe_trampoline_holder(void)
669
{
670
671
	asm volatile (
			".global kretprobe_trampoline\n"
672
			"kretprobe_trampoline: \n"
673
#ifdef CONFIG_X86_64
674
675
676
			/* We don't bother saving the ss register */
			"	pushq %rsp\n"
			"	pushfq\n"
677
			SAVE_REGS_STRING
678
679
680
681
			"	movq %rsp, %rdi\n"
			"	call trampoline_handler\n"
			/* Replace saved sp with true return address. */
			"	movq %rax, 152(%rsp)\n"
682
			RESTORE_REGS_STRING
683
			"	popfq\n"
684
685
#else
			"	pushf\n"
686
			SAVE_REGS_STRING
687
688
689
			"	movl %esp, %eax\n"
			"	call trampoline_handler\n"
			/* Move flags to cs */
690
691
			"	movl 56(%esp), %edx\n"
			"	movl %edx, 52(%esp)\n"
692
			/* Replace saved flags with true return address. */
693
			"	movl %eax, 56(%esp)\n"
694
			RESTORE_REGS_STRING
695
696
			"	popf\n"
#endif
697
			"	ret\n");
698
}
699
700

/*
701
 * Called from kretprobe_trampoline
702
 */
703
static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
704
{
bibo,mao's avatar
bibo,mao committed
705
	struct kretprobe_instance *ri = NULL;
706
	struct hlist_head *head, empty_rp;
bibo,mao's avatar
bibo,mao committed
707
	struct hlist_node *node, *tmp;
708
	unsigned long flags, orig_ret_address = 0;
709
	unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
710

711
	INIT_HLIST_HEAD(&empty_rp);
712
	kretprobe_hash_lock(current, &head, &flags);
713
	/* fixup registers */
714
#ifdef CONFIG_X86_64
715
	regs->cs = __KERNEL_CS;
716
717
#else
	regs->cs = __KERNEL_CS | get_kernel_rpl();
718
	regs->gs = 0;
719
#endif
720
	regs->ip = trampoline_address;
721
	regs->orig_ax = ~0UL;
722

723
724
	/*
	 * It is possible to have multiple instances associated with a given
725
	 * task either because multiple functions in the call path have
726
	 * return probes installed on them, and/or more than one
727
728
729
	 * return probe was registered for a target function.
	 *
	 * We can handle this because:
730
	 *     - instances are always pushed into the head of the list
731
	 *     - when multiple return probes are registered for the same
732
733
734
	 *	 function, the (chronologically) first instance's ret_addr
	 *	 will be the real return address, and all the rest will
	 *	 point to kretprobe_trampoline.
735
736
	 */
	hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
bibo,mao's avatar
bibo,mao committed
737
		if (ri->task != current)
738
			/* another task is sharing our hash bucket */
bibo,mao's avatar
bibo,mao committed
739
			continue;
740

741
742
743
		if (ri->rp && ri->rp->handler) {
			__get_cpu_var(current_kprobe) = &ri->rp->kp;
			get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
744
			ri->rp->handler(ri, regs);
745
746
			__get_cpu_var(current_kprobe) = NULL;
		}
747
748

		orig_ret_address = (unsigned long)ri->ret_addr;
749
		recycle_rp_inst(ri, &empty_rp);
750
751
752
753
754
755
756
757

		if (orig_ret_address != trampoline_address)
			/*
			 * This is the real return address. Any other
			 * instances associated with this task are for
			 * other calls deeper on the call stack
			 */
			break;
758
	}
759

760
	kretprobe_assert(ri, orig_ret_address, trampoline_address);
761

762
	kretprobe_hash_unlock(current, &flags);
763

764
765
766
767
	hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
		hlist_del(&ri->hlist);
		kfree(ri);
	}
768
	return (void *)orig_ret_address;
769
770
}

Linus Torvalds's avatar
Linus Torvalds committed
771
772
773
774
775
776
777
778
779
780
781
782
/*
 * Called after single-stepping.  p->addr is the address of the
 * instruction whose first byte has been replaced by the "int 3"
 * instruction.  To avoid the SMP problems that can occur when we
 * temporarily put back the original opcode to single-step, we
 * single-stepped a copy of the instruction.  The address of this
 * copy is p->ainsn.insn.
 *
 * This function prepares to return from the post-single-step
 * interrupt.  We have to fix up the stack as follows:
 *
 * 0) Except in the case of absolute or indirect jump or call instructions,
783
 * the new ip is relative to the copied instruction.  We need to make
Linus Torvalds's avatar
Linus Torvalds committed
784
785
786
 * it relative to the original instruction.
 *
 * 1) If the single-stepped instruction was pushfl, then the TF and IF
787
 * flags are set in the just-pushed flags, and may need to be cleared.
Linus Torvalds's avatar
Linus Torvalds committed
788
789
790
791
 *
 * 2) If the single-stepped instruction was a call, the return address
 * that is atop the stack is the address following the copied instruction.
 * We need to make it the address following the original instruction.
792
793
794
795
796
 *
 * If this is the first time we've single-stepped the instruction at
 * this probepoint, and the instruction is boostable, boost it: add a
 * jump instruction after the copied instruction, that jumps to the next
 * instruction after the probepoint.
Linus Torvalds's avatar
Linus Torvalds committed
797
 */
798
799
static void __kprobes resume_execution(struct kprobe *p,
		struct pt_regs *regs, struct kprobe_ctlblk *kcb)
Linus Torvalds's avatar
Linus Torvalds committed
800
{
801
802
803
	unsigned long *tos = stack_addr(regs);
	unsigned long copy_ip = (unsigned long)p->ainsn.insn;
	unsigned long orig_ip = (unsigned long)p->addr;
Linus Torvalds's avatar
Linus Torvalds committed
804
805
806
	kprobe_opcode_t *insn = p->ainsn.insn;

	/*skip the REX prefix*/
807
	if (is_REX_prefix(insn))
Linus Torvalds's avatar
Linus Torvalds committed
808
809
		insn++;

810
	regs->flags &= ~X86_EFLAGS_TF;
Linus Torvalds's avatar
Linus Torvalds committed
811
	switch (*insn) {
Masami Hiramatsu's avatar
Masami Hiramatsu committed
812
	case 0x9c:	/* pushfl */
813
		*tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
814
		*tos |= kcb->kprobe_old_flags;
Linus Torvalds's avatar
Linus Torvalds committed
815
		break;
Masami Hiramatsu's avatar
Masami Hiramatsu committed
816
817
	case 0xc2:	/* iret/ret/lret */
	case 0xc3:
818
	case 0xca:
Masami Hiramatsu's avatar
Masami Hiramatsu committed
819
820
821
822
	case 0xcb:
	case 0xcf:
	case 0xea:	/* jmp absolute -- ip is correct */
		/* ip is already adjusted, no more changes required */
823
		p->ainsn.boostable = 1;
Masami Hiramatsu's avatar
Masami Hiramatsu committed
824
825
		goto no_change;
	case 0xe8:	/* call relative - Fix return addr */
826
		*tos = orig_ip + (*tos - copy_ip);
Linus Torvalds's avatar
Linus Torvalds committed
827
		break;
828
#ifdef CONFIG_X86_32
829
830
831
832
	case 0x9a:	/* call absolute -- same as call absolute, indirect */
		*tos = orig_ip + (*tos - copy_ip);
		goto no_change;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
833
	case 0xff:
834
		if ((insn[1] & 0x30) == 0x10) {
835
836
837
838
839
840
			/*
			 * call absolute, indirect
			 * Fix return addr; ip is correct.
			 * But this is not boostable
			 */
			*tos = orig_ip + (*tos - copy_ip);
Masami Hiramatsu's avatar
Masami Hiramatsu committed
841
			goto no_change;
842
843
844
845
846
847
		} else if (((insn[1] & 0x31) == 0x20) ||
			   ((insn[1] & 0x31) == 0x21)) {
			/*
			 * jmp near and far, absolute indirect
			 * ip is correct. And this is boostable
			 */
848
			p->ainsn.boostable = 1;
Masami Hiramatsu's avatar
Masami Hiramatsu committed
849
			goto no_change;
Linus Torvalds's avatar
Linus Torvalds committed
850
851
852
853
854
		}
	default:
		break;
	}

855
	if (p->ainsn.boostable == 0) {
856
857
		if ((regs->ip > copy_ip) &&
		    (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
858
859
860
861
			/*
			 * These instructions can be executed directly if it
			 * jumps back to correct address.
			 */
862
863
			synthesize_reljump((void *)regs->ip,
				(void *)orig_ip + (regs->ip - copy_ip));
864
865
866
867
868
869
			p->ainsn.boostable = 1;
		} else {
			p->ainsn.boostable = -1;
		}
	}

870
	regs->ip += orig_ip - copy_ip;
871

Masami Hiramatsu's avatar
Masami Hiramatsu committed
872
no_change:
Roland McGrath's avatar
Roland McGrath committed
873
	restore_btf();
Linus Torvalds's avatar
Linus Torvalds committed
874
875
}

876
877
/*
 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
878
 * remain disabled throughout this function.
879
880
 */
static int __kprobes post_kprobe_handler(struct pt_regs *regs)
Linus Torvalds's avatar
Linus Torvalds committed
881
{
882
883
884
885
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

	if (!cur)
Linus Torvalds's avatar
Linus Torvalds committed
886
887
		return 0;

888
889
890
	resume_execution(cur, regs, kcb);
	regs->flags |= kcb->kprobe_saved_flags;

891
892
893
	if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
		kcb->kprobe_status = KPROBE_HIT_SSDONE;
		cur->post_handler(cur, regs, 0);
894
	}
Linus Torvalds's avatar
Linus Torvalds committed
895

896
	/* Restore back the original saved kprobes variables and continue. */
897
898
	if (kcb->kprobe_status == KPROBE_REENTER) {
		restore_previous_kprobe(kcb);
899
900
		goto out;
	}
901
	reset_current_kprobe();
902
out:
Linus Torvalds's avatar
Linus Torvalds committed
903
904
905
	preempt_enable_no_resched();

	/*
906
	 * if somebody else is singlestepping across a probe point, flags
Linus Torvalds's avatar
Linus Torvalds committed
907
908
909
	 * will have TF set, in which case, continue the remaining processing
	 * of do_debug, as if this is not a probe hit.
	 */
910
	if (regs->flags & X86_EFLAGS_TF)
Linus Torvalds's avatar
Linus Torvalds committed
911
912
913
914
915
		return 0;

	return 1;
}

916
int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
Linus Torvalds's avatar
Linus Torvalds committed
917
{
918
919
920
	struct kprobe *cur = kprobe_running();
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

921
	switch (kcb->kprobe_status) {
922
923
924
925
926
	case KPROBE_HIT_SS:
	case KPROBE_REENTER:
		/*
		 * We are here because the instruction being single
		 * stepped caused a page fault. We reset the current
927
		 * kprobe and the ip points back to the probe address
928
929
930
		 * and allow the page fault handler to continue as a
		 * normal page fault.
		 */
931
		regs->ip = (unsigned long)cur->addr;
932
		regs->flags |= kcb->kprobe_old_flags;
933
934
935
936
		if (kcb->kprobe_status == KPROBE_REENTER)
			restore_previous_kprobe(kcb);
		else
			reset_current_kprobe();
Linus Torvalds's avatar
Linus Torvalds committed
937
		preempt_enable_no_resched();
938
939
940
941
942
		break;
	case KPROBE_HIT_ACTIVE:
	case KPROBE_HIT_SSDONE:
		/*
		 * We increment the nmissed count for accounting,
943
		 * we can also use npre/npostfault count for accounting
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
		 * these specific fault cases.
		 */
		kprobes_inc_nmissed_count(cur);

		/*
		 * We come here because instructions in the pre/post
		 * handler caused the page_fault, this could happen
		 * if handler tries to access user space by
		 * copy_from_user(), get_user() etc. Let the
		 * user-specified handler try to fix it first.
		 */
		if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
			return 1;

		/*
		 * In case the user-specified fault handler returned
		 * zero, try to fix up.
		 */
962
963
		if (fixup_exception(regs))
			return 1;
Harvey Harrison's avatar
Harvey Harrison committed
964

965
		/*
966
		 * fixup routine could not handle it,
967
968
969
970
971
		 * Let do_page_fault() fix it.
		 */
		break;
	default:
		break;
Linus Torvalds's avatar
Linus Torvalds committed
972
973
974
975
976
977
978
	}
	return 0;
}

/*
 * Wrapper routine for handling exceptions.
 */
979
980
int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
				       unsigned long val, void *data)
Linus Torvalds's avatar
Linus Torvalds committed
981
{
Jan Engelhardt's avatar
Jan Engelhardt committed
982
	struct die_args *args = data;
983
984
	int ret = NOTIFY_DONE;

985
	if (args->regs && user_mode_vm(args->regs))
986
987
		return ret;

Linus Torvalds's avatar
Linus Torvalds committed
988
989
990
	switch (val) {
	case DIE_INT3:
		if (kprobe_handler(args->regs))
991
			ret = NOTIFY_STOP;
Linus Torvalds's avatar
Linus Torvalds committed
992
993
		break;
	case DIE_DEBUG:
994
995
996
997
998
999
		if (post_kprobe_handler(args->regs)) {
			/*
			 * Reset the BS bit in dr6 (pointed by args->err) to
			 * denote completion of processing
			 */
			(*(unsigned long *)ERR_PTR(args->err)) &= ~DR_STEP;
1000
			ret = NOTIFY_STOP;
1001
		}
Linus Torvalds's avatar
Linus Torvalds committed
1002
1003
		break;
	case DIE_GPF:
1004
1005
1006
1007
1008
1009
		/*
		 * To be potentially processing a kprobe fault and to
		 * trust the result from kprobe_running(), we have
		 * be non-preemptible.
		 */
		if (!preemptible() && kprobe_running() &&
Linus Torvalds's avatar
Linus Torvalds committed
1010
		    kprobe_fault_handler(args->regs, args->trapnr))
1011
			ret = NOTIFY_STOP;
Linus Torvalds's avatar
Linus Torvalds committed
1012
1013
1014
1015
		break;
	default:
		break;
	}
1016
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
1017
1018
}

1019
int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds's avatar
Linus Torvalds committed
1020
1021
1022
{
	struct jprobe *jp = container_of(p, struct jprobe, kp);
	unsigned long addr;
1023
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Linus Torvalds's avatar
Linus Torvalds committed
1024

1025
	kcb->jprobe_saved_regs = *regs;
1026
1027
1028
	kcb->jprobe_saved_sp = stack_addr(regs);
	addr = (unsigned long)(kcb->jprobe_saved_sp);

Linus Torvalds's avatar
Linus Torvalds committed
1029
1030
1031
1032
1033
1034
1035
	/*
	 * As Linus pointed out, gcc assumes that the callee
	 * owns the argument space and could overwrite it, e.g.
	 * tailcall optimization. So, to be absolutely safe
	 * we also save and restore enough stack bytes to cover
	 * the argument area.
	 */
1036
	memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
1037
	       MIN_STACK_SIZE(addr));
1038
	regs->flags &= ~X86_EFLAGS_IF;
1039
	trace_hardirqs_off();
1040
	regs->ip = (unsigned long)(jp->entry);
Linus Torvalds's avatar
Linus Torvalds committed
1041
1042
1043
	return 1;
}

1044
void __kprobes jprobe_return(void)
Linus Torvalds's avatar
Linus Torvalds committed
1045
{
1046
1047
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();

1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
	asm volatile (
#ifdef CONFIG_X86_64
			"       xchg   %%rbx,%%rsp	\n"
#else
			"       xchgl   %%ebx,%%esp	\n"
#endif
			"       int3			\n"
			"       .globl jprobe_return_end\n"
			"       jprobe_return_end:	\n"
			"       nop			\n"::"b"
			(kcb->jprobe_saved_sp):"memory");
Linus Torvalds's avatar
Linus Torvalds committed
1059
1060
}

1061
int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds's avatar
Linus Torvalds committed
1062
{
1063
	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1064
	u8 *addr = (u8 *) (regs->ip - 1);
Linus Torvalds's avatar
Linus Torvalds committed
1065
1066
	struct jprobe *jp = container_of(p, struct jprobe, kp);

1067
1068
	if ((addr > (u8 *) jprobe_return) &&
	    (addr < (u8 *) jprobe_return_end)) {
1069
		if (stack_addr(regs) != kcb->jprobe_saved_sp) {
Masami Hiramatsu's avatar
Masami Hiramatsu committed
1070
			struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
1071
1072
			printk(KERN_ERR
			       "current sp %p does not match saved sp %p\n",
1073
			       stack_addr(regs), kcb->jprobe_saved_sp);
1074
			printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
Linus Torvalds's avatar
Linus Torvalds committed
1075
			show_registers(saved_regs);
1076
			printk(KERN_ERR "Current registers\n");
Linus Torvalds's avatar
Linus Torvalds committed
1077
1078
1079
			show_registers(regs);
			BUG();
		}
1080
		*regs = kcb->jprobe_saved_regs;
1081
1082
1083
		memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
		       kcb->jprobes_stack,
		       MIN_STACK_SIZE(kcb->jprobe_saved_sp));
1084
		preempt_enable_no_resched();
Linus Torvalds's avatar
Linus Torvalds committed
1085
1086
1087
1088
		return 1;
	}
	return 0;
}
1089

1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251