smpboot.c 33 KB
Newer Older
1 2 3
/*
 *	x86 SMP booting functions
 *
4
 *	(c) 1995 Alan Cox, Building #3 <alan@lxorguk.ukuu.org.uk>
Ingo Molnar's avatar
Ingo Molnar committed
5
 *	(c) 1998, 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
 *	Copyright 2001 Andi Kleen, SuSE Labs.
 *
 *	Much of the core SMP work is based on previous work by Thomas Radke, to
 *	whom a great many thanks are extended.
 *
 *	Thanks to Intel for making available several different Pentium,
 *	Pentium Pro and Pentium-II/Xeon MP machines.
 *	Original development of Linux SMP code supported by Caldera.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 *
 *	Fixes
 *		Felix Koop	:	NR_CPUS used properly
 *		Jose Renau	:	Handle single CPU case.
 *		Alan Cox	:	By repeated request 8) - Total BogoMIPS report.
 *		Greg Wright	:	Fix for kernel stacks panic.
 *		Erich Boleyn	:	MP v1.4 and additional changes.
 *	Matthias Sattler	:	Changes for 2.1 kernel map.
 *	Michel Lespinasse	:	Changes for 2.1 kernel map.
 *	Michael Chastain	:	Change trampoline.S to gnu as.
 *		Alan Cox	:	Dumb bug: 'B' step PPro's are fine
 *		Ingo Molnar	:	Added APIC timers, based on code
 *					from Jose Renau
 *		Ingo Molnar	:	various cleanups and rewrites
 *		Tigran Aivazian	:	fixed "0.00 in /proc/uptime on SMP" bug.
 *	Maciej W. Rozycki	:	Bits for genuine 82489DX APICs
 *	Andi Kleen		:	Changed for SMP boot into long mode.
 *		Martin J. Bligh	: 	Added support for multi-quad systems
 *		Dave Jones	:	Report invalid combinations of Athlon CPUs.
 *		Rusty Russell	:	Hacked into shape for new "hotplug" boot process.
 *      Andi Kleen              :       Converted to new state machine.
 *	Ashok Raj		: 	CPU hotplug support
 *	Glauber Costa		:	i386 and x86_64 integration
 */

42 43
#include <linux/init.h>
#include <linux/smp.h>
44
#include <linux/module.h>
45
#include <linux/sched.h>
46
#include <linux/percpu.h>
Glauber Costa's avatar
Glauber Costa committed
47
#include <linux/bootmem.h>
48 49
#include <linux/err.h>
#include <linux/nmi.h>
50
#include <linux/tboot.h>
51

52
#include <asm/acpi.h>
53
#include <asm/desc.h>
54 55
#include <asm/nmi.h>
#include <asm/irq.h>
56
#include <asm/idle.h>
57
#include <asm/trampoline.h>
58 59
#include <asm/cpu.h>
#include <asm/numa.h>
60 61 62
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/mtrr.h>
63
#include <asm/vmi.h>
Ingo Molnar's avatar
Ingo Molnar committed
64
#include <asm/apic.h>
65
#include <asm/setup.h>
Tejun Heo's avatar
Tejun Heo committed
66
#include <asm/uv/uv.h>
67
#include <linux/mc146818rtc.h>
68

69
#include <asm/smpboot_hooks.h>
70

71
#ifdef CONFIG_X86_32
72
u8 apicid_2_node[MAX_APICID];
73
static int low_mappings;
74 75
#endif

76 77 78
/* State of each CPU */
DEFINE_PER_CPU(int, cpu_state) = { 0 };

79 80 81 82 83 84 85 86 87 88 89 90 91
/* Store all idle threads, this can be reused instead of creating
* a new thread. Also avoids complicated thread destroy functionality
* for idle threads.
*/
#ifdef CONFIG_HOTPLUG_CPU
/*
 * Needed only for CONFIG_HOTPLUG_CPU because __cpuinitdata is
 * removed after init for !CONFIG_HOTPLUG_CPU.
 */
static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
#define get_idle_for_cpu(x)      (per_cpu(idle_thread_array, x))
#define set_idle_for_cpu(x, p)   (per_cpu(idle_thread_array, x) = (p))
#else
92
static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
93 94 95
#define get_idle_for_cpu(x)      (idle_thread_array[(x)])
#define set_idle_for_cpu(x, p)   (idle_thread_array[(x)] = (p))
#endif
96

97 98 99 100 101 102 103 104
/* Number of siblings per CPU package */
int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);

/* Last level cache ID of each logical CPU */
DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;

/* representing HT siblings of each logical CPU */
105
DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
106 107 108
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);

/* representing HT and core siblings of each logical CPU */
109
DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
110 111 112 113 114
EXPORT_PER_CPU_SYMBOL(cpu_core_map);

/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
EXPORT_PER_CPU_SYMBOL(cpu_info);
115

116
atomic_t init_deasserted;
117

118 119 120 121 122 123 124 125 126
#if defined(CONFIG_NUMA) && defined(CONFIG_X86_32)
/* which node each logical CPU is on */
int cpu_to_node_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = 0 };
EXPORT_SYMBOL(cpu_to_node_map);

/* set up a mapping between cpu and node. */
static void map_cpu_to_node(int cpu, int node)
{
	printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node);
127
	cpumask_set_cpu(cpu, node_to_cpumask_map[node]);
128 129 130 131 132 133 134 135 136 137
	cpu_to_node_map[cpu] = node;
}

/* undo a mapping between cpu and node. */
static void unmap_cpu_to_node(int cpu)
{
	int node;

	printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu);
	for (node = 0; node < MAX_NUMNODES; node++)
138
		cpumask_clear_cpu(cpu, node_to_cpumask_map[node]);
139 140 141 142 143 144 145 146
	cpu_to_node_map[cpu] = 0;
}
#else /* !(CONFIG_NUMA && CONFIG_X86_32) */
#define map_cpu_to_node(cpu, node)	({})
#define unmap_cpu_to_node(cpu)	({})
#endif

#ifdef CONFIG_X86_32
147 148
static int boot_cpu_logical_apicid;

149 150 151
u8 cpu_2_logical_apicid[NR_CPUS] __read_mostly =
					{ [0 ... NR_CPUS-1] = BAD_APICID };

Ingo Molnar's avatar
Ingo Molnar committed
152
static void map_cpu_to_logical_apicid(void)
153 154 155
{
	int cpu = smp_processor_id();
	int apicid = logical_smp_processor_id();
156
	int node = apic->apicid_to_node(apicid);
157 158 159 160 161 162 163 164

	if (!node_online(node))
		node = first_online_node;

	cpu_2_logical_apicid[cpu] = apicid;
	map_cpu_to_node(cpu, node);
}

165
void numa_remove_cpu(int cpu)
166 167 168 169 170 171 172 173
{
	cpu_2_logical_apicid[cpu] = BAD_APICID;
	unmap_cpu_to_node(cpu);
}
#else
#define map_cpu_to_logical_apicid()  do {} while (0)
#endif

174 175 176 177
/*
 * Report back to the Boot Processor.
 * Running on AP.
 */
Ingo Molnar's avatar
Ingo Molnar committed
178
static void __cpuinit smp_callin(void)
179 180 181 182 183 184 185 186 187 188
{
	int cpuid, phys_id;
	unsigned long timeout;

	/*
	 * If waken up by an INIT in an 82489DX configuration
	 * we may get here before an INIT-deassert IPI reaches
	 * our local APIC.  We have to wait for the IPI or we'll
	 * lock up on an APIC access.
	 */
189 190
	if (apic->wait_for_init_deassert)
		apic->wait_for_init_deassert(&init_deasserted);
191 192 193 194

	/*
	 * (This works even if the APIC is not enabled.)
	 */
195
	phys_id = read_apic_id();
196
	cpuid = smp_processor_id();
197
	if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
198 199 200
		panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
					phys_id, cpuid);
	}
201
	pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218

	/*
	 * STARTUP IPIs are fragile beasts as they might sometimes
	 * trigger some glue motherboard logic. Complete APIC bus
	 * silence for 1 second, this overestimates the time the
	 * boot CPU is spending to send the up to 2 STARTUP IPIs
	 * by a factor of two. This should be enough.
	 */

	/*
	 * Waiting 2s total for startup (udelay is not yet working)
	 */
	timeout = jiffies + 2*HZ;
	while (time_before(jiffies, timeout)) {
		/*
		 * Has the boot CPU finished it's STARTUP sequence?
		 */
219
		if (cpumask_test_cpu(cpuid, cpu_callout_mask))
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
			break;
		cpu_relax();
	}

	if (!time_before(jiffies, timeout)) {
		panic("%s: CPU%d started up but did not get a callout!\n",
		      __func__, cpuid);
	}

	/*
	 * the boot CPU has finished the init stage and is spinning
	 * on callin_map until we finish. We are free to set up this
	 * CPU, first the APIC. (this is probably redundant on most
	 * boards)
	 */

236
	pr_debug("CALLIN, before setup_local_APIC().\n");
237 238
	if (apic->smp_callin_clear_local_apic)
		apic->smp_callin_clear_local_apic();
239 240 241 242
	setup_local_APIC();
	end_local_APIC_setup();
	map_cpu_to_logical_apicid();

243
	notify_cpu_starting(cpuid);
244 245 246 247 248 249 250 251 252
	/*
	 * Get our bogomips.
	 *
	 * Need to enable IRQs because it can take longer and then
	 * the NMI watchdog might kill us.
	 */
	local_irq_enable();
	calibrate_delay();
	local_irq_disable();
253
	pr_debug("Stack at about %p\n", &cpuid);
254 255 256 257 258 259 260 261 262

	/*
	 * Save our processor parameters
	 */
	smp_store_cpu_info(cpuid);

	/*
	 * Allow the master to continue.
	 */
263
	cpumask_set_cpu(cpuid, cpu_callin_mask);
264 265
}

266 267 268
/*
 * Activate a secondary processor.
 */
269
notrace static void __cpuinit start_secondary(void *unused)
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
{
	/*
	 * Don't put *anything* before cpu_init(), SMP booting is too
	 * fragile that we want to limit the things done here to the
	 * most necessary things.
	 */
	vmi_bringup();
	cpu_init();
	preempt_disable();
	smp_callin();

	/* otherwise gcc will move up smp_processor_id before the cpu_init */
	barrier();
	/*
	 * Check TSC synchronization with the BP:
	 */
	check_tsc_sync_target();

	if (nmi_watchdog == NMI_IO_APIC) {
		disable_8259A_irq(0);
		enable_NMI_through_LVT0();
		enable_8259A_irq(0);
	}

294 295 296 297 298 299
#ifdef CONFIG_X86_32
	while (low_mappings)
		cpu_relax();
	__flush_tlb_all();
#endif

300
	/* This must be done before setting cpu_online_mask */
301 302 303 304 305 306 307 308 309 310
	set_cpu_sibling_map(raw_smp_processor_id());
	wmb();

	/*
	 * We need to hold call_lock, so there is no inconsistency
	 * between the time smp_call_function() determines number of
	 * IPI recipients, and the time when the determination is made
	 * for which cpus receive the IPI. Holding this
	 * lock helps us to not include this cpu in a currently in progress
	 * smp_call_function().
311 312 313 314
	 *
	 * We need to hold vector_lock so there the set of online cpus
	 * does not change while we are assigning vectors to cpus.  Holding
	 * this lock ensures we don't half assign or remove an irq from a cpu.
315
	 */
316
	ipi_call_lock();
317 318
	lock_vector_lock();
	__setup_vector_irq(smp_processor_id());
319
	set_cpu_online(smp_processor_id(), true);
320
	unlock_vector_lock();
321
	ipi_call_unlock();
322 323
	per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;

324 325 326
	/* enable local interrupts */
	local_irq_enable();

327
	x86_cpuinit.setup_percpu_clockev();
328 329 330 331 332

	wmb();
	cpu_idle();
}

333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
#ifdef CONFIG_CPUMASK_OFFSTACK
/* In this case, llc_shared_map is a pointer to a cpumask. */
static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
				    const struct cpuinfo_x86 *src)
{
	struct cpumask *llc = dst->llc_shared_map;
	*dst = *src;
	dst->llc_shared_map = llc;
}
#else
static inline void copy_cpuinfo_x86(struct cpuinfo_x86 *dst,
				    const struct cpuinfo_x86 *src)
{
	*dst = *src;
}
#endif /* CONFIG_CPUMASK_OFFSTACK */

350 351 352 353 354 355 356 357 358
/*
 * The bootstrap kernel entry code has set these up. Save them for
 * a given CPU
 */

void __cpuinit smp_store_cpu_info(int id)
{
	struct cpuinfo_x86 *c = &cpu_data(id);

359
	copy_cpuinfo_x86(c, &boot_cpu_data);
360 361 362 363 364 365
	c->cpu_index = id;
	if (id != 0)
		identify_secondary_cpu(c);
}


366 367 368 369 370
void __cpuinit set_cpu_sibling_map(int cpu)
{
	int i;
	struct cpuinfo_x86 *c = &cpu_data(cpu);

371
	cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
372 373

	if (smp_num_siblings > 1) {
374 375 376 377 378 379 380 381 382
		for_each_cpu(i, cpu_sibling_setup_mask) {
			struct cpuinfo_x86 *o = &cpu_data(i);

			if (c->phys_proc_id == o->phys_proc_id &&
			    c->cpu_core_id == o->cpu_core_id) {
				cpumask_set_cpu(i, cpu_sibling_mask(cpu));
				cpumask_set_cpu(cpu, cpu_sibling_mask(i));
				cpumask_set_cpu(i, cpu_core_mask(cpu));
				cpumask_set_cpu(cpu, cpu_core_mask(i));
383 384
				cpumask_set_cpu(i, c->llc_shared_map);
				cpumask_set_cpu(cpu, o->llc_shared_map);
385 386 387
			}
		}
	} else {
388
		cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
389 390
	}

391
	cpumask_set_cpu(cpu, c->llc_shared_map);
392 393

	if (current_cpu_data.x86_max_cores == 1) {
394
		cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu));
395 396 397 398
		c->booted_cores = 1;
		return;
	}

399
	for_each_cpu(i, cpu_sibling_setup_mask) {
400 401
		if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
		    per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
402 403
			cpumask_set_cpu(i, c->llc_shared_map);
			cpumask_set_cpu(cpu, cpu_data(i).llc_shared_map);
404 405
		}
		if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
406 407
			cpumask_set_cpu(i, cpu_core_mask(cpu));
			cpumask_set_cpu(cpu, cpu_core_mask(i));
408 409 410
			/*
			 *  Does this new cpu bringup a new core?
			 */
411
			if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
412 413 414 415
				/*
				 * for each core in package, increment
				 * the booted_cores for this new cpu
				 */
416
				if (cpumask_first(cpu_sibling_mask(i)) == i)
417 418 419 420 421 422 423 424 425 426 427 428 429
					c->booted_cores++;
				/*
				 * increment the core count for all
				 * the other cpus in this package
				 */
				if (i != cpu)
					cpu_data(i).booted_cores++;
			} else if (i != cpu && !c->booted_cores)
				c->booted_cores = cpu_data(i).booted_cores;
		}
	}
}

430
/* maps the cpu to the sched domain representing multi-core */
431
const struct cpumask *cpu_coregroup_mask(int cpu)
432 433 434 435 436 437
{
	struct cpuinfo_x86 *c = &cpu_data(cpu);
	/*
	 * For perf, we return last level cache shared map.
	 * And for power savings, we return cpu_core_map
	 */
438 439
	if ((sched_mc_power_savings || sched_smt_power_savings) &&
	    !(cpu_has(c, X86_FEATURE_AMD_DCM)))
440
		return cpu_core_mask(cpu);
441
	else
442
		return c->llc_shared_map;
443 444
}

Ingo Molnar's avatar
Ingo Molnar committed
445
static void impress_friends(void)
446 447 448 449 450 451
{
	int cpu;
	unsigned long bogosum = 0;
	/*
	 * Allow the user to impress friends.
	 */
452
	pr_debug("Before bogomips.\n");
453
	for_each_possible_cpu(cpu)
454
		if (cpumask_test_cpu(cpu, cpu_callout_mask))
455 456 457
			bogosum += cpu_data(cpu).loops_per_jiffy;
	printk(KERN_INFO
		"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
458
		num_online_cpus(),
459 460 461
		bogosum/(500000/HZ),
		(bogosum/(5000/HZ))%100);

462
	pr_debug("Before bogocount - setting activated=1.\n");
463 464
}

465
void __inquire_remote_apic(int apicid)
466 467 468 469 470 471
{
	unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
	char *names[] = { "ID", "VERSION", "SPIV" };
	int timeout;
	u32 status;

472
	printk(KERN_INFO "Inquiring remote APIC 0x%x...\n", apicid);
473 474

	for (i = 0; i < ARRAY_SIZE(regs); i++) {
475
		printk(KERN_INFO "... APIC 0x%x %s: ", apicid, names[i]);
476 477 478 479 480 481 482 483 484

		/*
		 * Wait for idle.
		 */
		status = safe_apic_wait_icr_idle();
		if (status)
			printk(KERN_CONT
			       "a previous APIC delivery may have failed\n");

485
		apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508

		timeout = 0;
		do {
			udelay(100);
			status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
		} while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);

		switch (status) {
		case APIC_ICR_RR_VALID:
			status = apic_read(APIC_RRR);
			printk(KERN_CONT "%08x\n", status);
			break;
		default:
			printk(KERN_CONT "failed\n");
		}
	}
}

/*
 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
 * won't ... remember to clear down the APIC, etc later.
 */
509
int __cpuinit
510
wakeup_secondary_cpu_via_nmi(int logical_apicid, unsigned long start_eip)
511 512 513 514 515 516 517
{
	unsigned long send_status, accept_status = 0;
	int maxlvt;

	/* Target chip */
	/* Boot on the stack */
	/* Kick the second */
518
	apic_icr_write(APIC_DM_NMI | apic->dest_logical, logical_apicid);
519

520
	pr_debug("Waiting for send to finish...\n");
521 522 523 524 525 526
	send_status = safe_apic_wait_icr_idle();

	/*
	 * Give the other CPU some time to accept the IPI.
	 */
	udelay(200);
527
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
528 529 530 531 532
		maxlvt = lapic_get_maxlvt();
		if (maxlvt > 3)			/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
	}
533
	pr_debug("NMI sent.\n");
534 535 536 537 538 539 540 541 542

	if (send_status)
		printk(KERN_ERR "APIC never delivered???\n");
	if (accept_status)
		printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);

	return (send_status | accept_status);
}

543
static int __cpuinit
544
wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
545 546 547 548
{
	unsigned long send_status, accept_status = 0;
	int maxlvt, num_starts, j;

549 550
	maxlvt = lapic_get_maxlvt();

551 552 553 554
	/*
	 * Be paranoid about clearing APIC errors.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid])) {
555 556
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
557 558 559
		apic_read(APIC_ESR);
	}

560
	pr_debug("Asserting INIT.\n");
561 562 563 564 565 566 567

	/*
	 * Turn INIT on target chip
	 */
	/*
	 * Send IPI
	 */
568 569
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
		       phys_apicid);
570

571
	pr_debug("Waiting for send to finish...\n");
572 573 574 575
	send_status = safe_apic_wait_icr_idle();

	mdelay(10);

576
	pr_debug("Deasserting INIT.\n");
577 578 579

	/* Target chip */
	/* Send IPI */
580
	apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
581

582
	pr_debug("Waiting for send to finish...\n");
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
	send_status = safe_apic_wait_icr_idle();

	mb();
	atomic_set(&init_deasserted, 1);

	/*
	 * Should we send STARTUP IPIs ?
	 *
	 * Determine this based on the APIC version.
	 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
	 */
	if (APIC_INTEGRATED(apic_version[phys_apicid]))
		num_starts = 2;
	else
		num_starts = 0;

	/*
	 * Paravirt / VMI wants a startup IPI hook here to set up the
	 * target processor state.
	 */
	startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
			 (unsigned long)stack_start.sp);

	/*
	 * Run STARTUP IPI loop.
	 */
609
	pr_debug("#startup loops: %d.\n", num_starts);
610 611

	for (j = 1; j <= num_starts; j++) {
612
		pr_debug("Sending STARTUP #%d.\n", j);
613 614
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
			apic_write(APIC_ESR, 0);
615
		apic_read(APIC_ESR);
616
		pr_debug("After apic_write.\n");
617 618 619 620 621 622 623 624

		/*
		 * STARTUP IPI
		 */

		/* Target chip */
		/* Boot on the stack */
		/* Kick the second */
625 626
		apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
			       phys_apicid);
627 628 629 630 631 632

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(300);

633
		pr_debug("Startup point 1.\n");
634

635
		pr_debug("Waiting for send to finish...\n");
636 637 638 639 640 641
		send_status = safe_apic_wait_icr_idle();

		/*
		 * Give the other CPU some time to accept the IPI.
		 */
		udelay(200);
642
		if (maxlvt > 3)		/* Due to the Pentium erratum 3AP.  */
643 644 645 646 647
			apic_write(APIC_ESR, 0);
		accept_status = (apic_read(APIC_ESR) & 0xEF);
		if (send_status || accept_status)
			break;
	}
648
	pr_debug("After Startup.\n");
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673

	if (send_status)
		printk(KERN_ERR "APIC never delivered???\n");
	if (accept_status)
		printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);

	return (send_status | accept_status);
}

struct create_idle {
	struct work_struct work;
	struct task_struct *idle;
	struct completion done;
	int cpu;
};

static void __cpuinit do_fork_idle(struct work_struct *work)
{
	struct create_idle *c_idle =
		container_of(work, struct create_idle, work);

	c_idle->idle = fork_idle(c_idle->cpu);
	complete(&c_idle->done);
}

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
/* reduce the number of lines printed when booting a large cpu count system */
static void __cpuinit announce_cpu(int cpu, int apicid)
{
	static int current_node = -1;
	int node = cpu_to_node(cpu);

	if (system_state == SYSTEM_BOOTING) {
		if (node != current_node) {
			if (current_node > (-1))
				pr_cont(" Ok.\n");
			current_node = node;
			pr_info("Booting Node %3d, Processors ", node);
		}
		pr_cont(" #%d%s", cpu, cpu == (nr_cpu_ids - 1) ? " Ok.\n" : "");
		return;
	} else
		pr_info("Booting Node %d Processor %d APIC 0x%x\n",
			node, cpu, apicid);
}

694 695 696
/*
 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
697 698
 * Returns zero if CPU booted OK, else error code from
 * ->wakeup_secondary_cpu.
699
 */
700
static int __cpuinit do_boot_cpu(int apicid, int cpu)
701 702 703
{
	unsigned long boot_error = 0;
	unsigned long start_ip;
704
	int timeout;
705
	struct create_idle c_idle = {
706 707
		.cpu	= cpu,
		.done	= COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
708
	};
709

710
	INIT_WORK_ON_STACK(&c_idle.work, do_fork_idle);
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735

	alternatives_smp_switch(1);

	c_idle.idle = get_idle_for_cpu(cpu);

	/*
	 * We can't use kernel_thread since we must avoid to
	 * reschedule the child.
	 */
	if (c_idle.idle) {
		c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
			(THREAD_SIZE +  task_stack_page(c_idle.idle))) - 1);
		init_idle(c_idle.idle, cpu);
		goto do_rest;
	}

	if (!keventd_up() || current_is_keventd())
		c_idle.work.func(&c_idle.work);
	else {
		schedule_work(&c_idle.work);
		wait_for_completion(&c_idle.done);
	}

	if (IS_ERR(c_idle.idle)) {
		printk("failed fork for CPU %d\n", cpu);
736
		destroy_work_on_stack(&c_idle.work);
737 738 739 740 741 742
		return PTR_ERR(c_idle.idle);
	}

	set_idle_for_cpu(cpu, c_idle.idle);
do_rest:
	per_cpu(current_task, cpu) = c_idle.idle;
743
#ifdef CONFIG_X86_32
744 745 746 747
	/* Stack for startup_32 can be just as for start_secondary onwards */
	irq_ctx_init(cpu);
#else
	clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
748
	initial_gs = per_cpu_offset(cpu);
749 750 751
	per_cpu(kernel_stack, cpu) =
		(unsigned long)task_stack_page(c_idle.idle) -
		KERNEL_STACK_OFFSET + THREAD_SIZE;
752
#endif
753
	early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
754
	initial_code = (unsigned long)start_secondary;
Glauber Costa's avatar
Glauber Costa committed
755
	stack_start.sp = (void *) c_idle.idle->thread.sp;
756 757 758 759

	/* start_ip had better be page-aligned! */
	start_ip = setup_trampoline();

760 761
	/* So we see what's up */
	announce_cpu(cpu, apicid);
762 763 764 765 766 767 768 769

	/*
	 * This grunge runs the startup process for
	 * the targeted processor.
	 */

	atomic_set(&init_deasserted, 0);

Jack Steiner's avatar
Jack Steiner committed
770
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
771

772
		pr_debug("Setting warm reset code and vector.\n");
773

Jack Steiner's avatar
Jack Steiner committed
774 775 776
		smpboot_setup_warm_reset_vector(start_ip);
		/*
		 * Be paranoid about clearing APIC errors.
777 778 779 780 781
		*/
		if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
			apic_write(APIC_ESR, 0);
			apic_read(APIC_ESR);
		}
Jack Steiner's avatar
Jack Steiner committed
782
	}
783 784

	/*
785 786
	 * Kick the secondary CPU. Use the method in the APIC driver
	 * if it's defined - or use an INIT boot APIC message otherwise:
787
	 */
788 789 790 791
	if (apic->wakeup_secondary_cpu)
		boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
	else
		boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
792 793 794 795 796

	if (!boot_error) {
		/*
		 * allow APs to start initializing.
		 */
797
		pr_debug("Before Callout %d.\n", cpu);
798
		cpumask_set_cpu(cpu, cpu_callout_mask);
799
		pr_debug("After Callout %d.\n", cpu);
800 801 802 803 804

		/*
		 * Wait 5s total for a response
		 */
		for (timeout = 0; timeout < 50000; timeout++) {
805
			if (cpumask_test_cpu(cpu, cpu_callin_mask))
806 807 808 809
				break;	/* It has booted */
			udelay(100);
		}

810 811 812
		if (cpumask_test_cpu(cpu, cpu_callin_mask))
			pr_debug("CPU%d: has booted.\n", cpu);
		else {
813 814 815 816
			boot_error = 1;
			if (*((volatile unsigned char *)trampoline_base)
					== 0xA5)
				/* trampoline started but...? */
817
				pr_err("CPU%d: Stuck ??\n", cpu);
818 819
			else
				/* trampoline code not run */
820
				pr_err("CPU%d: Not responding.\n", cpu);
821 822
			if (apic->inquire_remote_apic)
				apic->inquire_remote_apic(apicid);
823 824
		}
	}
825

826 827
	if (boot_error) {
		/* Try to put things back the way they were before ... */
828
		numa_remove_cpu(cpu); /* was set by numa_add_cpu */
829 830 831 832 833 834 835 836

		/* was set by do_boot_cpu() */
		cpumask_clear_cpu(cpu, cpu_callout_mask);

		/* was set by cpu_init() */
		cpumask_clear_cpu(cpu, cpu_initialized_mask);

		set_cpu_present(cpu, false);
837 838 839 840 841 842
		per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
	}

	/* mark "stuck" area as not stuck */
	*((volatile unsigned long *)trampoline_base) = 0;

843 844 845 846 847 848
	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
		/*
		 * Cleanup possible dangling ends...
		 */
		smpboot_restore_warm_reset_vector();
	}
849

850
	destroy_work_on_stack(&c_idle.work);
851 852 853 854 855
	return boot_error;
}

int __cpuinit native_cpu_up(unsigned int cpu)
{
856
	int apicid = apic->cpu_present_to_apicid(cpu);
857 858 859 860 861
	unsigned long flags;
	int err;

	WARN_ON(irqs_disabled());

862
	pr_debug("++++++++++++++++++++=_---CPU UP  %u\n", cpu);
863 864 865 866 867 868 869 870 871 872

	if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
	    !physid_isset(apicid, phys_cpu_present_map)) {
		printk(KERN_ERR "%s: bad cpu %d\n", __func__, cpu);
		return -EINVAL;
	}

	/*
	 * Already booted CPU?
	 */
873
	if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
874
		pr_debug("do_boot_cpu %d Already started\n", cpu);
875 876 877 878 879 880 881 882 883 884 885 886 887
		return -ENOSYS;
	}

	/*
	 * Save current MTRR state in case it was changed since early boot
	 * (e.g. by the ACPI SMI) to initialize new CPUs with MTRRs in sync:
	 */
	mtrr_save_state();

	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;

#ifdef CONFIG_X86_32
	/* init low mem mapping */
888
	clone_pgd_range(swapper_pg_dir, swapper_pg_dir + KERNEL_PGD_BOUNDARY,
889
		min_t(unsigned long, KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
890
	flush_tlb_all();
891
	low_mappings = 1;
892 893

	err = do_boot_cpu(apicid, cpu);
894

895
	zap_low_mappings(false);
896 897 898 899 900
	low_mappings = 0;
#else
	err = do_boot_cpu(apicid, cpu);
#endif
	if (err) {
901
		pr_debug("do_boot_cpu failed %d\n", err);
902
		return -EIO;
903 904 905 906 907 908 909 910 911 912
	}

	/*
	 * Check TSC synchronization with the AP (keep irqs disabled
	 * while doing so):
	 */
	local_irq_save(flags);
	check_tsc_sync_source(cpu);
	local_irq_restore(flags);

913
	while (!cpu_online(cpu)) {
914 915 916 917 918 919 920
		cpu_relax();
		touch_nmi_watchdog();
	}

	return 0;
}

921 922 923 924 925 926 927
/*
 * Fall back to non SMP mode after errors.
 *
 * RED-PEN audit/test this more. I bet there is more state messed up here.
 */
static __init void disable_smp(void)
{
928 929
	init_cpu_present(cpumask_of(0));
	init_cpu_possible(cpumask_of(0));
930
	smpboot_clear_io_apic_irqs();
931

932
	if (smp_found_config)
933
		physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
934
	else
935
		physid_set_mask_of_physid(0, &phys_cpu_present_map);
936
	map_cpu_to_logical_apicid();
937 938
	cpumask_set_cpu(0, cpu_sibling_mask(0));
	cpumask_set_cpu(0, cpu_core_mask(0));
939 940 941 942 943 944 945
}

/*
 * Various sanity checks.
 */
static int __init smp_sanity_check(unsigned max_cpus)
{
Jack Steiner's avatar
Jack Steiner committed
946
	preempt_disable();
947

948
#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
949 950 951 952 953 954
	if (def_to_bigsmp && nr_cpu_ids > 8) {
		unsigned int cpu;
		unsigned nr;

		printk(KERN_WARNING
		       "More than 8 CPUs detected - skipping them.\n"
955
		       "Use CONFIG_X86_BIGSMP.\n");
956 957 958 959

		nr = 0;
		for_each_present_cpu(cpu) {
			if (nr >= 8)
960
				set_cpu_present(cpu, false);
961 962 963 964 965 966
			nr++;
		}

		nr = 0;
		for_each_possible_cpu(cpu) {
			if (nr >= 8)
967
				set_cpu_possible(cpu, false);
968 969 970 971 972 973 974
			nr++;
		}

		nr_cpu_ids = 8;
	}
#endif

975
	if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
976 977 978 979
		printk(KERN_WARNING
			"weird, boot CPU (#%d) not listed by the BIOS.\n",
			hard_smp_processor_id());

980 981 982 983 984 985 986 987
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}

	/*
	 * If we couldn't find an SMP configuration at boot time,
	 * get out of here now!
	 */
	if (!smp_found_config && !acpi_lapic) {
Jack Steiner's avatar
Jack Steiner committed
988
		preempt_enable();
989 990 991 992 993 994 995 996 997 998 999 1000
		printk(KERN_NOTICE "SMP motherboard not detected.\n");
		disable_smp();
		if (APIC_init_uniprocessor())
			printk(KERN_NOTICE "Local APIC not detected."
					   " Using dummy APIC emulation.\n");
		return -1;
	}

	/*
	 * Should not be necessary because the MP table should list the boot
	 * CPU too, but we do it for the sake of robustness anyway.
	 */
1001
	if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
1002 1003 1004 1005 1006
		printk(KERN_NOTICE
			"weird, boot CPU (#%d) not listed by the BIOS.\n",
			boot_cpu_physical_apicid);
		physid_set(hard_smp_processor_id(), phys_cpu_present_map);
	}
Jack Steiner's avatar
Jack Steiner committed
1007
	preempt_enable();
1008 1009 1010 1011 1012 1013

	/*
	 * If we couldn't find a local APIC, then get out of here now!
	 */
	if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
	    !cpu_has_apic) {
1014 1015 1016 1017
		if (!disable_apic) {
			pr_err("BIOS bug, local APIC #%d not detected!...\n",
				boot_cpu_physical_apicid);
			pr_err("... forcing use of dummy APIC emulation."
1018
				"(tell your hw vendor)\n");
1019
		}
1020
		smpboot_clear_io_apic();
1021
		arch_disable_smp_support();
1022 1023 1024 1025 1026 1027 1028 1029 1030
		return -1;
	}

	verify_local_APIC();

	/*
	 * If SMP should be disabled, then really disable it!
	 */
	if (!max_cpus) {
1031
		printk(KERN_INFO "SMP mode deactivated.\n");
1032
		smpboot_clear_io_apic();
1033 1034 1035

		localise_nmi_watchdog();

1036 1037 1038
		connect_bsp_APIC();
		setup_local_APIC();
		end_local_APIC_setup();
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
		return -1;
	}

	return 0;
}

static void __init smp_cpu_index_default(void)
{
	int i;
	struct cpuinfo_x86 *c;

1050
	for_each_possible_cpu(i) {
1051 1052
		c = &cpu_data(i);
		/* mark all to hotplug */
1053
		c->cpu_index = nr_cpu_ids;
1054 1055 1056 1057 1058 1059 1060 1061 1062
	}
}

/*
 * Prepare for SMP bootup.  The MP table or ACPI has been read
 * earlier.  Just do some sanity checking here and enable APIC mode.
 */
void __init native_smp_prepare_cpus(unsigned int max_cpus)
{
1063 1064
	unsigned int i;

1065
	preempt_disable();
1066 1067
	smp_cpu_index_default();
	current_cpu_data = boot_cpu_data;
1068
	cpumask_copy(cpu_callin_mask, cpumask_of(0));
1069 1070 1071 1072 1073
	mb();
	/*
	 * Setup boot CPU information
	 */
	smp_store_cpu_info(0); /* Final full version of the data */
1074
#ifdef CONFIG_X86_32
1075
	boot_cpu_logical_apicid = logical_smp_processor_id();
1076
#endif
1077
	current_thread_info()->cpu = 0;  /* needed? */
1078
	for_each_possible_cpu(i) {
1079 1080 1081
		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
		zalloc_cpumask_var(&cpu_data(i).llc_shared_map, GFP_KERNEL);
1082
	}
1083 1084
	set_cpu_sibling_map(0);

1085
	enable_IR_x2apic();
1086
	default_setup_apic_routing();
1087

1088 1089 1090
	if (smp_sanity_check(max_cpus) < 0) {
		printk(KERN_INFO "SMP disabled\n");
		disable_smp();
1091
		goto out;
1092 1093
	}

Jack Steiner's avatar
Jack Steiner committed
1094
	preempt_disable();
1095
	if (read_apic_id() != boot_cpu_physical_apicid) {
1096
		panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1097
		     read_apic_id(), boot_cpu_physical_apicid);
1098 1099
		/* Or can we switch back to PIC here? */
	}
Jack Steiner's avatar
Jack Steiner committed
1100
	preempt_enable();
1101 1102

	connect_bsp_APIC();
1103

1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
	/*
	 * Switch from PIC to APIC mode.
	 */
	setup_local_APIC();

	/*
	 * Enable IO APIC before setting up error vector
	 */
	if (!skip_ioapic_setup && nr_ioapics)
		enable_IO_APIC();
1114

1115 1116 1117 1118
	end_local_APIC_setup();

	map_cpu_to_logical_apicid();

1119 1120
	if (apic->setup_portio_remap)
		apic->setup_portio_remap();
1121 1122 1123 1124 1125 1126 1127 1128

	smpboot_setup_io_apic();
	/*
	 * Set up local APIC timer on boot CPU.
	 */

	printk(KERN_INFO "CPU%d: ", 0);
	print_cpu_info(&cpu_data(0));
1129
	x86_init.timers.setup_percpu_clockev();
1130 1131 1132

	if (is_uv_system())
		uv_system_init();
1133 1134

	set_mtrr_aps_delayed_init();
1135 1136
out:
	preempt_enable();
1137
}
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148

void arch_enable_nonboot_cpus_begin(void)