4 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
5 * deal of code from the sparc and intel versions.
7 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
9 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
10 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version
15 * 2 of the License, or (at your option) any later version.
20 #include <linux/config.h>
21 #include <linux/kernel.h>
22 #include <linux/module.h>
23 #include <linux/sched.h>
24 #include <linux/smp.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <linux/init.h>
28 #include <linux/spinlock.h>
29 #include <linux/cache.h>
30 #include <linux/err.h>
31 #include <linux/sysdev.h>
32 #include <linux/cpu.h>
33 #include <linux/notifier.h>
35 #include <asm/ptrace.h>
36 #include <asm/atomic.h>
39 #include <asm/pgtable.h>
44 #include <asm/machdep.h>
45 #include <asm/cputable.h>
46 #include <asm/system.h>
47 #include <asm/abs_addr.h>
52 #define DBG(fmt...) udbg_printf(fmt)
57 cpumask_t cpu_possible_map
= CPU_MASK_NONE
;
58 cpumask_t cpu_online_map
= CPU_MASK_NONE
;
59 cpumask_t cpu_sibling_map
[NR_CPUS
] = { [0 ... NR_CPUS
-1] = CPU_MASK_NONE
};
61 EXPORT_SYMBOL(cpu_online_map
);
62 EXPORT_SYMBOL(cpu_possible_map
);
64 struct smp_ops_t
*smp_ops
;
66 static volatile unsigned int cpu_callin_map
[NR_CPUS
];
68 extern unsigned char stab_array
[];
70 void smp_call_function_interrupt(void);
72 int smt_enabled_at_boot
= 1;
75 void smp_mpic_message_pass(int target
, int msg
)
77 /* make sure we're sending something that translates to an IPI */
79 printk("SMP %d: smp_message_pass: unknown msg %d\n",
80 smp_processor_id(), msg
);
86 mpic_send_ipi(msg
, 0xffffffff);
88 case MSG_ALL_BUT_SELF
:
89 mpic_send_ipi(msg
, 0xffffffff & ~(1 << smp_processor_id()));
92 mpic_send_ipi(msg
, 1 << target
);
97 int __init
smp_mpic_probe(void)
101 DBG("smp_mpic_probe()...\n");
103 nr_cpus
= cpus_weight(cpu_possible_map
);
105 DBG("nr_cpus: %d\n", nr_cpus
);
113 void __devinit
smp_mpic_setup_cpu(int cpu
)
115 mpic_setup_this_cpu();
118 void __devinit
smp_generic_kick_cpu(int nr
)
120 BUG_ON(nr
< 0 || nr
>= NR_CPUS
);
123 * The processor is currently spinning, waiting for the
124 * cpu_start field to become non-zero After we set cpu_start,
125 * the processor will continue on to secondary_start
127 paca
[nr
].cpu_start
= 1;
131 #endif /* CONFIG_MPIC */
133 static void __init
smp_space_timers(unsigned int max_cpus
)
136 unsigned long offset
= tb_ticks_per_jiffy
/ max_cpus
;
137 unsigned long previous_tb
= paca
[boot_cpuid
].next_jiffy_update_tb
;
140 if (i
!= boot_cpuid
) {
141 paca
[i
].next_jiffy_update_tb
=
142 previous_tb
+ offset
;
143 previous_tb
= paca
[i
].next_jiffy_update_tb
;
148 void smp_message_recv(int msg
, struct pt_regs
*regs
)
151 case PPC_MSG_CALL_FUNCTION
:
152 smp_call_function_interrupt();
154 case PPC_MSG_RESCHEDULE
:
155 /* XXX Do we have to do this? */
159 case PPC_MSG_MIGRATE_TASK
:
163 #ifdef CONFIG_DEBUGGER
164 case PPC_MSG_DEBUGGER_BREAK
:
169 printk("SMP %d: smp_message_recv(): unknown msg %d\n",
170 smp_processor_id(), msg
);
175 void smp_send_reschedule(int cpu
)
177 smp_ops
->message_pass(cpu
, PPC_MSG_RESCHEDULE
);
180 #ifdef CONFIG_DEBUGGER
181 void smp_send_debugger_break(int cpu
)
183 smp_ops
->message_pass(cpu
, PPC_MSG_DEBUGGER_BREAK
);
187 static void stop_this_cpu(void *dummy
)
194 void smp_send_stop(void)
196 smp_call_function(stop_this_cpu
, NULL
, 1, 0);
200 * Structure and data for smp_call_function(). This is designed to minimise
201 * static memory requirements. It also looks cleaner.
202 * Stolen from the i386 version.
204 static __cacheline_aligned_in_smp
DEFINE_SPINLOCK(call_lock
);
206 static struct call_data_struct
{
207 void (*func
) (void *info
);
214 /* delay of at least 8 seconds on 1GHz cpu */
215 #define SMP_CALL_TIMEOUT (1UL << (30 + 3))
218 * This function sends a 'generic call function' IPI to all other CPUs
221 * [SUMMARY] Run a function on all other CPUs.
222 * <func> The function to run. This must be fast and non-blocking.
223 * <info> An arbitrary pointer to pass to the function.
224 * <nonatomic> currently unused.
225 * <wait> If true, wait (atomically) until function has completed on other CPUs.
226 * [RETURNS] 0 on success, else a negative status code. Does not return until
227 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
229 * You must not call this function with disabled interrupts or from a
230 * hardware interrupt handler or from a bottom half handler.
232 int smp_call_function (void (*func
) (void *info
), void *info
, int nonatomic
,
235 struct call_data_struct data
;
237 unsigned long timeout
;
239 /* Can deadlock when called with interrupts disabled */
240 WARN_ON(irqs_disabled());
244 atomic_set(&data
.started
, 0);
247 atomic_set(&data
.finished
, 0);
249 spin_lock(&call_lock
);
250 /* Must grab online cpu count with preempt disabled, otherwise
252 cpus
= num_online_cpus() - 1;
260 /* Send a message to all other CPUs and wait for them to respond */
261 smp_ops
->message_pass(MSG_ALL_BUT_SELF
, PPC_MSG_CALL_FUNCTION
);
263 /* Wait for response */
264 timeout
= SMP_CALL_TIMEOUT
;
265 while (atomic_read(&data
.started
) != cpus
) {
267 if (--timeout
== 0) {
268 printk("smp_call_function on cpu %d: other cpus not "
269 "responding (%d)\n", smp_processor_id(),
270 atomic_read(&data
.started
));
277 timeout
= SMP_CALL_TIMEOUT
;
278 while (atomic_read(&data
.finished
) != cpus
) {
280 if (--timeout
== 0) {
281 printk("smp_call_function on cpu %d: other "
282 "cpus not finishing (%d/%d)\n",
284 atomic_read(&data
.finished
),
285 atomic_read(&data
.started
));
297 spin_unlock(&call_lock
);
301 EXPORT_SYMBOL(smp_call_function
);
303 void smp_call_function_interrupt(void)
305 void (*func
) (void *info
);
309 /* call_data will be NULL if the sender timed out while
310 * waiting on us to receive the call.
315 func
= call_data
->func
;
316 info
= call_data
->info
;
317 wait
= call_data
->wait
;
320 smp_mb__before_atomic_inc();
323 * Notify initiating CPU that I've grabbed the data and am
324 * about to execute the function
326 atomic_inc(&call_data
->started
);
328 * At this point the info structure may be out of scope unless wait==1
332 smp_mb__before_atomic_inc();
333 atomic_inc(&call_data
->finished
);
337 extern struct gettimeofday_struct do_gtod
;
339 struct thread_info
*current_set
[NR_CPUS
];
341 DECLARE_PER_CPU(unsigned int, pvr
);
343 static void __devinit
smp_store_cpu_info(int id
)
345 per_cpu(pvr
, id
) = mfspr(SPRN_PVR
);
348 static void __init
smp_create_idle(unsigned int cpu
)
350 struct task_struct
*p
;
352 /* create a process for the processor */
355 panic("failed fork for CPU %u: %li", cpu
, PTR_ERR(p
));
356 paca
[cpu
].__current
= p
;
357 current_set
[cpu
] = p
->thread_info
;
360 void __init
smp_prepare_cpus(unsigned int max_cpus
)
364 DBG("smp_prepare_cpus\n");
367 * setup_cpu may need to be called on the boot cpu. We havent
368 * spun any cpus up but lets be paranoid.
370 BUG_ON(boot_cpuid
!= smp_processor_id());
373 smp_store_cpu_info(boot_cpuid
);
374 cpu_callin_map
[boot_cpuid
] = 1;
376 #ifndef CONFIG_PPC_ISERIES
377 paca
[boot_cpuid
].next_jiffy_update_tb
= tb_last_stamp
= get_tb();
380 * Should update do_gtod.stamp_xsec.
381 * For now we leave it which means the time can be some
382 * number of msecs off until someone does a settimeofday()
384 do_gtod
.varp
->tb_orig_stamp
= tb_last_stamp
;
385 systemcfg
->tb_orig_stamp
= tb_last_stamp
;
388 max_cpus
= smp_ops
->probe();
390 smp_space_timers(max_cpus
);
393 if (cpu
!= boot_cpuid
)
394 smp_create_idle(cpu
);
397 void __devinit
smp_prepare_boot_cpu(void)
399 BUG_ON(smp_processor_id() != boot_cpuid
);
401 cpu_set(boot_cpuid
, cpu_online_map
);
403 paca
[boot_cpuid
].__current
= current
;
404 current_set
[boot_cpuid
] = current
->thread_info
;
407 #ifdef CONFIG_HOTPLUG_CPU
408 /* State of each CPU during hotplug phases */
409 DEFINE_PER_CPU(int, cpu_state
) = { 0 };
411 int generic_cpu_disable(void)
413 unsigned int cpu
= smp_processor_id();
415 if (cpu
== boot_cpuid
)
418 systemcfg
->processorCount
--;
419 cpu_clear(cpu
, cpu_online_map
);
420 fixup_irqs(cpu_online_map
);
424 int generic_cpu_enable(unsigned int cpu
)
426 /* Do the normal bootup if we haven't
427 * already bootstrapped. */
428 if (system_state
!= SYSTEM_RUNNING
)
431 /* get the target out of it's holding state */
432 per_cpu(cpu_state
, cpu
) = CPU_UP_PREPARE
;
435 while (!cpu_online(cpu
))
438 fixup_irqs(cpu_online_map
);
439 /* counter the irq disable in fixup_irqs */
444 void generic_cpu_die(unsigned int cpu
)
448 for (i
= 0; i
< 100; i
++) {
450 if (per_cpu(cpu_state
, cpu
) == CPU_DEAD
)
454 printk(KERN_ERR
"CPU%d didn't die...\n", cpu
);
457 void generic_mach_cpu_die(void)
462 cpu
= smp_processor_id();
463 printk(KERN_DEBUG
"CPU%d offline\n", cpu
);
464 __get_cpu_var(cpu_state
) = CPU_DEAD
;
466 while (__get_cpu_var(cpu_state
) != CPU_UP_PREPARE
)
470 cpu_set(cpu
, cpu_online_map
);
475 static int __devinit
cpu_enable(unsigned int cpu
)
477 if (smp_ops
->cpu_enable
)
478 return smp_ops
->cpu_enable(cpu
);
483 int __devinit
__cpu_up(unsigned int cpu
)
487 if (!cpu_enable(cpu
))
490 if (smp_ops
->cpu_bootable
&& !smp_ops
->cpu_bootable(cpu
))
493 paca
[cpu
].default_decr
= tb_ticks_per_jiffy
;
495 if (!cpu_has_feature(CPU_FTR_SLB
)) {
498 /* maximum of 48 CPUs on machines with a segment table */
502 tmp
= &stab_array
[PAGE_SIZE
* cpu
];
503 memset(tmp
, 0, PAGE_SIZE
);
504 paca
[cpu
].stab_addr
= (unsigned long)tmp
;
505 paca
[cpu
].stab_real
= virt_to_abs(tmp
);
508 /* Make sure callin-map entry is 0 (can be leftover a CPU
511 cpu_callin_map
[cpu
] = 0;
513 /* The information for processor bringup must
514 * be written out to main store before we release
520 DBG("smp: kicking cpu %d\n", cpu
);
521 smp_ops
->kick_cpu(cpu
);
524 * wait to see if the cpu made a callin (is actually up).
525 * use this value that I found through experimentation.
528 if (system_state
< SYSTEM_RUNNING
)
529 for (c
= 5000; c
&& !cpu_callin_map
[cpu
]; c
--)
531 #ifdef CONFIG_HOTPLUG_CPU
534 * CPUs can take much longer to come up in the
535 * hotplug case. Wait five seconds.
537 for (c
= 25; c
&& !cpu_callin_map
[cpu
]; c
--) {
542 if (!cpu_callin_map
[cpu
]) {
543 printk("Processor %u is stuck.\n", cpu
);
547 printk("Processor %u found.\n", cpu
);
549 if (smp_ops
->give_timebase
)
550 smp_ops
->give_timebase();
552 /* Wait until cpu puts itself in the online map */
553 while (!cpu_online(cpu
))
560 /* Activate a secondary processor. */
561 int __devinit
start_secondary(void *unused
)
563 unsigned int cpu
= smp_processor_id();
565 atomic_inc(&init_mm
.mm_count
);
566 current
->active_mm
= &init_mm
;
568 smp_store_cpu_info(cpu
);
569 set_dec(paca
[cpu
].default_decr
);
570 cpu_callin_map
[cpu
] = 1;
572 smp_ops
->setup_cpu(cpu
);
573 if (smp_ops
->take_timebase
)
574 smp_ops
->take_timebase();
576 spin_lock(&call_lock
);
577 cpu_set(cpu
, cpu_online_map
);
578 spin_unlock(&call_lock
);
586 int setup_profiling_timer(unsigned int multiplier
)
591 void __init
smp_cpus_done(unsigned int max_cpus
)
595 /* We want the setup_cpu() here to be called from CPU 0, but our
596 * init thread may have been "borrowed" by another CPU in the meantime
597 * se we pin us down to CPU 0 for a short while
599 old_mask
= current
->cpus_allowed
;
600 set_cpus_allowed(current
, cpumask_of_cpu(boot_cpuid
));
602 smp_ops
->setup_cpu(boot_cpuid
);
604 set_cpus_allowed(current
, old_mask
);
607 #ifdef CONFIG_HOTPLUG_CPU
608 int __cpu_disable(void)
610 if (smp_ops
->cpu_disable
)
611 return smp_ops
->cpu_disable();
616 void __cpu_die(unsigned int cpu
)
618 if (smp_ops
->cpu_die
)
619 smp_ops
->cpu_die(cpu
);