2 * SMP related functions
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Denis Joseph Barrow,
6 * Martin Schwidefsky <schwidefsky@de.ibm.com>,
7 * Heiko Carstens <heiko.carstens@de.ibm.com>,
9 * based on other smp stuff by
10 * (c) 1995 Alan Cox, CymruNET Ltd <alan@cymru.net>
11 * (c) 1998 Ingo Molnar
13 * The code outside of smp.c uses logical cpu numbers, only smp.c does
14 * the translation of logical to physical cpu ids. All new code that
15 * operates on physical cpu numbers needs to go into smp.c.
18 #define KMSG_COMPONENT "cpu"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 #include <linux/workqueue.h>
22 #include <linux/module.h>
23 #include <linux/init.h>
25 #include <linux/err.h>
26 #include <linux/spinlock.h>
27 #include <linux/kernel_stat.h>
28 #include <linux/delay.h>
29 #include <linux/interrupt.h>
30 #include <linux/irqflags.h>
31 #include <linux/cpu.h>
32 #include <linux/slab.h>
33 #include <linux/crash_dump.h>
34 #include <linux/memblock.h>
35 #include <asm/asm-offsets.h>
36 #include <asm/switch_to.h>
37 #include <asm/facility.h>
39 #include <asm/setup.h>
41 #include <asm/tlbflush.h>
42 #include <asm/vtimer.h>
43 #include <asm/lowcore.h>
46 #include <asm/debug.h>
47 #include <asm/os_info.h>
54 ec_call_function_single
,
63 static DEFINE_PER_CPU(struct cpu
*, cpu_device
);
66 struct _lowcore
*lowcore
; /* lowcore page(s) for the cpu */
67 unsigned long ec_mask
; /* bit mask for ec_xxx functions */
68 signed char state
; /* physical cpu state */
69 signed char polarization
; /* physical polarization */
70 u16 address
; /* physical cpu address */
73 static u8 boot_core_type
;
74 static struct pcpu pcpu_devices
[NR_CPUS
];
76 unsigned int smp_cpu_mt_shift
;
77 EXPORT_SYMBOL(smp_cpu_mt_shift
);
79 unsigned int smp_cpu_mtid
;
80 EXPORT_SYMBOL(smp_cpu_mtid
);
82 static unsigned int smp_max_threads __initdata
= -1U;
84 static int __init
early_nosmt(char *s
)
89 early_param("nosmt", early_nosmt
);
91 static int __init
early_smt(char *s
)
93 get_option(&s
, &smp_max_threads
);
96 early_param("smt", early_smt
);
99 * The smp_cpu_state_mutex must be held when changing the state or polarization
100 * member of a pcpu data structure within the pcpu_devices arreay.
102 DEFINE_MUTEX(smp_cpu_state_mutex
);
105 * Signal processor helper functions.
107 static inline int __pcpu_sigp_relax(u16 addr
, u8 order
, unsigned long parm
,
113 cc
= __pcpu_sigp(addr
, order
, parm
, NULL
);
114 if (cc
!= SIGP_CC_BUSY
)
120 static int pcpu_sigp_retry(struct pcpu
*pcpu
, u8 order
, u32 parm
)
124 for (retry
= 0; ; retry
++) {
125 cc
= __pcpu_sigp(pcpu
->address
, order
, parm
, NULL
);
126 if (cc
!= SIGP_CC_BUSY
)
134 static inline int pcpu_stopped(struct pcpu
*pcpu
)
136 u32
uninitialized_var(status
);
138 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE
,
139 0, &status
) != SIGP_CC_STATUS_STORED
)
141 return !!(status
& (SIGP_STATUS_CHECK_STOP
|SIGP_STATUS_STOPPED
));
144 static inline int pcpu_running(struct pcpu
*pcpu
)
146 if (__pcpu_sigp(pcpu
->address
, SIGP_SENSE_RUNNING
,
147 0, NULL
) != SIGP_CC_STATUS_STORED
)
149 /* Status stored condition code is equivalent to cpu not running. */
154 * Find struct pcpu by cpu address.
156 static struct pcpu
*pcpu_find_address(const struct cpumask
*mask
, u16 address
)
160 for_each_cpu(cpu
, mask
)
161 if (pcpu_devices
[cpu
].address
== address
)
162 return pcpu_devices
+ cpu
;
166 static void pcpu_ec_call(struct pcpu
*pcpu
, int ec_bit
)
170 if (test_and_set_bit(ec_bit
, &pcpu
->ec_mask
))
172 order
= pcpu_running(pcpu
) ? SIGP_EXTERNAL_CALL
: SIGP_EMERGENCY_SIGNAL
;
173 pcpu_sigp_retry(pcpu
, order
, 0);
176 #define ASYNC_FRAME_OFFSET (ASYNC_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
177 #define PANIC_FRAME_OFFSET (PAGE_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE)
179 static int pcpu_alloc_lowcore(struct pcpu
*pcpu
, int cpu
)
181 unsigned long async_stack
, panic_stack
;
184 if (pcpu
!= &pcpu_devices
[0]) {
185 pcpu
->lowcore
= (struct _lowcore
*)
186 __get_free_pages(GFP_KERNEL
| GFP_DMA
, LC_ORDER
);
187 async_stack
= __get_free_pages(GFP_KERNEL
, ASYNC_ORDER
);
188 panic_stack
= __get_free_page(GFP_KERNEL
);
189 if (!pcpu
->lowcore
|| !panic_stack
|| !async_stack
)
192 async_stack
= pcpu
->lowcore
->async_stack
- ASYNC_FRAME_OFFSET
;
193 panic_stack
= pcpu
->lowcore
->panic_stack
- PANIC_FRAME_OFFSET
;
196 memcpy(lc
, &S390_lowcore
, 512);
197 memset((char *) lc
+ 512, 0, sizeof(*lc
) - 512);
198 lc
->async_stack
= async_stack
+ ASYNC_FRAME_OFFSET
;
199 lc
->panic_stack
= panic_stack
+ PANIC_FRAME_OFFSET
;
201 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
203 lc
->vector_save_area_addr
=
204 (unsigned long) &lc
->vector_save_area
;
205 if (vdso_alloc_per_cpu(lc
))
207 lowcore_ptr
[cpu
] = lc
;
208 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, (u32
)(unsigned long) lc
);
211 if (pcpu
!= &pcpu_devices
[0]) {
212 free_page(panic_stack
);
213 free_pages(async_stack
, ASYNC_ORDER
);
214 free_pages((unsigned long) pcpu
->lowcore
, LC_ORDER
);
219 #ifdef CONFIG_HOTPLUG_CPU
221 static void pcpu_free_lowcore(struct pcpu
*pcpu
)
223 pcpu_sigp_retry(pcpu
, SIGP_SET_PREFIX
, 0);
224 lowcore_ptr
[pcpu
- pcpu_devices
] = NULL
;
225 vdso_free_per_cpu(pcpu
->lowcore
);
226 if (pcpu
== &pcpu_devices
[0])
228 free_page(pcpu
->lowcore
->panic_stack
-PANIC_FRAME_OFFSET
);
229 free_pages(pcpu
->lowcore
->async_stack
-ASYNC_FRAME_OFFSET
, ASYNC_ORDER
);
230 free_pages((unsigned long) pcpu
->lowcore
, LC_ORDER
);
233 #endif /* CONFIG_HOTPLUG_CPU */
235 static void pcpu_prepare_secondary(struct pcpu
*pcpu
, int cpu
)
237 struct _lowcore
*lc
= pcpu
->lowcore
;
239 if (MACHINE_HAS_TLB_LC
)
240 cpumask_set_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
241 cpumask_set_cpu(cpu
, mm_cpumask(&init_mm
));
242 atomic_inc(&init_mm
.context
.attach_count
);
244 lc
->spinlock_lockval
= arch_spin_lockval(cpu
);
245 lc
->percpu_offset
= __per_cpu_offset
[cpu
];
246 lc
->kernel_asce
= S390_lowcore
.kernel_asce
;
247 lc
->machine_flags
= S390_lowcore
.machine_flags
;
248 lc
->user_timer
= lc
->system_timer
= lc
->steal_timer
= 0;
249 __ctl_store(lc
->cregs_save_area
, 0, 15);
250 save_access_regs((unsigned int *) lc
->access_regs_save_area
);
251 memcpy(lc
->stfle_fac_list
, S390_lowcore
.stfle_fac_list
,
255 static void pcpu_attach_task(struct pcpu
*pcpu
, struct task_struct
*tsk
)
257 struct _lowcore
*lc
= pcpu
->lowcore
;
258 struct thread_info
*ti
= task_thread_info(tsk
);
260 lc
->kernel_stack
= (unsigned long) task_stack_page(tsk
)
261 + THREAD_SIZE
- STACK_FRAME_OVERHEAD
- sizeof(struct pt_regs
);
262 lc
->thread_info
= (unsigned long) task_thread_info(tsk
);
263 lc
->current_task
= (unsigned long) tsk
;
264 lc
->user_timer
= ti
->user_timer
;
265 lc
->system_timer
= ti
->system_timer
;
269 static void pcpu_start_fn(struct pcpu
*pcpu
, void (*func
)(void *), void *data
)
271 struct _lowcore
*lc
= pcpu
->lowcore
;
273 lc
->restart_stack
= lc
->kernel_stack
;
274 lc
->restart_fn
= (unsigned long) func
;
275 lc
->restart_data
= (unsigned long) data
;
276 lc
->restart_source
= -1UL;
277 pcpu_sigp_retry(pcpu
, SIGP_RESTART
, 0);
281 * Call function via PSW restart on pcpu and stop the current cpu.
283 static void pcpu_delegate(struct pcpu
*pcpu
, void (*func
)(void *),
284 void *data
, unsigned long stack
)
286 struct _lowcore
*lc
= lowcore_ptr
[pcpu
- pcpu_devices
];
287 unsigned long source_cpu
= stap();
289 __load_psw_mask(PSW_KERNEL_BITS
);
290 if (pcpu
->address
== source_cpu
)
291 func(data
); /* should not return */
292 /* Stop target cpu (if func returns this stops the current cpu). */
293 pcpu_sigp_retry(pcpu
, SIGP_STOP
, 0);
294 /* Restart func on the target cpu and stop the current cpu. */
295 mem_assign_absolute(lc
->restart_stack
, stack
);
296 mem_assign_absolute(lc
->restart_fn
, (unsigned long) func
);
297 mem_assign_absolute(lc
->restart_data
, (unsigned long) data
);
298 mem_assign_absolute(lc
->restart_source
, source_cpu
);
300 "0: sigp 0,%0,%2 # sigp restart to target cpu\n"
301 " brc 2,0b # busy, try again\n"
302 "1: sigp 0,%1,%3 # sigp stop to current cpu\n"
303 " brc 2,1b # busy, try again\n"
304 : : "d" (pcpu
->address
), "d" (source_cpu
),
305 "K" (SIGP_RESTART
), "K" (SIGP_STOP
)
311 * Enable additional logical cpus for multi-threading.
313 static int pcpu_set_smt(unsigned int mtid
)
315 register unsigned long reg1
asm ("1") = (unsigned long) mtid
;
318 if (smp_cpu_mtid
== mtid
)
321 " sigp %1,0,%2 # sigp set multi-threading\n"
324 : "=d" (cc
) : "d" (reg1
), "K" (SIGP_SET_MULTI_THREADING
)
328 smp_cpu_mt_shift
= 0;
329 while (smp_cpu_mtid
>= (1U << smp_cpu_mt_shift
))
331 pcpu_devices
[0].address
= stap();
337 * Call function on an online CPU.
339 void smp_call_online_cpu(void (*func
)(void *), void *data
)
343 /* Use the current cpu if it is online. */
344 pcpu
= pcpu_find_address(cpu_online_mask
, stap());
346 /* Use the first online cpu. */
347 pcpu
= pcpu_devices
+ cpumask_first(cpu_online_mask
);
348 pcpu_delegate(pcpu
, func
, data
, (unsigned long) restart_stack
);
352 * Call function on the ipl CPU.
354 void smp_call_ipl_cpu(void (*func
)(void *), void *data
)
356 pcpu_delegate(&pcpu_devices
[0], func
, data
,
357 pcpu_devices
->lowcore
->panic_stack
-
358 PANIC_FRAME_OFFSET
+ PAGE_SIZE
);
361 int smp_find_processor_id(u16 address
)
365 for_each_present_cpu(cpu
)
366 if (pcpu_devices
[cpu
].address
== address
)
371 int smp_vcpu_scheduled(int cpu
)
373 return pcpu_running(pcpu_devices
+ cpu
);
376 void smp_yield_cpu(int cpu
)
378 if (MACHINE_HAS_DIAG9C
)
379 asm volatile("diag %0,0,0x9c"
380 : : "d" (pcpu_devices
[cpu
].address
));
381 else if (MACHINE_HAS_DIAG44
)
382 asm volatile("diag 0,0,0x44");
386 * Send cpus emergency shutdown signal. This gives the cpus the
387 * opportunity to complete outstanding interrupts.
389 static void smp_emergency_stop(cpumask_t
*cpumask
)
394 end
= get_tod_clock() + (1000000UL << 12);
395 for_each_cpu(cpu
, cpumask
) {
396 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
397 set_bit(ec_stop_cpu
, &pcpu
->ec_mask
);
398 while (__pcpu_sigp(pcpu
->address
, SIGP_EMERGENCY_SIGNAL
,
399 0, NULL
) == SIGP_CC_BUSY
&&
400 get_tod_clock() < end
)
403 while (get_tod_clock() < end
) {
404 for_each_cpu(cpu
, cpumask
)
405 if (pcpu_stopped(pcpu_devices
+ cpu
))
406 cpumask_clear_cpu(cpu
, cpumask
);
407 if (cpumask_empty(cpumask
))
414 * Stop all cpus but the current one.
416 void smp_send_stop(void)
421 /* Disable all interrupts/machine checks */
422 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
423 trace_hardirqs_off();
425 debug_set_critical();
426 cpumask_copy(&cpumask
, cpu_online_mask
);
427 cpumask_clear_cpu(smp_processor_id(), &cpumask
);
429 if (oops_in_progress
)
430 smp_emergency_stop(&cpumask
);
432 /* stop all processors */
433 for_each_cpu(cpu
, &cpumask
) {
434 struct pcpu
*pcpu
= pcpu_devices
+ cpu
;
435 pcpu_sigp_retry(pcpu
, SIGP_STOP
, 0);
436 while (!pcpu_stopped(pcpu
))
442 * This is the main routine where commands issued by other
445 static void smp_handle_ext_call(void)
449 /* handle bit signal external calls */
450 bits
= xchg(&pcpu_devices
[smp_processor_id()].ec_mask
, 0);
451 if (test_bit(ec_stop_cpu
, &bits
))
453 if (test_bit(ec_schedule
, &bits
))
455 if (test_bit(ec_call_function_single
, &bits
))
456 generic_smp_call_function_single_interrupt();
459 static void do_ext_call_interrupt(struct ext_code ext_code
,
460 unsigned int param32
, unsigned long param64
)
462 inc_irq_stat(ext_code
.code
== 0x1202 ? IRQEXT_EXC
: IRQEXT_EMS
);
463 smp_handle_ext_call();
466 void arch_send_call_function_ipi_mask(const struct cpumask
*mask
)
470 for_each_cpu(cpu
, mask
)
471 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
474 void arch_send_call_function_single_ipi(int cpu
)
476 pcpu_ec_call(pcpu_devices
+ cpu
, ec_call_function_single
);
480 * this function sends a 'reschedule' IPI to another CPU.
481 * it goes straight through and wastes no time serializing
482 * anything. Worst case is that we lose a reschedule ...
484 void smp_send_reschedule(int cpu
)
486 pcpu_ec_call(pcpu_devices
+ cpu
, ec_schedule
);
490 * parameter area for the set/clear control bit callbacks
492 struct ec_creg_mask_parms
{
494 unsigned long andval
;
499 * callback for setting/clearing control bits
501 static void smp_ctl_bit_callback(void *info
)
503 struct ec_creg_mask_parms
*pp
= info
;
504 unsigned long cregs
[16];
506 __ctl_store(cregs
, 0, 15);
507 cregs
[pp
->cr
] = (cregs
[pp
->cr
] & pp
->andval
) | pp
->orval
;
508 __ctl_load(cregs
, 0, 15);
512 * Set a bit in a control register of all cpus
514 void smp_ctl_set_bit(int cr
, int bit
)
516 struct ec_creg_mask_parms parms
= { 1UL << bit
, -1UL, cr
};
518 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
520 EXPORT_SYMBOL(smp_ctl_set_bit
);
523 * Clear a bit in a control register of all cpus
525 void smp_ctl_clear_bit(int cr
, int bit
)
527 struct ec_creg_mask_parms parms
= { 0, ~(1UL << bit
), cr
};
529 on_each_cpu(smp_ctl_bit_callback
, &parms
, 1);
531 EXPORT_SYMBOL(smp_ctl_clear_bit
);
533 #ifdef CONFIG_CRASH_DUMP
535 static void __smp_store_cpu_state(struct save_area_ext
*sa_ext
, u16 address
,
538 void *lc
= (void *)(unsigned long) store_prefix();
542 /* Copy the registers of the boot CPU. */
543 copy_oldmem_page(1, (void *) &sa_ext
->sa
, sizeof(sa_ext
->sa
),
544 SAVE_AREA_BASE
- PAGE_SIZE
, 0);
546 save_vx_regs_safe(sa_ext
->vx_regs
);
549 /* Get the registers of a non-boot cpu. */
550 __pcpu_sigp_relax(address
, SIGP_STOP_AND_STORE_STATUS
, 0, NULL
);
551 memcpy_real(&sa_ext
->sa
, lc
+ SAVE_AREA_BASE
, sizeof(sa_ext
->sa
));
554 /* Get the VX registers */
555 vx_sa
= memblock_alloc(PAGE_SIZE
, PAGE_SIZE
);
557 panic("could not allocate memory for VX save area\n");
558 __pcpu_sigp_relax(address
, SIGP_STORE_ADDITIONAL_STATUS
, vx_sa
, NULL
);
559 memcpy(sa_ext
->vx_regs
, (void *) vx_sa
, sizeof(sa_ext
->vx_regs
));
560 memblock_free(vx_sa
, PAGE_SIZE
);
563 int smp_store_status(int cpu
)
568 pcpu
= pcpu_devices
+ cpu
;
569 if (__pcpu_sigp_relax(pcpu
->address
, SIGP_STOP_AND_STORE_STATUS
,
570 0, NULL
) != SIGP_CC_ORDER_CODE_ACCEPTED
)
574 vx_sa
= __pa(pcpu
->lowcore
->vector_save_area_addr
);
575 __pcpu_sigp_relax(pcpu
->address
, SIGP_STORE_ADDITIONAL_STATUS
,
580 #endif /* CONFIG_CRASH_DUMP */
583 * Collect CPU state of the previous, crashed system.
584 * There are four cases:
585 * 1) standard zfcp dump
586 * condition: OLDMEM_BASE == NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
587 * The state for all CPUs except the boot CPU needs to be collected
588 * with sigp stop-and-store-status. The boot CPU state is located in
589 * the absolute lowcore of the memory stored in the HSA. The zcore code
590 * will allocate the save area and copy the boot CPU state from the HSA.
591 * 2) stand-alone kdump for SCSI (zfcp dump with swapped memory)
592 * condition: OLDMEM_BASE != NULL && ipl_info.type == IPL_TYPE_FCP_DUMP
593 * The state for all CPUs except the boot CPU needs to be collected
594 * with sigp stop-and-store-status. The firmware or the boot-loader
595 * stored the registers of the boot CPU in the absolute lowcore in the
596 * memory of the old system.
597 * 3) kdump and the old kernel did not store the CPU state,
598 * or stand-alone kdump for DASD
599 * condition: OLDMEM_BASE != NULL && !is_kdump_kernel()
600 * The state for all CPUs except the boot CPU needs to be collected
601 * with sigp stop-and-store-status. The kexec code or the boot-loader
602 * stored the registers of the boot CPU in the memory of the old system.
603 * 4) kdump and the old kernel stored the CPU state
604 * condition: OLDMEM_BASE != NULL && is_kdump_kernel()
605 * The state of all CPUs is stored in ELF sections in the memory of the
606 * old system. The ELF sections are picked up by the crash_dump code
607 * via elfcorehdr_addr.
609 void __init
smp_save_dump_cpus(void)
611 #ifdef CONFIG_CRASH_DUMP
612 int addr
, cpu
, boot_cpu_addr
, max_cpu_addr
;
613 struct save_area_ext
*sa_ext
;
616 if (is_kdump_kernel())
617 /* Previous system stored the CPU states. Nothing to do. */
619 if (!(OLDMEM_BASE
|| ipl_info
.type
== IPL_TYPE_FCP_DUMP
))
620 /* No previous system present, normal boot. */
622 /* Set multi-threading state to the previous system. */
623 pcpu_set_smt(sclp
.mtid_prev
);
624 max_cpu_addr
= SCLP_MAX_CORES
<< sclp
.mtid_prev
;
625 for (cpu
= 0, addr
= 0; addr
<= max_cpu_addr
; addr
++) {
626 if (__pcpu_sigp_relax(addr
, SIGP_SENSE
, 0, NULL
) ==
627 SIGP_CC_NOT_OPERATIONAL
)
631 dump_save_areas
.areas
= (void *)memblock_alloc(sizeof(void *) * cpu
, 8);
632 dump_save_areas
.count
= cpu
;
633 boot_cpu_addr
= stap();
634 for (cpu
= 0, addr
= 0; addr
<= max_cpu_addr
; addr
++) {
635 if (__pcpu_sigp_relax(addr
, SIGP_SENSE
, 0, NULL
) ==
636 SIGP_CC_NOT_OPERATIONAL
)
638 sa_ext
= (void *) memblock_alloc(sizeof(*sa_ext
), 8);
639 dump_save_areas
.areas
[cpu
] = sa_ext
;
641 panic("could not allocate memory for save area\n");
642 is_boot_cpu
= (addr
== boot_cpu_addr
);
644 if (is_boot_cpu
&& !OLDMEM_BASE
)
645 /* Skip boot CPU for standard zfcp dump. */
647 /* Get state for this CPU. */
648 __smp_store_cpu_state(sa_ext
, addr
, is_boot_cpu
);
652 #endif /* CONFIG_CRASH_DUMP */
655 void smp_cpu_set_polarization(int cpu
, int val
)
657 pcpu_devices
[cpu
].polarization
= val
;
660 int smp_cpu_get_polarization(int cpu
)
662 return pcpu_devices
[cpu
].polarization
;
665 static struct sclp_core_info
*smp_get_core_info(void)
667 static int use_sigp_detection
;
668 struct sclp_core_info
*info
;
671 info
= kzalloc(sizeof(*info
), GFP_KERNEL
);
672 if (info
&& (use_sigp_detection
|| sclp_get_core_info(info
))) {
673 use_sigp_detection
= 1;
675 address
< (SCLP_MAX_CORES
<< smp_cpu_mt_shift
);
676 address
+= (1U << smp_cpu_mt_shift
)) {
677 if (__pcpu_sigp_relax(address
, SIGP_SENSE
, 0, NULL
) ==
678 SIGP_CC_NOT_OPERATIONAL
)
680 info
->core
[info
->configured
].core_id
=
681 address
>> smp_cpu_mt_shift
;
684 info
->combined
= info
->configured
;
689 static int smp_add_present_cpu(int cpu
);
691 static int __smp_rescan_cpus(struct sclp_core_info
*info
, int sysfs_add
)
699 cpumask_xor(&avail
, cpu_possible_mask
, cpu_present_mask
);
700 cpu
= cpumask_first(&avail
);
701 for (i
= 0; (i
< info
->combined
) && (cpu
< nr_cpu_ids
); i
++) {
702 if (sclp
.has_core_type
&& info
->core
[i
].type
!= boot_core_type
)
704 address
= info
->core
[i
].core_id
<< smp_cpu_mt_shift
;
705 for (j
= 0; j
<= smp_cpu_mtid
; j
++) {
706 if (pcpu_find_address(cpu_present_mask
, address
+ j
))
708 pcpu
= pcpu_devices
+ cpu
;
709 pcpu
->address
= address
+ j
;
711 (cpu
>= info
->configured
*(smp_cpu_mtid
+ 1)) ?
712 CPU_STATE_STANDBY
: CPU_STATE_CONFIGURED
;
713 smp_cpu_set_polarization(cpu
, POLARIZATION_UNKNOWN
);
714 set_cpu_present(cpu
, true);
715 if (sysfs_add
&& smp_add_present_cpu(cpu
) != 0)
716 set_cpu_present(cpu
, false);
719 cpu
= cpumask_next(cpu
, &avail
);
720 if (cpu
>= nr_cpu_ids
)
727 static void __init
smp_detect_cpus(void)
729 unsigned int cpu
, mtid
, c_cpus
, s_cpus
;
730 struct sclp_core_info
*info
;
733 /* Get CPU information */
734 info
= smp_get_core_info();
736 panic("smp_detect_cpus failed to allocate memory\n");
738 /* Find boot CPU type */
739 if (sclp
.has_core_type
) {
741 for (cpu
= 0; cpu
< info
->combined
; cpu
++)
742 if (info
->core
[cpu
].core_id
== address
) {
743 /* The boot cpu dictates the cpu type. */
744 boot_core_type
= info
->core
[cpu
].type
;
747 if (cpu
>= info
->combined
)
748 panic("Could not find boot CPU type");
751 /* Set multi-threading state for the current system */
752 mtid
= boot_core_type
? sclp
.mtid
: sclp
.mtid_cp
;
753 mtid
= (mtid
< smp_max_threads
) ? mtid
: smp_max_threads
- 1;
756 /* Print number of CPUs */
758 for (cpu
= 0; cpu
< info
->combined
; cpu
++) {
759 if (sclp
.has_core_type
&&
760 info
->core
[cpu
].type
!= boot_core_type
)
762 if (cpu
< info
->configured
)
763 c_cpus
+= smp_cpu_mtid
+ 1;
765 s_cpus
+= smp_cpu_mtid
+ 1;
767 pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus
, s_cpus
);
769 /* Add CPUs present at boot */
771 __smp_rescan_cpus(info
, 0);
777 * Activate a secondary processor.
779 static void smp_start_secondary(void *cpuvoid
)
781 S390_lowcore
.last_update_clock
= get_tod_clock();
782 S390_lowcore
.restart_stack
= (unsigned long) restart_stack
;
783 S390_lowcore
.restart_fn
= (unsigned long) do_restart
;
784 S390_lowcore
.restart_data
= 0;
785 S390_lowcore
.restart_source
= -1UL;
786 restore_access_regs(S390_lowcore
.access_regs_save_area
);
787 __ctl_load(S390_lowcore
.cregs_save_area
, 0, 15);
788 __load_psw_mask(PSW_KERNEL_BITS
| PSW_MASK_DAT
);
794 notify_cpu_starting(smp_processor_id());
795 set_cpu_online(smp_processor_id(), true);
796 inc_irq_stat(CPU_RST
);
798 cpu_startup_entry(CPUHP_ONLINE
);
801 /* Upping and downing of CPUs */
802 int __cpu_up(unsigned int cpu
, struct task_struct
*tidle
)
807 pcpu
= pcpu_devices
+ cpu
;
808 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
810 base
= cpu
- (cpu
% (smp_cpu_mtid
+ 1));
811 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
812 if (base
+ i
< nr_cpu_ids
)
813 if (cpu_online(base
+ i
))
817 * If this is the first CPU of the core to get online
818 * do an initial CPU reset.
820 if (i
> smp_cpu_mtid
&&
821 pcpu_sigp_retry(pcpu_devices
+ base
, SIGP_INITIAL_CPU_RESET
, 0) !=
822 SIGP_CC_ORDER_CODE_ACCEPTED
)
825 rc
= pcpu_alloc_lowcore(pcpu
, cpu
);
828 pcpu_prepare_secondary(pcpu
, cpu
);
829 pcpu_attach_task(pcpu
, tidle
);
830 pcpu_start_fn(pcpu
, smp_start_secondary
, NULL
);
831 /* Wait until cpu puts itself in the online & active maps */
832 while (!cpu_online(cpu
) || !cpu_active(cpu
))
837 static unsigned int setup_possible_cpus __initdata
;
839 static int __init
_setup_possible_cpus(char *s
)
841 get_option(&s
, &setup_possible_cpus
);
844 early_param("possible_cpus", _setup_possible_cpus
);
846 #ifdef CONFIG_HOTPLUG_CPU
848 int __cpu_disable(void)
850 unsigned long cregs
[16];
852 /* Handle possible pending IPIs */
853 smp_handle_ext_call();
854 set_cpu_online(smp_processor_id(), false);
855 /* Disable pseudo page faults on this cpu. */
857 /* Disable interrupt sources via control register. */
858 __ctl_store(cregs
, 0, 15);
859 cregs
[0] &= ~0x0000ee70UL
; /* disable all external interrupts */
860 cregs
[6] &= ~0xff000000UL
; /* disable all I/O interrupts */
861 cregs
[14] &= ~0x1f000000UL
; /* disable most machine checks */
862 __ctl_load(cregs
, 0, 15);
863 clear_cpu_flag(CIF_NOHZ_DELAY
);
867 void __cpu_die(unsigned int cpu
)
871 /* Wait until target cpu is down */
872 pcpu
= pcpu_devices
+ cpu
;
873 while (!pcpu_stopped(pcpu
))
875 pcpu_free_lowcore(pcpu
);
876 atomic_dec(&init_mm
.context
.attach_count
);
877 cpumask_clear_cpu(cpu
, mm_cpumask(&init_mm
));
878 if (MACHINE_HAS_TLB_LC
)
879 cpumask_clear_cpu(cpu
, &init_mm
.context
.cpu_attach_mask
);
882 void __noreturn
cpu_die(void)
885 pcpu_sigp_retry(pcpu_devices
+ smp_processor_id(), SIGP_STOP
, 0);
889 #endif /* CONFIG_HOTPLUG_CPU */
891 void __init
smp_fill_possible_mask(void)
893 unsigned int possible
, sclp_max
, cpu
;
895 sclp_max
= max(sclp
.mtid
, sclp
.mtid_cp
) + 1;
896 sclp_max
= min(smp_max_threads
, sclp_max
);
897 sclp_max
= sclp
.max_cores
* sclp_max
?: nr_cpu_ids
;
898 possible
= setup_possible_cpus
?: nr_cpu_ids
;
899 possible
= min(possible
, sclp_max
);
900 for (cpu
= 0; cpu
< possible
&& cpu
< nr_cpu_ids
; cpu
++)
901 set_cpu_possible(cpu
, true);
904 void __init
smp_prepare_cpus(unsigned int max_cpus
)
906 /* request the 0x1201 emergency signal external interrupt */
907 if (register_external_irq(EXT_IRQ_EMERGENCY_SIG
, do_ext_call_interrupt
))
908 panic("Couldn't request external interrupt 0x1201");
909 /* request the 0x1202 external call external interrupt */
910 if (register_external_irq(EXT_IRQ_EXTERNAL_CALL
, do_ext_call_interrupt
))
911 panic("Couldn't request external interrupt 0x1202");
915 void __init
smp_prepare_boot_cpu(void)
917 struct pcpu
*pcpu
= pcpu_devices
;
919 pcpu
->state
= CPU_STATE_CONFIGURED
;
920 pcpu
->address
= stap();
921 pcpu
->lowcore
= (struct _lowcore
*)(unsigned long) store_prefix();
922 S390_lowcore
.percpu_offset
= __per_cpu_offset
[0];
923 smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN
);
924 set_cpu_present(0, true);
925 set_cpu_online(0, true);
928 void __init
smp_cpus_done(unsigned int max_cpus
)
932 void __init
smp_setup_processor_id(void)
934 S390_lowcore
.cpu_nr
= 0;
935 S390_lowcore
.spinlock_lockval
= arch_spin_lockval(0);
939 * the frequency of the profiling timer can be changed
940 * by writing a multiplier value into /proc/profile.
942 * usually you want to run this on all CPUs ;)
944 int setup_profiling_timer(unsigned int multiplier
)
949 #ifdef CONFIG_HOTPLUG_CPU
950 static ssize_t
cpu_configure_show(struct device
*dev
,
951 struct device_attribute
*attr
, char *buf
)
955 mutex_lock(&smp_cpu_state_mutex
);
956 count
= sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].state
);
957 mutex_unlock(&smp_cpu_state_mutex
);
961 static ssize_t
cpu_configure_store(struct device
*dev
,
962 struct device_attribute
*attr
,
963 const char *buf
, size_t count
)
969 if (sscanf(buf
, "%d %c", &val
, &delim
) != 1)
971 if (val
!= 0 && val
!= 1)
974 mutex_lock(&smp_cpu_state_mutex
);
976 /* disallow configuration changes of online cpus and cpu 0 */
978 cpu
-= cpu
% (smp_cpu_mtid
+ 1);
981 for (i
= 0; i
<= smp_cpu_mtid
; i
++)
982 if (cpu_online(cpu
+ i
))
984 pcpu
= pcpu_devices
+ cpu
;
988 if (pcpu
->state
!= CPU_STATE_CONFIGURED
)
990 rc
= sclp_core_deconfigure(pcpu
->address
>> smp_cpu_mt_shift
);
993 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
994 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
996 pcpu
[i
].state
= CPU_STATE_STANDBY
;
997 smp_cpu_set_polarization(cpu
+ i
,
998 POLARIZATION_UNKNOWN
);
1000 topology_expect_change();
1003 if (pcpu
->state
!= CPU_STATE_STANDBY
)
1005 rc
= sclp_core_configure(pcpu
->address
>> smp_cpu_mt_shift
);
1008 for (i
= 0; i
<= smp_cpu_mtid
; i
++) {
1009 if (cpu
+ i
>= nr_cpu_ids
|| !cpu_present(cpu
+ i
))
1011 pcpu
[i
].state
= CPU_STATE_CONFIGURED
;
1012 smp_cpu_set_polarization(cpu
+ i
,
1013 POLARIZATION_UNKNOWN
);
1015 topology_expect_change();
1021 mutex_unlock(&smp_cpu_state_mutex
);
1023 return rc
? rc
: count
;
1025 static DEVICE_ATTR(configure
, 0644, cpu_configure_show
, cpu_configure_store
);
1026 #endif /* CONFIG_HOTPLUG_CPU */
1028 static ssize_t
show_cpu_address(struct device
*dev
,
1029 struct device_attribute
*attr
, char *buf
)
1031 return sprintf(buf
, "%d\n", pcpu_devices
[dev
->id
].address
);
1033 static DEVICE_ATTR(address
, 0444, show_cpu_address
, NULL
);
1035 static struct attribute
*cpu_common_attrs
[] = {
1036 #ifdef CONFIG_HOTPLUG_CPU
1037 &dev_attr_configure
.attr
,
1039 &dev_attr_address
.attr
,
1043 static struct attribute_group cpu_common_attr_group
= {
1044 .attrs
= cpu_common_attrs
,
1047 static struct attribute
*cpu_online_attrs
[] = {
1048 &dev_attr_idle_count
.attr
,
1049 &dev_attr_idle_time_us
.attr
,
1053 static struct attribute_group cpu_online_attr_group
= {
1054 .attrs
= cpu_online_attrs
,
1057 static int smp_cpu_notify(struct notifier_block
*self
, unsigned long action
,
1060 unsigned int cpu
= (unsigned int)(long)hcpu
;
1061 struct device
*s
= &per_cpu(cpu_device
, cpu
)->dev
;
1064 switch (action
& ~CPU_TASKS_FROZEN
) {
1066 err
= sysfs_create_group(&s
->kobj
, &cpu_online_attr_group
);
1069 sysfs_remove_group(&s
->kobj
, &cpu_online_attr_group
);
1072 return notifier_from_errno(err
);
1075 static int smp_add_present_cpu(int cpu
)
1081 c
= kzalloc(sizeof(*c
), GFP_KERNEL
);
1084 per_cpu(cpu_device
, cpu
) = c
;
1086 c
->hotpluggable
= 1;
1087 rc
= register_cpu(c
, cpu
);
1090 rc
= sysfs_create_group(&s
->kobj
, &cpu_common_attr_group
);
1093 if (cpu_online(cpu
)) {
1094 rc
= sysfs_create_group(&s
->kobj
, &cpu_online_attr_group
);
1098 rc
= topology_cpu_init(c
);
1104 if (cpu_online(cpu
))
1105 sysfs_remove_group(&s
->kobj
, &cpu_online_attr_group
);
1107 sysfs_remove_group(&s
->kobj
, &cpu_common_attr_group
);
1109 #ifdef CONFIG_HOTPLUG_CPU
1116 #ifdef CONFIG_HOTPLUG_CPU
1118 int __ref
smp_rescan_cpus(void)
1120 struct sclp_core_info
*info
;
1123 info
= smp_get_core_info();
1127 mutex_lock(&smp_cpu_state_mutex
);
1128 nr
= __smp_rescan_cpus(info
, 1);
1129 mutex_unlock(&smp_cpu_state_mutex
);
1133 topology_schedule_update();
1137 static ssize_t __ref
rescan_store(struct device
*dev
,
1138 struct device_attribute
*attr
,
1144 rc
= smp_rescan_cpus();
1145 return rc
? rc
: count
;
1147 static DEVICE_ATTR(rescan
, 0200, NULL
, rescan_store
);
1148 #endif /* CONFIG_HOTPLUG_CPU */
1150 static int __init
s390_smp_init(void)
1154 #ifdef CONFIG_HOTPLUG_CPU
1155 rc
= device_create_file(cpu_subsys
.dev_root
, &dev_attr_rescan
);
1159 cpu_notifier_register_begin();
1160 for_each_present_cpu(cpu
) {
1161 rc
= smp_add_present_cpu(cpu
);
1166 __hotcpu_notifier(smp_cpu_notify
, 0);
1169 cpu_notifier_register_done();
1172 subsys_initcall(s390_smp_init
);