| blob | 412a7b8783d7b99c47f5b3715a328d4fdf0ea1ed |
1 /*
2 * arch/s390/kernel/entry64.S
3 * S390 low-level entry points.
4 *
5 * Copyright (C) IBM Corp. 1999,2010
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
12 #include <linux/init.h>
13 #include <linux/linkage.h>
14 #include <asm/cache.h>
15 #include <asm/errno.h>
16 #include <asm/ptrace.h>
17 #include <asm/thread_info.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/unistd.h>
20 #include <asm/page.h>
22 __PT_R0 = __PT_GPRS
23 __PT_R1 = __PT_GPRS + 8
24 __PT_R2 = __PT_GPRS + 16
25 __PT_R3 = __PT_GPRS + 24
26 __PT_R4 = __PT_GPRS + 32
27 __PT_R5 = __PT_GPRS + 40
28 __PT_R6 = __PT_GPRS + 48
29 __PT_R7 = __PT_GPRS + 56
30 __PT_R8 = __PT_GPRS + 64
31 __PT_R9 = __PT_GPRS + 72
32 __PT_R10 = __PT_GPRS + 80
33 __PT_R11 = __PT_GPRS + 88
34 __PT_R12 = __PT_GPRS + 96
35 __PT_R13 = __PT_GPRS + 104
36 __PT_R14 = __PT_GPRS + 112
37 __PT_R15 = __PT_GPRS + 120
39 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
40 STACK_SIZE = 1 << STACK_SHIFT
42 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
43 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
44 _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
45 _TIF_MCCK_PENDING)
46 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
47 _TIF_SYSCALL_TRACEPOINT)
48 _TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING)
50 #define BASED(name) name-system_call(%r13)
52 .macro TRACE_IRQS_ON
53 #ifdef CONFIG_TRACE_IRQFLAGS
54 basr %r2,%r0
55 brasl %r14,trace_hardirqs_on_caller
56 #endif
57 .endm
59 .macro TRACE_IRQS_OFF
60 #ifdef CONFIG_TRACE_IRQFLAGS
61 basr %r2,%r0
62 brasl %r14,trace_hardirqs_off_caller
63 #endif
64 .endm
66 .macro LOCKDEP_SYS_EXIT
67 #ifdef CONFIG_LOCKDEP
68 tm __PT_PSW+1(%r11),0x01 # returning to user ?
69 jz .+10
70 brasl %r14,lockdep_sys_exit
71 #endif
72 .endm
74 .macro SPP newpp
75 #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
76 tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_SPP
77 jz .+8
78 .insn s,0xb2800000,\newpp
79 #endif
80 .endm
82 .macro HANDLE_SIE_INTERCEPT scratch
83 #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
84 tm __TI_flags+6(%r12),_TIF_SIE>>8
85 jz .+42
86 tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_SPP
87 jz .+8
88 .insn s,0xb2800000,BASED(.Lhost_id) # set host id
89 lgr \scratch,%r9
90 slg \scratch,BASED(.Lsie_loop)
91 clg \scratch,BASED(.Lsie_length)
92 jhe .+10
93 lg %r9,BASED(.Lsie_loop)
94 #endif
95 .endm
97 .macro CHECK_STACK stacksize,savearea
98 #ifdef CONFIG_CHECK_STACK
99 tml %r15,\stacksize - CONFIG_STACK_GUARD
100 lghi %r14,\savearea
101 jz stack_overflow
102 #endif
103 .endm
105 .macro SWITCH_ASYNC savearea,stack,shift
106 tmhh %r8,0x0001 # interrupting from user ?
107 jnz 1f
108 lgr %r14,%r9
109 slg %r14,BASED(.Lcritical_start)
110 clg %r14,BASED(.Lcritical_length)
111 jhe 0f
112 lghi %r11,\savearea # inside critical section, do cleanup
113 brasl %r14,cleanup_critical
114 tmhh %r8,0x0001 # retest problem state after cleanup
115 jnz 1f
116 0: lg %r14,\stack # are we already on the target stack?
117 slgr %r14,%r15
118 srag %r14,%r14,\shift
119 jnz 1f
120 CHECK_STACK 1<<\shift,\savearea
121 j 2f
122 1: lg %r15,\stack # load target stack
123 2: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
124 la %r11,STACK_FRAME_OVERHEAD(%r15)
125 .endm
127 .macro UPDATE_VTIME scratch,enter_timer
128 lg \scratch,__LC_EXIT_TIMER
129 slg \scratch,\enter_timer
130 alg \scratch,__LC_USER_TIMER
131 stg \scratch,__LC_USER_TIMER
132 lg \scratch,__LC_LAST_UPDATE_TIMER
133 slg \scratch,__LC_EXIT_TIMER
134 alg \scratch,__LC_SYSTEM_TIMER
135 stg \scratch,__LC_SYSTEM_TIMER
136 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
137 .endm
139 .macro LAST_BREAK scratch
140 srag \scratch,%r10,23
141 jz .+10
142 stg %r10,__TI_last_break(%r12)
143 .endm
145 .macro REENABLE_IRQS
146 stg %r8,__LC_RETURN_PSW
147 ni __LC_RETURN_PSW,0xbf
148 ssm __LC_RETURN_PSW
149 .endm
151 .section .kprobes.text, "ax"
153 /*
154 * Scheduler resume function, called by switch_to
155 * gpr2 = (task_struct *) prev
156 * gpr3 = (task_struct *) next
157 * Returns:
158 * gpr2 = prev
159 */
160 ENTRY(__switch_to)
161 lg %r4,__THREAD_info(%r2) # get thread_info of prev
162 lg %r5,__THREAD_info(%r3) # get thread_info of next
163 tm __TI_flags+7(%r4),_TIF_MCCK_PENDING # machine check pending?
164 jz 0f
165 ni __TI_flags+7(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
166 oi __TI_flags+7(%r5),_TIF_MCCK_PENDING # set it in next
167 0: stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
168 stg %r15,__THREAD_ksp(%r2) # store kernel stack of prev
169 lg %r15,__THREAD_ksp(%r3) # load kernel stack of next
170 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
171 lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
172 stg %r3,__LC_CURRENT # store task struct of next
173 mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next
174 stg %r5,__LC_THREAD_INFO # store thread info of next
175 aghi %r5,STACK_SIZE # end of kernel stack of next
176 stg %r5,__LC_KERNEL_STACK # store end of kernel stack
177 br %r14
179 __critical_start:
180 /*
181 * SVC interrupt handler routine. System calls are synchronous events and
182 * are executed with interrupts enabled.
183 */
185 ENTRY(system_call)
186 stpt __LC_SYNC_ENTER_TIMER
187 sysc_stmg:
188 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
189 lg %r10,__LC_LAST_BREAK
190 lg %r12,__LC_THREAD_INFO
191 larl %r13,system_call
192 sysc_per:
193 lg %r15,__LC_KERNEL_STACK
194 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
195 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
196 sysc_vtime:
197 UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER
198 LAST_BREAK %r13
199 stmg %r0,%r7,__PT_R0(%r11)
200 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
201 mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW
202 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
203 sysc_do_svc:
204 oi __TI_flags+7(%r12),_TIF_SYSCALL
205 llgh %r8,__PT_INT_CODE+2(%r11)
206 slag %r8,%r8,2 # shift and test for svc 0
207 jnz sysc_nr_ok
208 # svc 0: system call number in %r1
209 llgfr %r1,%r1 # clear high word in r1
210 cghi %r1,NR_syscalls
211 jnl sysc_nr_ok
212 sth %r1,__PT_INT_CODE+2(%r11)
213 slag %r8,%r1,2
214 sysc_nr_ok:
215 larl %r10,sys_call_table # 64 bit system call table
216 #ifdef CONFIG_COMPAT
217 tm __TI_flags+5(%r12),(_TIF_31BIT>>16)
218 jno sysc_noemu
219 larl %r10,sys_call_table_emu # 31 bit system call table
220 sysc_noemu:
221 #endif
222 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
223 stg %r2,__PT_ORIG_GPR2(%r11)
224 stg %r7,STACK_FRAME_OVERHEAD(%r15)
225 lgf %r9,0(%r8,%r10) # get system call add.
226 tm __TI_flags+6(%r12),_TIF_TRACE >> 8
227 jnz sysc_tracesys
228 basr %r14,%r9 # call sys_xxxx
229 stg %r2,__PT_R2(%r11) # store return value
231 sysc_return:
232 LOCKDEP_SYS_EXIT
233 sysc_tif:
234 tm __PT_PSW+1(%r11),0x01 # returning to user ?
235 jno sysc_restore
236 tm __TI_flags+7(%r12),_TIF_WORK_SVC
237 jnz sysc_work # check for work
238 ni __TI_flags+7(%r12),255-_TIF_SYSCALL
239 sysc_restore:
240 lg %r14,__LC_VDSO_PER_CPU
241 lmg %r0,%r10,__PT_R0(%r11)
242 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
243 stpt __LC_EXIT_TIMER
244 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
245 lmg %r11,%r15,__PT_R11(%r11)
246 lpswe __LC_RETURN_PSW
247 sysc_done:
249 #
250 # One of the work bits is on. Find out which one.
251 #
252 sysc_work:
253 tm __TI_flags+7(%r12),_TIF_MCCK_PENDING
254 jo sysc_mcck_pending
255 tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
256 jo sysc_reschedule
257 tm __TI_flags+7(%r12),_TIF_SIGPENDING
258 jo sysc_sigpending
259 tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
260 jo sysc_notify_resume
261 tm __TI_flags+7(%r12),_TIF_PER_TRAP
262 jo sysc_singlestep
263 j sysc_return # beware of critical section cleanup
265 #
266 # _TIF_NEED_RESCHED is set, call schedule
267 #
268 sysc_reschedule:
269 larl %r14,sysc_return
270 jg schedule
272 #
273 # _TIF_MCCK_PENDING is set, call handler
274 #
275 sysc_mcck_pending:
276 larl %r14,sysc_return
277 jg s390_handle_mcck # TIF bit will be cleared by handler
279 #
280 # _TIF_SIGPENDING is set, call do_signal
281 #
282 sysc_sigpending:
283 ni __TI_flags+7(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
284 lgr %r2,%r11 # pass pointer to pt_regs
285 brasl %r14,do_signal
286 tm __TI_flags+7(%r12),_TIF_SYSCALL
287 jno sysc_return
288 lmg %r2,%r7,__PT_R2(%r11) # load svc arguments
289 lghi %r8,0 # svc 0 returns -ENOSYS
290 lh %r1,__PT_INT_CODE+2(%r11) # load new svc number
291 cghi %r1,NR_syscalls
292 jnl sysc_nr_ok # invalid svc number -> do svc 0
293 slag %r8,%r1,2
294 j sysc_nr_ok # restart svc
296 #
297 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
298 #
299 sysc_notify_resume:
300 lgr %r2,%r11 # pass pointer to pt_regs
301 larl %r14,sysc_return
302 jg do_notify_resume
304 #
305 # _TIF_PER_TRAP is set, call do_per_trap
306 #
307 sysc_singlestep:
308 ni __TI_flags+7(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP)
309 lgr %r2,%r11 # pass pointer to pt_regs
310 larl %r14,sysc_return
311 jg do_per_trap
313 #
314 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
315 # and after the system call
316 #
317 sysc_tracesys:
318 lgr %r2,%r11 # pass pointer to pt_regs
319 la %r3,0
320 llgh %r0,__PT_INT_CODE+2(%r11)
321 stg %r0,__PT_R2(%r11)
322 brasl %r14,do_syscall_trace_enter
323 lghi %r0,NR_syscalls
324 clgr %r0,%r2
325 jnh sysc_tracenogo
326 sllg %r8,%r2,2
327 lgf %r9,0(%r8,%r10)
328 sysc_tracego:
329 lmg %r3,%r7,__PT_R3(%r11)
330 stg %r7,STACK_FRAME_OVERHEAD(%r15)
331 lg %r2,__PT_ORIG_GPR2(%r11)
332 basr %r14,%r9 # call sys_xxx
333 stg %r2,__PT_R2(%r11) # store return value
334 sysc_tracenogo:
335 tm __TI_flags+6(%r12),_TIF_TRACE >> 8
336 jz sysc_return
337 lgr %r2,%r11 # pass pointer to pt_regs
338 larl %r14,sysc_return
339 jg do_syscall_trace_exit
341 #
342 # a new process exits the kernel with ret_from_fork
343 #
344 ENTRY(ret_from_fork)
345 la %r11,STACK_FRAME_OVERHEAD(%r15)
346 lg %r12,__LC_THREAD_INFO
347 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
348 jo 0f
349 stg %r15,__PT_R15(%r11) # store stack pointer for new kthread
350 0: brasl %r14,schedule_tail
351 TRACE_IRQS_ON
352 ssm __LC_SVC_NEW_PSW # reenable interrupts
353 j sysc_tracenogo
355 #
356 # kernel_execve function needs to deal with pt_regs that is not
357 # at the usual place
358 #
359 ENTRY(kernel_execve)
360 stmg %r12,%r15,96(%r15)
361 lgr %r14,%r15
362 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
363 stg %r14,__SF_BACKCHAIN(%r15)
364 la %r12,STACK_FRAME_OVERHEAD(%r15)
365 xc 0(__PT_SIZE,%r12),0(%r12)
366 lgr %r5,%r12
367 brasl %r14,do_execve
368 ltgfr %r2,%r2
369 je 0f
370 aghi %r15,(STACK_FRAME_OVERHEAD + __PT_SIZE)
371 lmg %r12,%r15,96(%r15)
372 br %r14
373 # execve succeeded.
374 0: ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
375 lg %r15,__LC_KERNEL_STACK # load ksp
376 aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
377 la %r11,STACK_FRAME_OVERHEAD(%r15)
378 mvc 0(__PT_SIZE,%r11),0(%r12) # copy pt_regs
379 lg %r12,__LC_THREAD_INFO
380 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
381 ssm __LC_SVC_NEW_PSW # reenable interrupts
382 brasl %r14,execve_tail
383 j sysc_return
385 /*
386 * Program check handler routine
387 */
389 ENTRY(pgm_check_handler)
390 stpt __LC_SYNC_ENTER_TIMER
391 stmg %r8,%r15,__LC_SAVE_AREA_SYNC
392 lg %r10,__LC_LAST_BREAK
393 lg %r12,__LC_THREAD_INFO
394 larl %r13,system_call
395 lmg %r8,%r9,__LC_PGM_OLD_PSW
396 HANDLE_SIE_INTERCEPT %r14
397 tmhh %r8,0x0001 # test problem state bit
398 jnz 1f # -> fault in user space
399 tmhh %r8,0x4000 # PER bit set in old PSW ?
400 jnz 0f # -> enabled, can't be a double fault
401 tm __LC_PGM_ILC+3,0x80 # check for per exception
402 jnz pgm_svcper # -> single stepped svc
403 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
404 j 2f
405 1: UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER
406 LAST_BREAK %r14
407 lg %r15,__LC_KERNEL_STACK
408 2: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
409 la %r11,STACK_FRAME_OVERHEAD(%r15)
410 stmg %r0,%r7,__PT_R0(%r11)
411 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
412 stmg %r8,%r9,__PT_PSW(%r11)
413 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
414 mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
415 stg %r10,__PT_ARGS(%r11)
416 tm __LC_PGM_ILC+3,0x80 # check for per exception
417 jz 0f
418 lg %r1,__TI_task(%r12)
419 tmhh %r8,0x0001 # kernel per event ?
420 jz pgm_kprobe
421 oi __TI_flags+7(%r12),_TIF_PER_TRAP
422 mvc __THREAD_per_address(8,%r1),__LC_PER_ADDRESS
423 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
424 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
425 0: REENABLE_IRQS
426 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
427 larl %r1,pgm_check_table
428 llgh %r10,__PT_INT_CODE+2(%r11)
429 nill %r10,0x007f
430 sll %r10,3
431 je sysc_return
432 lg %r1,0(%r10,%r1) # load address of handler routine
433 lgr %r2,%r11 # pass pointer to pt_regs
434 basr %r14,%r1 # branch to interrupt-handler
435 j sysc_return
437 #
438 # PER event in supervisor state, must be kprobes
439 #
440 pgm_kprobe:
441 REENABLE_IRQS
442 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
443 lgr %r2,%r11 # pass pointer to pt_regs
444 brasl %r14,do_per_trap
445 j sysc_return
447 #
448 # single stepped system call
449 #
450 pgm_svcper:
451 oi __TI_flags+7(%r12),_TIF_PER_TRAP
452 mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
453 larl %r14,sysc_per
454 stg %r14,__LC_RETURN_PSW+8
455 lpswe __LC_RETURN_PSW # branch to sysc_per and enable irqs
457 /*
458 * IO interrupt handler routine
459 */
460 ENTRY(io_int_handler)
461 stck __LC_INT_CLOCK
462 stpt __LC_ASYNC_ENTER_TIMER
463 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
464 lg %r10,__LC_LAST_BREAK
465 lg %r12,__LC_THREAD_INFO
466 larl %r13,system_call
467 lmg %r8,%r9,__LC_IO_OLD_PSW
468 HANDLE_SIE_INTERCEPT %r14
469 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
470 tmhh %r8,0x0001 # interrupting from user?
471 jz io_skip
472 UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER
473 LAST_BREAK %r14
474 io_skip:
475 stmg %r0,%r7,__PT_R0(%r11)
476 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
477 stmg %r8,%r9,__PT_PSW(%r11)
478 TRACE_IRQS_OFF
479 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
480 lgr %r2,%r11 # pass pointer to pt_regs
481 brasl %r14,do_IRQ
482 io_return:
483 LOCKDEP_SYS_EXIT
484 TRACE_IRQS_ON
485 io_tif:
486 tm __TI_flags+7(%r12),_TIF_WORK_INT
487 jnz io_work # there is work to do (signals etc.)
488 io_restore:
489 lg %r14,__LC_VDSO_PER_CPU
490 lmg %r0,%r10,__PT_R0(%r11)
491 mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
492 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
493 stpt __LC_EXIT_TIMER
494 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
495 lmg %r11,%r15,__PT_R11(%r11)
496 lpswe __LC_RETURN_PSW
497 io_done:
499 #
500 # There is work todo, find out in which context we have been interrupted:
501 # 1) if we return to user space we can do all _TIF_WORK_INT work
502 # 2) if we return to kernel code and kvm is enabled check if we need to
503 # modify the psw to leave SIE
504 # 3) if we return to kernel code and preemptive scheduling is enabled check
505 # the preemption counter and if it is zero call preempt_schedule_irq
506 # Before any work can be done, a switch to the kernel stack is required.
507 #
508 io_work:
509 tm __PT_PSW+1(%r11),0x01 # returning to user ?
510 jo io_work_user # yes -> do resched & signal
511 #ifdef CONFIG_PREEMPT
512 # check for preemptive scheduling
513 icm %r0,15,__TI_precount(%r12)
514 jnz io_restore # preemption is disabled
515 tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
516 jno io_restore
517 # switch to kernel stack
518 lg %r1,__PT_R15(%r11)
519 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
520 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
521 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
522 la %r11,STACK_FRAME_OVERHEAD(%r1)
523 lgr %r15,%r1
524 # TRACE_IRQS_ON already done at io_return, call
525 # TRACE_IRQS_OFF to keep things symmetrical
526 TRACE_IRQS_OFF
527 brasl %r14,preempt_schedule_irq
528 j io_return
529 #else
530 j io_restore
531 #endif
533 #
534 # Need to do work before returning to userspace, switch to kernel stack
535 #
536 io_work_user:
537 lg %r1,__LC_KERNEL_STACK
538 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
539 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
540 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
541 la %r11,STACK_FRAME_OVERHEAD(%r1)
542 lgr %r15,%r1
544 #
545 # One of the work bits is on. Find out which one.
546 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
547 # and _TIF_MCCK_PENDING
548 #
549 io_work_tif:
550 tm __TI_flags+7(%r12),_TIF_MCCK_PENDING
551 jo io_mcck_pending
552 tm __TI_flags+7(%r12),_TIF_NEED_RESCHED
553 jo io_reschedule
554 tm __TI_flags+7(%r12),_TIF_SIGPENDING
555 jo io_sigpending
556 tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME
557 jo io_notify_resume
558 j io_return # beware of critical section cleanup
560 #
561 # _TIF_MCCK_PENDING is set, call handler
562 #
563 io_mcck_pending:
564 # TRACE_IRQS_ON already done at io_return
565 brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler
566 TRACE_IRQS_OFF
567 j io_return
569 #
570 # _TIF_NEED_RESCHED is set, call schedule
571 #
572 io_reschedule:
573 # TRACE_IRQS_ON already done at io_return
574 ssm __LC_SVC_NEW_PSW # reenable interrupts
575 brasl %r14,schedule # call scheduler
576 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
577 TRACE_IRQS_OFF
578 j io_return
580 #
581 # _TIF_SIGPENDING or is set, call do_signal
582 #
583 io_sigpending:
584 # TRACE_IRQS_ON already done at io_return
585 ssm __LC_SVC_NEW_PSW # reenable interrupts
586 lgr %r2,%r11 # pass pointer to pt_regs
587 brasl %r14,do_signal
588 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
589 TRACE_IRQS_OFF
590 j io_return
592 #
593 # _TIF_NOTIFY_RESUME or is set, call do_notify_resume
594 #
595 io_notify_resume:
596 # TRACE_IRQS_ON already done at io_return
597 ssm __LC_SVC_NEW_PSW # reenable interrupts
598 lgr %r2,%r11 # pass pointer to pt_regs
599 brasl %r14,do_notify_resume
600 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
601 TRACE_IRQS_OFF
602 j io_return
604 /*
605 * External interrupt handler routine
606 */
607 ENTRY(ext_int_handler)
608 stck __LC_INT_CLOCK
609 stpt __LC_ASYNC_ENTER_TIMER
610 stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
611 lg %r10,__LC_LAST_BREAK
612 lg %r12,__LC_THREAD_INFO
613 larl %r13,system_call
614 lmg %r8,%r9,__LC_EXT_OLD_PSW
615 HANDLE_SIE_INTERCEPT %r14
616 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
617 tmhh %r8,0x0001 # interrupting from user ?
618 jz ext_skip
619 UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER
620 LAST_BREAK %r14
621 ext_skip:
622 stmg %r0,%r7,__PT_R0(%r11)
623 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
624 stmg %r8,%r9,__PT_PSW(%r11)
625 TRACE_IRQS_OFF
626 lghi %r1,4096
627 lgr %r2,%r11 # pass pointer to pt_regs
628 llgf %r3,__LC_CPU_ADDRESS # get cpu address + interruption code
629 llgf %r4,__LC_EXT_PARAMS # get external parameter
630 lg %r5,__LC_EXT_PARAMS2-4096(%r1) # get 64 bit external parameter
631 brasl %r14,do_extint
632 j io_return
634 __critical_end:
636 /*
637 * Machine check handler routines
638 */
639 ENTRY(mcck_int_handler)
640 stck __LC_MCCK_CLOCK
641 la %r1,4095 # revalidate r1
642 spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer
643 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
644 lg %r10,__LC_LAST_BREAK
645 lg %r12,__LC_THREAD_INFO
646 larl %r13,system_call
647 lmg %r8,%r9,__LC_MCK_OLD_PSW
648 HANDLE_SIE_INTERCEPT %r14
649 tm __LC_MCCK_CODE,0x80 # system damage?
650 jo mcck_panic # yes -> rest of mcck code invalid
651 lghi %r14,__LC_CPU_TIMER_SAVE_AREA
652 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
653 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
654 jo 3f
655 la %r14,__LC_SYNC_ENTER_TIMER
656 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
657 jl 0f
658 la %r14,__LC_ASYNC_ENTER_TIMER
659 0: clc 0(8,%r14),__LC_EXIT_TIMER
660 jl 1f
661 la %r14,__LC_EXIT_TIMER
662 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
663 jl 2f
664 la %r14,__LC_LAST_UPDATE_TIMER
665 2: spt 0(%r14)
666 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
667 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
668 jno mcck_panic # no -> skip cleanup critical
669 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_PANIC_STACK,PAGE_SHIFT
670 tm %r8,0x0001 # interrupting from user ?
671 jz mcck_skip
672 UPDATE_VTIME %r14,__LC_MCCK_ENTER_TIMER
673 LAST_BREAK %r14
674 mcck_skip:
675 lghi %r14,__LC_GPREGS_SAVE_AREA
676 mvc __PT_R0(128,%r11),0(%r14)
677 stmg %r8,%r9,__PT_PSW(%r11)
678 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
679 lgr %r2,%r11 # pass pointer to pt_regs
680 brasl %r14,s390_do_machine_check
681 tm __PT_PSW+1(%r11),0x01 # returning to user ?
682 jno mcck_return
683 lg %r1,__LC_KERNEL_STACK # switch to kernel stack
684 aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
685 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
686 xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
687 la %r11,STACK_FRAME_OVERHEAD(%r1)
688 lgr %r15,%r1
689 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
690 tm __TI_flags+7(%r12),_TIF_MCCK_PENDING
691 jno mcck_return
692 TRACE_IRQS_OFF
693 brasl %r14,s390_handle_mcck
694 TRACE_IRQS_ON
695 mcck_return:
696 lg %r14,__LC_VDSO_PER_CPU
697 lmg %r0,%r10,__PT_R0(%r11)
698 mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
699 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
700 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
701 jno 0f
702 stpt __LC_EXIT_TIMER
703 mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
704 0: lmg %r11,%r15,__PT_R11(%r11)
705 lpswe __LC_RETURN_MCCK_PSW
707 mcck_panic:
708 lg %r14,__LC_PANIC_STACK
709 slgr %r14,%r15
710 srag %r14,%r14,PAGE_SHIFT
711 jz 0f
712 lg %r15,__LC_PANIC_STACK
713 0: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
714 j mcck_skip
716 /*
717 * Restart interruption handler, kick starter for additional CPUs
718 */
719 #ifdef CONFIG_SMP
720 __CPUINIT
721 ENTRY(restart_int_handler)
722 basr %r1,0
723 restart_base:
724 spt restart_vtime-restart_base(%r1)
725 stck __LC_LAST_UPDATE_CLOCK
726 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
727 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
728 lghi %r10,__LC_GPREGS_SAVE_AREA
729 lg %r15,120(%r10) # load ksp
730 lghi %r10,__LC_CREGS_SAVE_AREA
731 lctlg %c0,%c15,0(%r10) # get new ctl regs
732 lghi %r10,__LC_AREGS_SAVE_AREA
733 lam %a0,%a15,0(%r10)
734 lmg %r6,%r15,__SF_GPRS(%r15)# load registers from clone
735 lg %r1,__LC_THREAD_INFO
736 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
737 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
738 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
739 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
740 brasl %r14,start_secondary
741 .align 8
742 restart_vtime:
743 .long 0x7fffffff,0xffffffff
744 .previous
745 #else
746 /*
747 * If we do not run with SMP enabled, let the new CPU crash ...
748 */
749 ENTRY(restart_int_handler)
750 basr %r1,0
751 restart_base:
752 lpswe restart_crash-restart_base(%r1)
753 .align 8
754 restart_crash:
755 .long 0x000a0000,0x00000000,0x00000000,0x00000000
756 restart_go:
757 #endif
759 #
760 # PSW restart interrupt handler
761 #
762 ENTRY(psw_restart_int_handler)
763 stg %r15,__LC_SAVE_AREA_RESTART
764 larl %r15,restart_stack # load restart stack
765 lg %r15,0(%r15)
766 aghi %r15,-__PT_SIZE # create pt_regs on stack
767 stmg %r0,%r14,__PT_R0(%r15)
768 mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
769 mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
770 aghi %r15,-STACK_FRAME_OVERHEAD
771 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
772 brasl %r14,do_restart
773 larl %r14,restart_psw_crash # load disabled wait PSW if
774 lpswe 0(%r14) # do_restart returns
775 .align 8
776 restart_psw_crash:
777 .quad 0x0002000080000000,0x0000000000000000 + restart_psw_crash
779 .section .kprobes.text, "ax"
781 #ifdef CONFIG_CHECK_STACK
782 /*
783 * The synchronous or the asynchronous stack overflowed. We are dead.
784 * No need to properly save the registers, we are going to panic anyway.
785 * Setup a pt_regs so that show_trace can provide a good call trace.
786 */
787 stack_overflow:
788 lg %r11,__LC_PANIC_STACK # change to panic stack
789 aghi %r11,-__PT_SIZE # create pt_regs
790 stmg %r0,%r7,__PT_R0(%r11)
791 stmg %r8,%r9,__PT_PSW(%r11)
792 mvc __PT_R8(64,%r11),0(%r14)
793 stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
794 lgr %r15,%r11
795 aghi %r15,-STACK_FRAME_OVERHEAD
796 xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
797 lgr %r2,%r11 # pass pointer to pt_regs
798 jg kernel_stack_overflow
799 #endif
801 .align 8
802 cleanup_table:
803 .quad system_call
804 .quad sysc_do_svc
805 .quad sysc_tif
806 .quad sysc_restore
807 .quad sysc_done
808 .quad io_tif
809 .quad io_restore
810 .quad io_done
812 cleanup_critical:
813 clg %r9,BASED(cleanup_table) # system_call
814 jl 0f
815 clg %r9,BASED(cleanup_table+8) # sysc_do_svc
816 jl cleanup_system_call
817 clg %r9,BASED(cleanup_table+16) # sysc_tif
818 jl 0f
819 clg %r9,BASED(cleanup_table+24) # sysc_restore
820 jl cleanup_sysc_tif
821 clg %r9,BASED(cleanup_table+32) # sysc_done
822 jl cleanup_sysc_restore
823 clg %r9,BASED(cleanup_table+40) # io_tif
824 jl 0f
825 clg %r9,BASED(cleanup_table+48) # io_restore
826 jl cleanup_io_tif
827 clg %r9,BASED(cleanup_table+56) # io_done
828 jl cleanup_io_restore
829 0: br %r14
832 cleanup_system_call:
833 # check if stpt has been executed
834 clg %r9,BASED(cleanup_system_call_insn)
835 jh 0f
836 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
837 cghi %r11,__LC_SAVE_AREA_ASYNC
838 je 0f
839 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
840 0: # check if stmg has been executed
841 clg %r9,BASED(cleanup_system_call_insn+8)
842 jh 0f
843 mvc __LC_SAVE_AREA_SYNC(64),0(%r11)
844 0: # check if base register setup + TIF bit load has been done
845 clg %r9,BASED(cleanup_system_call_insn+16)
846 jhe 0f
847 # set up saved registers r10 and r12
848 stg %r10,16(%r11) # r10 last break
849 stg %r12,32(%r11) # r12 thread-info pointer
850 0: # check if the user time update has been done
851 clg %r9,BASED(cleanup_system_call_insn+24)
852 jh 0f
853 lg %r15,__LC_EXIT_TIMER
854 slg %r15,__LC_SYNC_ENTER_TIMER
855 alg %r15,__LC_USER_TIMER
856 stg %r15,__LC_USER_TIMER
857 0: # check if the system time update has been done
858 clg %r9,BASED(cleanup_system_call_insn+32)
859 jh 0f
860 lg %r15,__LC_LAST_UPDATE_TIMER
861 slg %r15,__LC_EXIT_TIMER
862 alg %r15,__LC_SYSTEM_TIMER
863 stg %r15,__LC_SYSTEM_TIMER
864 0: # update accounting time stamp
865 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
866 # do LAST_BREAK
867 lg %r9,16(%r11)
868 srag %r9,%r9,23
869 jz 0f
870 mvc __TI_last_break(8,%r12),16(%r11)
871 0: # set up saved register r11
872 lg %r15,__LC_KERNEL_STACK
873 aghi %r15,-__PT_SIZE
874 stg %r15,24(%r11) # r11 pt_regs pointer
875 # fill pt_regs
876 mvc __PT_R8(64,%r15),__LC_SAVE_AREA_SYNC
877 stmg %r0,%r7,__PT_R0(%r15)
878 mvc __PT_PSW(16,%r15),__LC_SVC_OLD_PSW
879 mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC
880 # setup saved register r15
881 aghi %r15,-STACK_FRAME_OVERHEAD
882 stg %r15,56(%r11) # r15 stack pointer
883 # set new psw address and exit
884 larl %r9,sysc_do_svc
885 br %r14
886 cleanup_system_call_insn:
887 .quad system_call
888 .quad sysc_stmg
889 .quad sysc_per
890 .quad sysc_vtime+18
891 .quad sysc_vtime+42
893 cleanup_sysc_tif:
894 larl %r9,sysc_tif
895 br %r14
897 cleanup_sysc_restore:
898 clg %r9,BASED(cleanup_sysc_restore_insn)
899 je 0f
900 lg %r9,24(%r11) # get saved pointer to pt_regs
901 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
902 mvc 0(64,%r11),__PT_R8(%r9)
903 lmg %r0,%r7,__PT_R0(%r9)
904 0: lmg %r8,%r9,__LC_RETURN_PSW
905 br %r14
906 cleanup_sysc_restore_insn:
907 .quad sysc_done - 4
909 cleanup_io_tif:
910 larl %r9,io_tif
911 br %r14
913 cleanup_io_restore:
914 clg %r9,BASED(cleanup_io_restore_insn)
915 je 0f
916 lg %r9,24(%r11) # get saved r11 pointer to pt_regs
917 mvc __LC_RETURN_PSW(16),__PT_PSW(%r9)
918 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
919 mvc 0(64,%r11),__PT_R8(%r9)
920 lmg %r0,%r7,__PT_R0(%r9)
921 0: lmg %r8,%r9,__LC_RETURN_PSW
922 br %r14
923 cleanup_io_restore_insn:
924 .quad io_done - 4
926 /*
927 * Integer constants
928 */
929 .align 8
930 .Lcritical_start:
931 .quad __critical_start
932 .Lcritical_length:
933 .quad __critical_end - __critical_start
936 #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
937 /*
938 * sie64a calling convention:
939 * %r2 pointer to sie control block
940 * %r3 guest register save area
941 */
942 ENTRY(sie64a)
943 stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
944 stg %r2,__SF_EMPTY(%r15) # save control block pointer
945 stg %r3,__SF_EMPTY+8(%r15) # save guest register save area
946 xc __SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # host id == 0
947 lmg %r0,%r13,0(%r3) # load guest gprs 0-13
948 lg %r14,__LC_THREAD_INFO # pointer thread_info struct
949 oi __TI_flags+6(%r14),_TIF_SIE>>8
950 sie_loop:
951 lg %r14,__LC_THREAD_INFO # pointer thread_info struct
952 tm __TI_flags+7(%r14),_TIF_EXIT_SIE
953 jnz sie_exit
954 lg %r14,__LC_GMAP # get gmap pointer
955 ltgr %r14,%r14
956 jz sie_gmap
957 lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
958 sie_gmap:
959 lg %r14,__SF_EMPTY(%r15) # get control block pointer
960 SPP __SF_EMPTY(%r15) # set guest id
961 sie 0(%r14)
962 sie_done:
963 SPP __SF_EMPTY+16(%r15) # set host id
964 lg %r14,__LC_THREAD_INFO # pointer thread_info struct
965 sie_exit:
966 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
967 ni __TI_flags+6(%r14),255-(_TIF_SIE>>8)
968 lg %r14,__SF_EMPTY+8(%r15) # load guest register save area
969 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
970 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
971 lghi %r2,0
972 br %r14
973 sie_fault:
974 lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
975 lg %r14,__LC_THREAD_INFO # pointer thread_info struct
976 ni __TI_flags+6(%r14),255-(_TIF_SIE>>8)
977 lg %r14,__SF_EMPTY+8(%r15) # load guest register save area
978 stmg %r0,%r13,0(%r14) # save guest gprs 0-13
979 lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
980 lghi %r2,-EFAULT
981 br %r14
983 .align 8
984 .Lsie_loop:
985 .quad sie_loop
986 .Lsie_length:
987 .quad sie_done - sie_loop
988 .Lhost_id:
989 .quad 0
991 .section __ex_table,"a"
992 .quad sie_loop,sie_fault
993 .previous
994 #endif
996 .section .rodata, "a"
997 #define SYSCALL(esa,esame,emu) .long esame
998 .globl sys_call_table
999 sys_call_table:
1000 #include "syscalls.S"
1001 #undef SYSCALL
1003 #ifdef CONFIG_COMPAT
1005 #define SYSCALL(esa,esame,emu) .long emu
1006 sys_call_table_emu:
1007 #include "syscalls.S"
1008 #undef SYSCALL
1009 #endif