| blob | 3705700ed37451ace0685532f6da7d8f12eb8100 |
1 /*
2 * arch/s390/kernel/entry.S
3 * S390 low-level entry points.
4 *
5 * Copyright (C) IBM Corp. 1999,2006
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Hartmut Penner (hp@de.ibm.com),
8 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 */
12 #include <linux/init.h>
13 #include <linux/linkage.h>
14 #include <asm/cache.h>
15 #include <asm/errno.h>
16 #include <asm/ptrace.h>
17 #include <asm/thread_info.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/unistd.h>
20 #include <asm/page.h>
22 __PT_R0 = __PT_GPRS
23 __PT_R1 = __PT_GPRS + 4
24 __PT_R2 = __PT_GPRS + 8
25 __PT_R3 = __PT_GPRS + 12
26 __PT_R4 = __PT_GPRS + 16
27 __PT_R5 = __PT_GPRS + 20
28 __PT_R6 = __PT_GPRS + 24
29 __PT_R7 = __PT_GPRS + 28
30 __PT_R8 = __PT_GPRS + 32
31 __PT_R9 = __PT_GPRS + 36
32 __PT_R10 = __PT_GPRS + 40
33 __PT_R11 = __PT_GPRS + 44
34 __PT_R12 = __PT_GPRS + 48
35 __PT_R13 = __PT_GPRS + 524
36 __PT_R14 = __PT_GPRS + 56
37 __PT_R15 = __PT_GPRS + 60
39 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
40 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
41 _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
42 _TIF_MCCK_PENDING)
43 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
44 _TIF_SYSCALL_TRACEPOINT)
46 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
47 STACK_SIZE = 1 << STACK_SHIFT
49 #define BASED(name) name-system_call(%r13)
51 .macro TRACE_IRQS_ON
52 #ifdef CONFIG_TRACE_IRQFLAGS
53 basr %r2,%r0
54 l %r1,BASED(.Lhardirqs_on)
55 basr %r14,%r1 # call trace_hardirqs_on_caller
56 #endif
57 .endm
59 .macro TRACE_IRQS_OFF
60 #ifdef CONFIG_TRACE_IRQFLAGS
61 basr %r2,%r0
62 l %r1,BASED(.Lhardirqs_off)
63 basr %r14,%r1 # call trace_hardirqs_off_caller
64 #endif
65 .endm
67 .macro LOCKDEP_SYS_EXIT
68 #ifdef CONFIG_LOCKDEP
69 tm __PT_PSW+1(%r11),0x01 # returning to user ?
70 jz .+10
71 l %r1,BASED(.Llockdep_sys_exit)
72 basr %r14,%r1 # call lockdep_sys_exit
73 #endif
74 .endm
76 .macro CHECK_STACK stacksize,savearea
77 #ifdef CONFIG_CHECK_STACK
78 tml %r15,\stacksize - CONFIG_STACK_GUARD
79 la %r14,\savearea
80 jz stack_overflow
81 #endif
82 .endm
84 .macro SWITCH_ASYNC savearea,stack,shift
85 tmh %r8,0x0001 # interrupting from user ?
86 jnz 1f
87 lr %r14,%r9
88 sl %r14,BASED(.Lcritical_start)
89 cl %r14,BASED(.Lcritical_length)
90 jhe 0f
91 la %r11,\savearea # inside critical section, do cleanup
92 bras %r14,cleanup_critical
93 tmh %r8,0x0001 # retest problem state after cleanup
94 jnz 1f
95 0: l %r14,\stack # are we already on the target stack?
96 slr %r14,%r15
97 sra %r14,\shift
98 jnz 1f
99 CHECK_STACK 1<<\shift,\savearea
100 j 2f
101 1: l %r15,\stack # load target stack
102 2: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
103 la %r11,STACK_FRAME_OVERHEAD(%r15)
104 .endm
106 .macro ADD64 high,low,timer
107 al \high,\timer
108 al \low,\timer+4
109 brc 12,.+8
110 ahi \high,1
111 .endm
113 .macro SUB64 high,low,timer
114 sl \high,\timer
115 sl \low,\timer+4
116 brc 3,.+8
117 ahi \high,-1
118 .endm
120 .macro UPDATE_VTIME high,low,enter_timer
121 lm \high,\low,__LC_EXIT_TIMER
122 SUB64 \high,\low,\enter_timer
123 ADD64 \high,\low,__LC_USER_TIMER
124 stm \high,\low,__LC_USER_TIMER
125 lm \high,\low,__LC_LAST_UPDATE_TIMER
126 SUB64 \high,\low,__LC_EXIT_TIMER
127 ADD64 \high,\low,__LC_SYSTEM_TIMER
128 stm \high,\low,__LC_SYSTEM_TIMER
129 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
130 .endm
132 .macro REENABLE_IRQS
133 st %r8,__LC_RETURN_PSW
134 ni __LC_RETURN_PSW,0xbf
135 ssm __LC_RETURN_PSW
136 .endm
138 .section .kprobes.text, "ax"
140 /*
141 * Scheduler resume function, called by switch_to
142 * gpr2 = (task_struct *) prev
143 * gpr3 = (task_struct *) next
144 * Returns:
145 * gpr2 = prev
146 */
147 ENTRY(__switch_to)
148 l %r4,__THREAD_info(%r2) # get thread_info of prev
149 l %r5,__THREAD_info(%r3) # get thread_info of next
150 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
151 jz 0f
152 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
153 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
154 0: stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
155 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
156 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
157 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
158 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
159 st %r3,__LC_CURRENT # store task struct of next
160 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
161 st %r5,__LC_THREAD_INFO # store thread info of next
162 ahi %r5,STACK_SIZE # end of kernel stack of next
163 st %r5,__LC_KERNEL_STACK # store end of kernel stack
164 br %r14
166 __critical_start:
167 /*
168 * SVC interrupt handler routine. System calls are synchronous events and
169 * are executed with interrupts enabled.
170 */
172 ENTRY(system_call)
173 stpt __LC_SYNC_ENTER_TIMER
174 sysc_stm:
175 stm %r8,%r15,__LC_SAVE_AREA_SYNC
176 l %r12,__LC_THREAD_INFO
177 l %r13,__LC_SVC_NEW_PSW+4
178 sysc_per:
179 l %r15,__LC_KERNEL_STACK
180 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
181 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
182 sysc_vtime:
183 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
184 stm %r0,%r7,__PT_R0(%r11)
185 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
186 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
187 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
188 sysc_do_svc:
189 oi __TI_flags+3(%r12),_TIF_SYSCALL
190 lh %r8,__PT_INT_CODE+2(%r11)
191 sla %r8,2 # shift and test for svc0
192 jnz sysc_nr_ok
193 # svc 0: system call number in %r1
194 cl %r1,BASED(.Lnr_syscalls)
195 jnl sysc_nr_ok
196 sth %r1,__PT_INT_CODE+2(%r11)
197 lr %r8,%r1
198 sla %r8,2
199 sysc_nr_ok:
200 l %r10,BASED(.Lsys_call_table) # 31 bit system call table
201 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
202 st %r2,__PT_ORIG_GPR2(%r11)
203 st %r7,STACK_FRAME_OVERHEAD(%r15)
204 l %r9,0(%r8,%r10) # get system call addr.
205 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
206 jnz sysc_tracesys
207 basr %r14,%r9 # call sys_xxxx
208 st %r2,__PT_R2(%r11) # store return value
210 sysc_return:
211 LOCKDEP_SYS_EXIT
212 sysc_tif:
213 tm __PT_PSW+1(%r11),0x01 # returning to user ?
214 jno sysc_restore
215 tm __TI_flags+3(%r12),_TIF_WORK_SVC
216 jnz sysc_work # check for work
217 ni __TI_flags+3(%r12),255-_TIF_SYSCALL
218 sysc_restore:
219 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
220 stpt __LC_EXIT_TIMER
221 lm %r0,%r15,__PT_R0(%r11)
222 lpsw __LC_RETURN_PSW
223 sysc_done:
225 #
226 # One of the work bits is on. Find out which one.
227 #
228 sysc_work:
229 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
230 jo sysc_mcck_pending
231 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
232 jo sysc_reschedule
233 tm __TI_flags+3(%r12),_TIF_SIGPENDING
234 jo sysc_sigpending
235 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
236 jo sysc_notify_resume
237 tm __TI_flags+3(%r12),_TIF_PER_TRAP
238 jo sysc_singlestep
239 j sysc_return # beware of critical section cleanup
241 #
242 # _TIF_NEED_RESCHED is set, call schedule
243 #
244 sysc_reschedule:
245 l %r1,BASED(.Lschedule)
246 la %r14,BASED(sysc_return)
247 br %r1 # call schedule
249 #
250 # _TIF_MCCK_PENDING is set, call handler
251 #
252 sysc_mcck_pending:
253 l %r1,BASED(.Lhandle_mcck)
254 la %r14,BASED(sysc_return)
255 br %r1 # TIF bit will be cleared by handler
257 #
258 # _TIF_SIGPENDING is set, call do_signal
259 #
260 sysc_sigpending:
261 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
262 lr %r2,%r11 # pass pointer to pt_regs
263 l %r1,BASED(.Ldo_signal)
264 basr %r14,%r1 # call do_signal
265 tm __TI_flags+3(%r12),_TIF_SYSCALL
266 jno sysc_return
267 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
268 xr %r8,%r8 # svc 0 returns -ENOSYS
269 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
270 jnl sysc_nr_ok # invalid svc number -> do svc 0
271 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
272 sla %r8,2
273 j sysc_nr_ok # restart svc
275 #
276 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
277 #
278 sysc_notify_resume:
279 lr %r2,%r11 # pass pointer to pt_regs
280 l %r1,BASED(.Ldo_notify_resume)
281 la %r14,BASED(sysc_return)
282 br %r1 # call do_notify_resume
284 #
285 # _TIF_PER_TRAP is set, call do_per_trap
286 #
287 sysc_singlestep:
288 ni __TI_flags+3(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP)
289 lr %r2,%r11 # pass pointer to pt_regs
290 l %r1,BASED(.Ldo_per_trap)
291 la %r14,BASED(sysc_return)
292 br %r1 # call do_per_trap
294 #
295 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
296 # and after the system call
297 #
298 sysc_tracesys:
299 l %r1,BASED(.Ltrace_enter)
300 lr %r2,%r11 # pass pointer to pt_regs
301 la %r3,0
302 xr %r0,%r0
303 icm %r0,3,__PT_INT_CODE+2(%r11)
304 st %r0,__PT_R2(%r11)
305 basr %r14,%r1 # call do_syscall_trace_enter
306 cl %r2,BASED(.Lnr_syscalls)
307 jnl sysc_tracenogo
308 lr %r8,%r2
309 sll %r8,2
310 l %r9,0(%r8,%r10)
311 sysc_tracego:
312 lm %r3,%r7,__PT_R3(%r11)
313 st %r7,STACK_FRAME_OVERHEAD(%r15)
314 l %r2,__PT_ORIG_GPR2(%r11)
315 basr %r14,%r9 # call sys_xxx
316 st %r2,__PT_R2(%r11) # store return value
317 sysc_tracenogo:
318 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
319 jz sysc_return
320 l %r1,BASED(.Ltrace_exit)
321 lr %r2,%r11 # pass pointer to pt_regs
322 la %r14,BASED(sysc_return)
323 br %r1 # call do_syscall_trace_exit
325 #
326 # a new process exits the kernel with ret_from_fork
327 #
328 ENTRY(ret_from_fork)
329 la %r11,STACK_FRAME_OVERHEAD(%r15)
330 l %r12,__LC_THREAD_INFO
331 l %r13,__LC_SVC_NEW_PSW+4
332 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
333 jo 0f
334 st %r15,__PT_R15(%r11) # store stack pointer for new kthread
335 0: l %r1,BASED(.Lschedule_tail)
336 basr %r14,%r1 # call schedule_tail
337 TRACE_IRQS_ON
338 ssm __LC_SVC_NEW_PSW # reenable interrupts
339 j sysc_tracenogo
341 #
342 # kernel_execve function needs to deal with pt_regs that is not
343 # at the usual place
344 #
345 ENTRY(kernel_execve)
346 stm %r12,%r15,48(%r15)
347 lr %r14,%r15
348 l %r13,__LC_SVC_NEW_PSW+4
349 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
350 st %r14,__SF_BACKCHAIN(%r15)
351 la %r12,STACK_FRAME_OVERHEAD(%r15)
352 xc 0(__PT_SIZE,%r12),0(%r12)
353 l %r1,BASED(.Ldo_execve)
354 lr %r5,%r12
355 basr %r14,%r1 # call do_execve
356 ltr %r2,%r2
357 je 0f
358 ahi %r15,(STACK_FRAME_OVERHEAD + __PT_SIZE)
359 lm %r12,%r15,48(%r15)
360 br %r14
361 # execve succeeded.
362 0: ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
363 l %r15,__LC_KERNEL_STACK # load ksp
364 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
365 la %r11,STACK_FRAME_OVERHEAD(%r15)
366 mvc 0(__PT_SIZE,%r11),0(%r12) # copy pt_regs
367 l %r12,__LC_THREAD_INFO
368 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
369 ssm __LC_SVC_NEW_PSW # reenable interrupts
370 l %r1,BASED(.Lexecve_tail)
371 basr %r14,%r1 # call execve_tail
372 j sysc_return
374 /*
375 * Program check handler routine
376 */
378 ENTRY(pgm_check_handler)
379 stpt __LC_SYNC_ENTER_TIMER
380 stm %r8,%r15,__LC_SAVE_AREA_SYNC
381 l %r12,__LC_THREAD_INFO
382 l %r13,__LC_SVC_NEW_PSW+4
383 lm %r8,%r9,__LC_PGM_OLD_PSW
384 tmh %r8,0x0001 # test problem state bit
385 jnz 1f # -> fault in user space
386 tmh %r8,0x4000 # PER bit set in old PSW ?
387 jnz 0f # -> enabled, can't be a double fault
388 tm __LC_PGM_ILC+3,0x80 # check for per exception
389 jnz pgm_svcper # -> single stepped svc
390 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
391 j 2f
392 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
393 l %r15,__LC_KERNEL_STACK
394 2: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
395 la %r11,STACK_FRAME_OVERHEAD(%r15)
396 stm %r0,%r7,__PT_R0(%r11)
397 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
398 stm %r8,%r9,__PT_PSW(%r11)
399 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
400 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
401 tm __LC_PGM_ILC+3,0x80 # check for per exception
402 jz 0f
403 l %r1,__TI_task(%r12)
404 tmh %r8,0x0001 # kernel per event ?
405 jz pgm_kprobe
406 oi __TI_flags+3(%r12),_TIF_PER_TRAP
407 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
408 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
409 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
410 0: REENABLE_IRQS
411 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
412 l %r1,BASED(.Ljump_table)
413 la %r10,0x7f
414 n %r10,__PT_INT_CODE(%r11)
415 je sysc_return
416 sll %r10,2
417 l %r1,0(%r10,%r1) # load address of handler routine
418 lr %r2,%r11 # pass pointer to pt_regs
419 basr %r14,%r1 # branch to interrupt-handler
420 j sysc_return
422 #
423 # PER event in supervisor state, must be kprobes
424 #
425 pgm_kprobe:
426 REENABLE_IRQS
427 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
428 l %r1,BASED(.Ldo_per_trap)
429 lr %r2,%r11 # pass pointer to pt_regs
430 basr %r14,%r1 # call do_per_trap
431 j sysc_return
433 #
434 # single stepped system call
435 #
436 pgm_svcper:
437 oi __TI_flags+3(%r12),_TIF_PER_TRAP
438 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
439 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
440 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
442 /*
443 * IO interrupt handler routine
444 */
446 ENTRY(io_int_handler)
447 stck __LC_INT_CLOCK
448 stpt __LC_ASYNC_ENTER_TIMER
449 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
450 l %r12,__LC_THREAD_INFO
451 l %r13,__LC_SVC_NEW_PSW+4
452 lm %r8,%r9,__LC_IO_OLD_PSW
453 tmh %r8,0x0001 # interrupting from user ?
454 jz io_skip
455 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
456 io_skip:
457 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
458 stm %r0,%r7,__PT_R0(%r11)
459 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
460 stm %r8,%r9,__PT_PSW(%r11)
461 TRACE_IRQS_OFF
462 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
463 l %r1,BASED(.Ldo_IRQ)
464 lr %r2,%r11 # pass pointer to pt_regs
465 basr %r14,%r1 # call do_IRQ
466 io_return:
467 LOCKDEP_SYS_EXIT
468 TRACE_IRQS_ON
469 io_tif:
470 tm __TI_flags+3(%r12),_TIF_WORK_INT
471 jnz io_work # there is work to do (signals etc.)
472 io_restore:
473 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
474 ni __LC_RETURN_PSW+1,0xfd # clean wait state bit
475 stpt __LC_EXIT_TIMER
476 lm %r0,%r15,__PT_R0(%r11)
477 lpsw __LC_RETURN_PSW
478 io_done:
480 #
481 # There is work todo, find out in which context we have been interrupted:
482 # 1) if we return to user space we can do all _TIF_WORK_INT work
483 # 2) if we return to kernel code and preemptive scheduling is enabled check
484 # the preemption counter and if it is zero call preempt_schedule_irq
485 # Before any work can be done, a switch to the kernel stack is required.
486 #
487 io_work:
488 tm __PT_PSW+1(%r11),0x01 # returning to user ?
489 jo io_work_user # yes -> do resched & signal
490 #ifdef CONFIG_PREEMPT
491 # check for preemptive scheduling
492 icm %r0,15,__TI_precount(%r12)
493 jnz io_restore # preemption disabled
494 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
495 jno io_restore
496 # switch to kernel stack
497 l %r1,__PT_R15(%r11)
498 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
499 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
500 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
501 la %r11,STACK_FRAME_OVERHEAD(%r1)
502 lr %r15,%r1
503 # TRACE_IRQS_ON already done at io_return, call
504 # TRACE_IRQS_OFF to keep things symmetrical
505 TRACE_IRQS_OFF
506 l %r1,BASED(.Lpreempt_irq)
507 basr %r14,%r1 # call preempt_schedule_irq
508 j io_return
509 #else
510 j io_restore
511 #endif
513 #
514 # Need to do work before returning to userspace, switch to kernel stack
515 #
516 io_work_user:
517 l %r1,__LC_KERNEL_STACK
518 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
519 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
520 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
521 la %r11,STACK_FRAME_OVERHEAD(%r1)
522 lr %r15,%r1
524 #
525 # One of the work bits is on. Find out which one.
526 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
527 # and _TIF_MCCK_PENDING
528 #
529 io_work_tif:
530 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
531 jo io_mcck_pending
532 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
533 jo io_reschedule
534 tm __TI_flags+3(%r12),_TIF_SIGPENDING
535 jo io_sigpending
536 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
537 jo io_notify_resume
538 j io_return # beware of critical section cleanup
540 #
541 # _TIF_MCCK_PENDING is set, call handler
542 #
543 io_mcck_pending:
544 # TRACE_IRQS_ON already done at io_return
545 l %r1,BASED(.Lhandle_mcck)
546 basr %r14,%r1 # TIF bit will be cleared by handler
547 TRACE_IRQS_OFF
548 j io_return
550 #
551 # _TIF_NEED_RESCHED is set, call schedule
552 #
553 io_reschedule:
554 # TRACE_IRQS_ON already done at io_return
555 l %r1,BASED(.Lschedule)
556 ssm __LC_SVC_NEW_PSW # reenable interrupts
557 basr %r14,%r1 # call scheduler
558 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
559 TRACE_IRQS_OFF
560 j io_return
562 #
563 # _TIF_SIGPENDING is set, call do_signal
564 #
565 io_sigpending:
566 # TRACE_IRQS_ON already done at io_return
567 l %r1,BASED(.Ldo_signal)
568 ssm __LC_SVC_NEW_PSW # reenable interrupts
569 lr %r2,%r11 # pass pointer to pt_regs
570 basr %r14,%r1 # call do_signal
571 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
572 TRACE_IRQS_OFF
573 j io_return
575 #
576 # _TIF_SIGPENDING is set, call do_signal
577 #
578 io_notify_resume:
579 # TRACE_IRQS_ON already done at io_return
580 l %r1,BASED(.Ldo_notify_resume)
581 ssm __LC_SVC_NEW_PSW # reenable interrupts
582 lr %r2,%r11 # pass pointer to pt_regs
583 basr %r14,%r1 # call do_notify_resume
584 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
585 TRACE_IRQS_OFF
586 j io_return
588 /*
589 * External interrupt handler routine
590 */
592 ENTRY(ext_int_handler)
593 stck __LC_INT_CLOCK
594 stpt __LC_ASYNC_ENTER_TIMER
595 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
596 l %r12,__LC_THREAD_INFO
597 l %r13,__LC_SVC_NEW_PSW+4
598 lm %r8,%r9,__LC_EXT_OLD_PSW
599 tmh %r8,0x0001 # interrupting from user ?
600 jz ext_skip
601 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
602 ext_skip:
603 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
604 stm %r0,%r7,__PT_R0(%r11)
605 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
606 stm %r8,%r9,__PT_PSW(%r11)
607 TRACE_IRQS_OFF
608 lr %r2,%r11 # pass pointer to pt_regs
609 l %r3,__LC_CPU_ADDRESS # get cpu address + interruption code
610 l %r4,__LC_EXT_PARAMS # get external parameters
611 l %r1,BASED(.Ldo_extint)
612 basr %r14,%r1 # call do_extint
613 j io_return
615 __critical_end:
617 /*
618 * Machine check handler routines
619 */
621 ENTRY(mcck_int_handler)
622 stck __LC_MCCK_CLOCK
623 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
624 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
625 l %r12,__LC_THREAD_INFO
626 l %r13,__LC_SVC_NEW_PSW+4
627 lm %r8,%r9,__LC_MCK_OLD_PSW
628 tm __LC_MCCK_CODE,0x80 # system damage?
629 jo mcck_panic # yes -> rest of mcck code invalid
630 la %r14,__LC_CPU_TIMER_SAVE_AREA
631 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
632 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
633 jo 3f
634 la %r14,__LC_SYNC_ENTER_TIMER
635 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
636 jl 0f
637 la %r14,__LC_ASYNC_ENTER_TIMER
638 0: clc 0(8,%r14),__LC_EXIT_TIMER
639 jl 1f
640 la %r14,__LC_EXIT_TIMER
641 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
642 jl 2f
643 la %r14,__LC_LAST_UPDATE_TIMER
644 2: spt 0(%r14)
645 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
646 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
647 jno mcck_panic # no -> skip cleanup critical
648 tm %r8,0x0001 # interrupting from user ?
649 jz mcck_skip
650 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
651 mcck_skip:
652 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
653 mvc __PT_R0(64,%r11),__LC_GPREGS_SAVE_AREA
654 stm %r8,%r9,__PT_PSW(%r11)
655 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
656 l %r1,BASED(.Ldo_machine_check)
657 lr %r2,%r11 # pass pointer to pt_regs
658 basr %r14,%r1 # call s390_do_machine_check
659 tm __PT_PSW+1(%r11),0x01 # returning to user ?
660 jno mcck_return
661 l %r1,__LC_KERNEL_STACK # switch to kernel stack
662 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
663 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
664 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
665 la %r11,STACK_FRAME_OVERHEAD(%r15)
666 lr %r15,%r1
667 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
668 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
669 jno mcck_return
670 TRACE_IRQS_OFF
671 l %r1,BASED(.Lhandle_mcck)
672 basr %r14,%r1 # call s390_handle_mcck
673 TRACE_IRQS_ON
674 mcck_return:
675 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
676 ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit
677 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
678 jno 0f
679 lm %r0,%r15,__PT_R0(%r11)
680 stpt __LC_EXIT_TIMER
681 lpsw __LC_RETURN_MCCK_PSW
682 0: lm %r0,%r15,__PT_R0(%r11)
683 lpsw __LC_RETURN_MCCK_PSW
685 mcck_panic:
686 l %r14,__LC_PANIC_STACK
687 slr %r14,%r15
688 sra %r14,PAGE_SHIFT
689 jz 0f
690 l %r15,__LC_PANIC_STACK
691 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
692 j mcck_skip
694 /*
695 * Restart interruption handler, kick starter for additional CPUs
696 */
697 #ifdef CONFIG_SMP
698 __CPUINIT
699 ENTRY(restart_int_handler)
700 basr %r1,0
701 restart_base:
702 spt restart_vtime-restart_base(%r1)
703 stck __LC_LAST_UPDATE_CLOCK
704 mvc __LC_LAST_UPDATE_TIMER(8),restart_vtime-restart_base(%r1)
705 mvc __LC_EXIT_TIMER(8),restart_vtime-restart_base(%r1)
706 l %r15,__LC_GPREGS_SAVE_AREA+60 # load ksp
707 lctl %c0,%c15,__LC_CREGS_SAVE_AREA # get new ctl regs
708 lam %a0,%a15,__LC_AREGS_SAVE_AREA
709 lm %r6,%r15,__SF_GPRS(%r15)# load registers from clone
710 l %r1,__LC_THREAD_INFO
711 mvc __LC_USER_TIMER(8),__TI_user_timer(%r1)
712 mvc __LC_SYSTEM_TIMER(8),__TI_system_timer(%r1)
713 xc __LC_STEAL_TIMER(8),__LC_STEAL_TIMER
714 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
715 basr %r14,0
716 l %r14,restart_addr-.(%r14)
717 basr %r14,%r14 # call start_secondary
718 restart_addr:
719 .long start_secondary
720 .align 8
721 restart_vtime:
722 .long 0x7fffffff,0xffffffff
723 .previous
724 #else
725 /*
726 * If we do not run with SMP enabled, let the new CPU crash ...
727 */
728 ENTRY(restart_int_handler)
729 basr %r1,0
730 restart_base:
731 lpsw restart_crash-restart_base(%r1)
732 .align 8
733 restart_crash:
734 .long 0x000a0000,0x00000000
735 restart_go:
736 #endif
738 #
739 # PSW restart interrupt handler
740 #
741 ENTRY(psw_restart_int_handler)
742 st %r15,__LC_SAVE_AREA_RESTART
743 basr %r15,0
744 0: l %r15,.Lrestart_stack-0b(%r15) # load restart stack
745 l %r15,0(%r15)
746 ahi %r15,-__PT_SIZE # create pt_regs on stack
747 stm %r0,%r14,__PT_R0(%r15)
748 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
749 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
750 ahi %r15,-STACK_FRAME_OVERHEAD
751 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
752 basr %r14,0
753 1: l %r14,.Ldo_restart-1b(%r14)
754 basr %r14,%r14
755 basr %r14,0 # load disabled wait PSW if
756 2: lpsw restart_psw_crash-2b(%r14) # do_restart returns
757 .align 4
758 .Ldo_restart:
759 .long do_restart
760 .Lrestart_stack:
761 .long restart_stack
762 .align 8
763 restart_psw_crash:
764 .long 0x000a0000,0x00000000 + restart_psw_crash
766 .section .kprobes.text, "ax"
768 #ifdef CONFIG_CHECK_STACK
769 /*
770 * The synchronous or the asynchronous stack overflowed. We are dead.
771 * No need to properly save the registers, we are going to panic anyway.
772 * Setup a pt_regs so that show_trace can provide a good call trace.
773 */
774 stack_overflow:
775 l %r15,__LC_PANIC_STACK # change to panic stack
776 ahi %r15,-__PT_SIZE # create pt_regs
777 stm %r0,%r7,__PT_R0(%r15)
778 stm %r8,%r9,__PT_PSW(%r15)
779 mvc __PT_R8(32,%r11),0(%r14)
780 lr %r15,%r11
781 ahi %r15,-STACK_FRAME_OVERHEAD
782 l %r1,BASED(1f)
783 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
784 lr %r2,%r11 # pass pointer to pt_regs
785 br %r1 # branch to kernel_stack_overflow
786 1: .long kernel_stack_overflow
787 #endif
789 cleanup_table:
790 .long system_call + 0x80000000
791 .long sysc_do_svc + 0x80000000
792 .long sysc_tif + 0x80000000
793 .long sysc_restore + 0x80000000
794 .long sysc_done + 0x80000000
795 .long io_tif + 0x80000000
796 .long io_restore + 0x80000000
797 .long io_done + 0x80000000
799 cleanup_critical:
800 cl %r9,BASED(cleanup_table) # system_call
801 jl 0f
802 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
803 jl cleanup_system_call
804 cl %r9,BASED(cleanup_table+8) # sysc_tif
805 jl 0f
806 cl %r9,BASED(cleanup_table+12) # sysc_restore
807 jl cleanup_sysc_tif
808 cl %r9,BASED(cleanup_table+16) # sysc_done
809 jl cleanup_sysc_restore
810 cl %r9,BASED(cleanup_table+20) # io_tif
811 jl 0f
812 cl %r9,BASED(cleanup_table+24) # io_restore
813 jl cleanup_io_tif
814 cl %r9,BASED(cleanup_table+28) # io_done
815 jl cleanup_io_restore
816 0: br %r14
818 cleanup_system_call:
819 # check if stpt has been executed
820 cl %r9,BASED(cleanup_system_call_insn)
821 jh 0f
822 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
823 chi %r11,__LC_SAVE_AREA_ASYNC
824 je 0f
825 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
826 0: # check if stm has been executed
827 cl %r9,BASED(cleanup_system_call_insn+4)
828 jh 0f
829 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
830 0: # set up saved registers r12, and r13
831 st %r12,16(%r11) # r12 thread-info pointer
832 st %r13,20(%r11) # r13 literal-pool pointer
833 # check if the user time calculation has been done
834 cl %r9,BASED(cleanup_system_call_insn+8)
835 jh 0f
836 l %r10,__LC_EXIT_TIMER
837 l %r15,__LC_EXIT_TIMER+4
838 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
839 ADD64 %r10,%r15,__LC_USER_TIMER
840 st %r10,__LC_USER_TIMER
841 st %r15,__LC_USER_TIMER+4
842 0: # check if the system time calculation has been done
843 cl %r9,BASED(cleanup_system_call_insn+12)
844 jh 0f
845 l %r10,__LC_LAST_UPDATE_TIMER
846 l %r15,__LC_LAST_UPDATE_TIMER+4
847 SUB64 %r10,%r15,__LC_EXIT_TIMER
848 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
849 st %r10,__LC_SYSTEM_TIMER
850 st %r15,__LC_SYSTEM_TIMER+4
851 0: # update accounting time stamp
852 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
853 # set up saved register 11
854 l %r15,__LC_KERNEL_STACK
855 ahi %r15,-__PT_SIZE
856 st %r15,12(%r11) # r11 pt_regs pointer
857 # fill pt_regs
858 mvc __PT_R8(32,%r15),__LC_SAVE_AREA_SYNC
859 stm %r0,%r7,__PT_R0(%r15)
860 mvc __PT_PSW(8,%r15),__LC_SVC_OLD_PSW
861 mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC
862 # setup saved register 15
863 ahi %r15,-STACK_FRAME_OVERHEAD
864 st %r15,28(%r11) # r15 stack pointer
865 # set new psw address and exit
866 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
867 br %r14
868 cleanup_system_call_insn:
869 .long system_call + 0x80000000
870 .long sysc_stm + 0x80000000
871 .long sysc_vtime + 0x80000000 + 36
872 .long sysc_vtime + 0x80000000 + 76
874 cleanup_sysc_tif:
875 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
876 br %r14
878 cleanup_sysc_restore:
879 cl %r9,BASED(cleanup_sysc_restore_insn)
880 jhe 0f
881 l %r9,12(%r11) # get saved pointer to pt_regs
882 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
883 mvc 0(32,%r11),__PT_R8(%r9)
884 lm %r0,%r7,__PT_R0(%r9)
885 0: lm %r8,%r9,__LC_RETURN_PSW
886 br %r14
887 cleanup_sysc_restore_insn:
888 .long sysc_done - 4 + 0x80000000
890 cleanup_io_tif:
891 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
892 br %r14
894 cleanup_io_restore:
895 cl %r9,BASED(cleanup_io_restore_insn)
896 jhe 0f
897 l %r9,12(%r11) # get saved r11 pointer to pt_regs
898 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
899 ni __LC_RETURN_PSW+1,0xfd # clear wait state bit
900 mvc 0(32,%r11),__PT_R8(%r9)
901 lm %r0,%r7,__PT_R0(%r9)
902 0: lm %r8,%r9,__LC_RETURN_PSW
903 br %r14
904 cleanup_io_restore_insn:
905 .long io_done - 4 + 0x80000000
907 /*
908 * Integer constants
909 */
910 .align 4
911 .Lnr_syscalls: .long NR_syscalls
913 /*
914 * Symbol constants
915 */
916 .Ldo_machine_check: .long s390_do_machine_check
917 .Lhandle_mcck: .long s390_handle_mcck
918 .Ldo_IRQ: .long do_IRQ
919 .Ldo_extint: .long do_extint
920 .Ldo_signal: .long do_signal
921 .Ldo_notify_resume: .long do_notify_resume
922 .Ldo_per_trap: .long do_per_trap
923 .Ldo_execve: .long do_execve
924 .Lexecve_tail: .long execve_tail
925 .Ljump_table: .long pgm_check_table
926 .Lschedule: .long schedule
927 #ifdef CONFIG_PREEMPT
928 .Lpreempt_irq: .long preempt_schedule_irq
929 #endif
930 .Ltrace_enter: .long do_syscall_trace_enter
931 .Ltrace_exit: .long do_syscall_trace_exit
932 .Lschedule_tail: .long schedule_tail
933 .Lsys_call_table: .long sys_call_table
934 .Lsysc_per: .long sysc_per + 0x80000000
935 #ifdef CONFIG_TRACE_IRQFLAGS
936 .Lhardirqs_on: .long trace_hardirqs_on_caller
937 .Lhardirqs_off: .long trace_hardirqs_off_caller
938 #endif
939 #ifdef CONFIG_LOCKDEP
940 .Llockdep_sys_exit: .long lockdep_sys_exit
941 #endif
942 .Lcritical_start: .long __critical_start + 0x80000000
943 .Lcritical_length: .long __critical_end - __critical_start
945 .section .rodata, "a"
946 #define SYSCALL(esa,esame,emu) .long esa
947 .globl sys_call_table
948 sys_call_table:
949 #include "syscalls.S"
950 #undef SYSCALL