| blob | be7a408be7a16bafde657665edb36fcde01c8866 |
1 /*
2 * S390 low-level entry points.
3 *
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Hartmut Penner (hp@de.ibm.com),
7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8 * Heiko Carstens <heiko.carstens@de.ibm.com>
9 */
11 #include <linux/init.h>
12 #include <linux/linkage.h>
13 #include <asm/cache.h>
14 #include <asm/errno.h>
15 #include <asm/ptrace.h>
16 #include <asm/thread_info.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/unistd.h>
19 #include <asm/page.h>
20 #include <asm/sigp.h>
22 __PT_R0 = __PT_GPRS
23 __PT_R1 = __PT_GPRS + 4
24 __PT_R2 = __PT_GPRS + 8
25 __PT_R3 = __PT_GPRS + 12
26 __PT_R4 = __PT_GPRS + 16
27 __PT_R5 = __PT_GPRS + 20
28 __PT_R6 = __PT_GPRS + 24
29 __PT_R7 = __PT_GPRS + 28
30 __PT_R8 = __PT_GPRS + 32
31 __PT_R9 = __PT_GPRS + 36
32 __PT_R10 = __PT_GPRS + 40
33 __PT_R11 = __PT_GPRS + 44
34 __PT_R12 = __PT_GPRS + 48
35 __PT_R13 = __PT_GPRS + 524
36 __PT_R14 = __PT_GPRS + 56
37 __PT_R15 = __PT_GPRS + 60
39 _TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
40 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
41 _TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
42 _TIF_MCCK_PENDING)
43 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
44 _TIF_SYSCALL_TRACEPOINT)
46 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
47 STACK_SIZE = 1 << STACK_SHIFT
48 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
50 #define BASED(name) name-system_call(%r13)
52 .macro TRACE_IRQS_ON
53 #ifdef CONFIG_TRACE_IRQFLAGS
54 basr %r2,%r0
55 l %r1,BASED(.Lhardirqs_on)
56 basr %r14,%r1 # call trace_hardirqs_on_caller
57 #endif
58 .endm
60 .macro TRACE_IRQS_OFF
61 #ifdef CONFIG_TRACE_IRQFLAGS
62 basr %r2,%r0
63 l %r1,BASED(.Lhardirqs_off)
64 basr %r14,%r1 # call trace_hardirqs_off_caller
65 #endif
66 .endm
68 .macro LOCKDEP_SYS_EXIT
69 #ifdef CONFIG_LOCKDEP
70 tm __PT_PSW+1(%r11),0x01 # returning to user ?
71 jz .+10
72 l %r1,BASED(.Llockdep_sys_exit)
73 basr %r14,%r1 # call lockdep_sys_exit
74 #endif
75 .endm
77 .macro CHECK_STACK stacksize,savearea
78 #ifdef CONFIG_CHECK_STACK
79 tml %r15,\stacksize - CONFIG_STACK_GUARD
80 la %r14,\savearea
81 jz stack_overflow
82 #endif
83 .endm
85 .macro SWITCH_ASYNC savearea,stack,shift
86 tmh %r8,0x0001 # interrupting from user ?
87 jnz 1f
88 lr %r14,%r9
89 sl %r14,BASED(.Lcritical_start)
90 cl %r14,BASED(.Lcritical_length)
91 jhe 0f
92 la %r11,\savearea # inside critical section, do cleanup
93 bras %r14,cleanup_critical
94 tmh %r8,0x0001 # retest problem state after cleanup
95 jnz 1f
96 0: l %r14,\stack # are we already on the target stack?
97 slr %r14,%r15
98 sra %r14,\shift
99 jnz 1f
100 CHECK_STACK 1<<\shift,\savearea
101 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
102 j 2f
103 1: l %r15,\stack # load target stack
104 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
105 .endm
107 .macro ADD64 high,low,timer
108 al \high,\timer
109 al \low,4+\timer
110 brc 12,.+8
111 ahi \high,1
112 .endm
114 .macro SUB64 high,low,timer
115 sl \high,\timer
116 sl \low,4+\timer
117 brc 3,.+8
118 ahi \high,-1
119 .endm
121 .macro UPDATE_VTIME high,low,enter_timer
122 lm \high,\low,__LC_EXIT_TIMER
123 SUB64 \high,\low,\enter_timer
124 ADD64 \high,\low,__LC_USER_TIMER
125 stm \high,\low,__LC_USER_TIMER
126 lm \high,\low,__LC_LAST_UPDATE_TIMER
127 SUB64 \high,\low,__LC_EXIT_TIMER
128 ADD64 \high,\low,__LC_SYSTEM_TIMER
129 stm \high,\low,__LC_SYSTEM_TIMER
130 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
131 .endm
133 .macro REENABLE_IRQS
134 st %r8,__LC_RETURN_PSW
135 ni __LC_RETURN_PSW,0xbf
136 ssm __LC_RETURN_PSW
137 .endm
139 .section .kprobes.text, "ax"
141 /*
142 * Scheduler resume function, called by switch_to
143 * gpr2 = (task_struct *) prev
144 * gpr3 = (task_struct *) next
145 * Returns:
146 * gpr2 = prev
147 */
148 ENTRY(__switch_to)
149 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
150 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
151 l %r4,__THREAD_info(%r2) # get thread_info of prev
152 l %r5,__THREAD_info(%r3) # get thread_info of next
153 lr %r15,%r5
154 ahi %r15,STACK_INIT # end of kernel stack of next
155 st %r3,__LC_CURRENT # store task struct of next
156 st %r5,__LC_THREAD_INFO # store thread info of next
157 st %r15,__LC_KERNEL_STACK # store end of kernel stack
158 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
159 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
160 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
161 tm __TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
162 jz 0f
163 ni __TI_flags+3(%r4),255-_TIF_MCCK_PENDING # clear flag in prev
164 oi __TI_flags+3(%r5),_TIF_MCCK_PENDING # set it in next
165 0: lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
166 br %r14
168 __critical_start:
169 /*
170 * SVC interrupt handler routine. System calls are synchronous events and
171 * are executed with interrupts enabled.
172 */
174 ENTRY(system_call)
175 stpt __LC_SYNC_ENTER_TIMER
176 sysc_stm:
177 stm %r8,%r15,__LC_SAVE_AREA_SYNC
178 l %r12,__LC_THREAD_INFO
179 l %r13,__LC_SVC_NEW_PSW+4
180 sysc_per:
181 l %r15,__LC_KERNEL_STACK
182 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
183 sysc_vtime:
184 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
185 stm %r0,%r7,__PT_R0(%r11)
186 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
187 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
188 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
189 sysc_do_svc:
190 oi __TI_flags+3(%r12),_TIF_SYSCALL
191 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
192 lh %r8,__PT_INT_CODE+2(%r11)
193 sla %r8,2 # shift and test for svc0
194 jnz sysc_nr_ok
195 # svc 0: system call number in %r1
196 cl %r1,BASED(.Lnr_syscalls)
197 jnl sysc_nr_ok
198 sth %r1,__PT_INT_CODE+2(%r11)
199 lr %r8,%r1
200 sla %r8,2
201 sysc_nr_ok:
202 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
203 st %r2,__PT_ORIG_GPR2(%r11)
204 st %r7,STACK_FRAME_OVERHEAD(%r15)
205 l %r9,0(%r8,%r10) # get system call addr.
206 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
207 jnz sysc_tracesys
208 basr %r14,%r9 # call sys_xxxx
209 st %r2,__PT_R2(%r11) # store return value
211 sysc_return:
212 LOCKDEP_SYS_EXIT
213 sysc_tif:
214 tm __PT_PSW+1(%r11),0x01 # returning to user ?
215 jno sysc_restore
216 tm __TI_flags+3(%r12),_TIF_WORK_SVC
217 jnz sysc_work # check for work
218 ni __TI_flags+3(%r12),255-_TIF_SYSCALL
219 sysc_restore:
220 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
221 stpt __LC_EXIT_TIMER
222 lm %r0,%r15,__PT_R0(%r11)
223 lpsw __LC_RETURN_PSW
224 sysc_done:
226 #
227 # One of the work bits is on. Find out which one.
228 #
229 sysc_work:
230 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
231 jo sysc_mcck_pending
232 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
233 jo sysc_reschedule
234 tm __TI_flags+3(%r12),_TIF_PER_TRAP
235 jo sysc_singlestep
236 tm __TI_flags+3(%r12),_TIF_SIGPENDING
237 jo sysc_sigpending
238 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
239 jo sysc_notify_resume
240 j sysc_return # beware of critical section cleanup
242 #
243 # _TIF_NEED_RESCHED is set, call schedule
244 #
245 sysc_reschedule:
246 l %r1,BASED(.Lschedule)
247 la %r14,BASED(sysc_return)
248 br %r1 # call schedule
250 #
251 # _TIF_MCCK_PENDING is set, call handler
252 #
253 sysc_mcck_pending:
254 l %r1,BASED(.Lhandle_mcck)
255 la %r14,BASED(sysc_return)
256 br %r1 # TIF bit will be cleared by handler
258 #
259 # _TIF_SIGPENDING is set, call do_signal
260 #
261 sysc_sigpending:
262 lr %r2,%r11 # pass pointer to pt_regs
263 l %r1,BASED(.Ldo_signal)
264 basr %r14,%r1 # call do_signal
265 tm __TI_flags+3(%r12),_TIF_SYSCALL
266 jno sysc_return
267 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
268 xr %r8,%r8 # svc 0 returns -ENOSYS
269 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
270 jnl sysc_nr_ok # invalid svc number -> do svc 0
271 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
272 sla %r8,2
273 j sysc_nr_ok # restart svc
275 #
276 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
277 #
278 sysc_notify_resume:
279 lr %r2,%r11 # pass pointer to pt_regs
280 l %r1,BASED(.Ldo_notify_resume)
281 la %r14,BASED(sysc_return)
282 br %r1 # call do_notify_resume
284 #
285 # _TIF_PER_TRAP is set, call do_per_trap
286 #
287 sysc_singlestep:
288 ni __TI_flags+3(%r12),255-_TIF_PER_TRAP
289 lr %r2,%r11 # pass pointer to pt_regs
290 l %r1,BASED(.Ldo_per_trap)
291 la %r14,BASED(sysc_return)
292 br %r1 # call do_per_trap
294 #
295 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
296 # and after the system call
297 #
298 sysc_tracesys:
299 l %r1,BASED(.Ltrace_enter)
300 lr %r2,%r11 # pass pointer to pt_regs
301 la %r3,0
302 xr %r0,%r0
303 icm %r0,3,__PT_INT_CODE+2(%r11)
304 st %r0,__PT_R2(%r11)
305 basr %r14,%r1 # call do_syscall_trace_enter
306 cl %r2,BASED(.Lnr_syscalls)
307 jnl sysc_tracenogo
308 lr %r8,%r2
309 sll %r8,2
310 l %r9,0(%r8,%r10)
311 sysc_tracego:
312 lm %r3,%r7,__PT_R3(%r11)
313 st %r7,STACK_FRAME_OVERHEAD(%r15)
314 l %r2,__PT_ORIG_GPR2(%r11)
315 basr %r14,%r9 # call sys_xxx
316 st %r2,__PT_R2(%r11) # store return value
317 sysc_tracenogo:
318 tm __TI_flags+2(%r12),_TIF_TRACE >> 8
319 jz sysc_return
320 l %r1,BASED(.Ltrace_exit)
321 lr %r2,%r11 # pass pointer to pt_regs
322 la %r14,BASED(sysc_return)
323 br %r1 # call do_syscall_trace_exit
325 #
326 # a new process exits the kernel with ret_from_fork
327 #
328 ENTRY(ret_from_fork)
329 la %r11,STACK_FRAME_OVERHEAD(%r15)
330 l %r12,__LC_THREAD_INFO
331 l %r13,__LC_SVC_NEW_PSW+4
332 l %r1,BASED(.Lschedule_tail)
333 basr %r14,%r1 # call schedule_tail
334 TRACE_IRQS_ON
335 ssm __LC_SVC_NEW_PSW # reenable interrupts
336 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
337 jne sysc_tracenogo
338 # it's a kernel thread
339 lm %r9,%r10,__PT_R9(%r11) # load gprs
340 ENTRY(kernel_thread_starter)
341 la %r2,0(%r10)
342 basr %r14,%r9
343 j sysc_tracenogo
345 /*
346 * Program check handler routine
347 */
349 ENTRY(pgm_check_handler)
350 stpt __LC_SYNC_ENTER_TIMER
351 stm %r8,%r15,__LC_SAVE_AREA_SYNC
352 l %r12,__LC_THREAD_INFO
353 l %r13,__LC_SVC_NEW_PSW+4
354 lm %r8,%r9,__LC_PGM_OLD_PSW
355 tmh %r8,0x0001 # test problem state bit
356 jnz 1f # -> fault in user space
357 tmh %r8,0x4000 # PER bit set in old PSW ?
358 jnz 0f # -> enabled, can't be a double fault
359 tm __LC_PGM_ILC+3,0x80 # check for per exception
360 jnz pgm_svcper # -> single stepped svc
361 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
362 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
363 j 2f
364 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
365 l %r15,__LC_KERNEL_STACK
366 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
367 stm %r0,%r7,__PT_R0(%r11)
368 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
369 stm %r8,%r9,__PT_PSW(%r11)
370 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
371 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
372 tm __LC_PGM_ILC+3,0x80 # check for per exception
373 jz 0f
374 l %r1,__TI_task(%r12)
375 tmh %r8,0x0001 # kernel per event ?
376 jz pgm_kprobe
377 oi __TI_flags+3(%r12),_TIF_PER_TRAP
378 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
379 mvc __THREAD_per_cause(2,%r1),__LC_PER_CAUSE
380 mvc __THREAD_per_paid(1,%r1),__LC_PER_PAID
381 0: REENABLE_IRQS
382 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
383 l %r1,BASED(.Ljump_table)
384 la %r10,0x7f
385 n %r10,__PT_INT_CODE(%r11)
386 je sysc_return
387 sll %r10,2
388 l %r1,0(%r10,%r1) # load address of handler routine
389 lr %r2,%r11 # pass pointer to pt_regs
390 basr %r14,%r1 # branch to interrupt-handler
391 j sysc_return
393 #
394 # PER event in supervisor state, must be kprobes
395 #
396 pgm_kprobe:
397 REENABLE_IRQS
398 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
399 l %r1,BASED(.Ldo_per_trap)
400 lr %r2,%r11 # pass pointer to pt_regs
401 basr %r14,%r1 # call do_per_trap
402 j sysc_return
404 #
405 # single stepped system call
406 #
407 pgm_svcper:
408 oi __TI_flags+3(%r12),_TIF_PER_TRAP
409 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
410 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
411 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
413 /*
414 * IO interrupt handler routine
415 */
417 ENTRY(io_int_handler)
418 stck __LC_INT_CLOCK
419 stpt __LC_ASYNC_ENTER_TIMER
420 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
421 l %r12,__LC_THREAD_INFO
422 l %r13,__LC_SVC_NEW_PSW+4
423 lm %r8,%r9,__LC_IO_OLD_PSW
424 tmh %r8,0x0001 # interrupting from user ?
425 jz io_skip
426 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
427 io_skip:
428 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
429 stm %r0,%r7,__PT_R0(%r11)
430 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
431 stm %r8,%r9,__PT_PSW(%r11)
432 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
433 TRACE_IRQS_OFF
434 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
435 io_loop:
436 l %r1,BASED(.Ldo_IRQ)
437 lr %r2,%r11 # pass pointer to pt_regs
438 basr %r14,%r1 # call do_IRQ
439 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR
440 jz io_return
441 tpi 0
442 jz io_return
443 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
444 j io_loop
445 io_return:
446 LOCKDEP_SYS_EXIT
447 TRACE_IRQS_ON
448 io_tif:
449 tm __TI_flags+3(%r12),_TIF_WORK_INT
450 jnz io_work # there is work to do (signals etc.)
451 io_restore:
452 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
453 stpt __LC_EXIT_TIMER
454 lm %r0,%r15,__PT_R0(%r11)
455 lpsw __LC_RETURN_PSW
456 io_done:
458 #
459 # There is work todo, find out in which context we have been interrupted:
460 # 1) if we return to user space we can do all _TIF_WORK_INT work
461 # 2) if we return to kernel code and preemptive scheduling is enabled check
462 # the preemption counter and if it is zero call preempt_schedule_irq
463 # Before any work can be done, a switch to the kernel stack is required.
464 #
465 io_work:
466 tm __PT_PSW+1(%r11),0x01 # returning to user ?
467 jo io_work_user # yes -> do resched & signal
468 #ifdef CONFIG_PREEMPT
469 # check for preemptive scheduling
470 icm %r0,15,__TI_precount(%r12)
471 jnz io_restore # preemption disabled
472 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
473 jno io_restore
474 # switch to kernel stack
475 l %r1,__PT_R15(%r11)
476 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
477 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
478 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
479 la %r11,STACK_FRAME_OVERHEAD(%r1)
480 lr %r15,%r1
481 # TRACE_IRQS_ON already done at io_return, call
482 # TRACE_IRQS_OFF to keep things symmetrical
483 TRACE_IRQS_OFF
484 l %r1,BASED(.Lpreempt_irq)
485 basr %r14,%r1 # call preempt_schedule_irq
486 j io_return
487 #else
488 j io_restore
489 #endif
491 #
492 # Need to do work before returning to userspace, switch to kernel stack
493 #
494 io_work_user:
495 l %r1,__LC_KERNEL_STACK
496 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
497 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
498 la %r11,STACK_FRAME_OVERHEAD(%r1)
499 lr %r15,%r1
501 #
502 # One of the work bits is on. Find out which one.
503 # Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
504 # and _TIF_MCCK_PENDING
505 #
506 io_work_tif:
507 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
508 jo io_mcck_pending
509 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
510 jo io_reschedule
511 tm __TI_flags+3(%r12),_TIF_SIGPENDING
512 jo io_sigpending
513 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
514 jo io_notify_resume
515 j io_return # beware of critical section cleanup
517 #
518 # _TIF_MCCK_PENDING is set, call handler
519 #
520 io_mcck_pending:
521 # TRACE_IRQS_ON already done at io_return
522 l %r1,BASED(.Lhandle_mcck)
523 basr %r14,%r1 # TIF bit will be cleared by handler
524 TRACE_IRQS_OFF
525 j io_return
527 #
528 # _TIF_NEED_RESCHED is set, call schedule
529 #
530 io_reschedule:
531 # TRACE_IRQS_ON already done at io_return
532 l %r1,BASED(.Lschedule)
533 ssm __LC_SVC_NEW_PSW # reenable interrupts
534 basr %r14,%r1 # call scheduler
535 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
536 TRACE_IRQS_OFF
537 j io_return
539 #
540 # _TIF_SIGPENDING is set, call do_signal
541 #
542 io_sigpending:
543 # TRACE_IRQS_ON already done at io_return
544 l %r1,BASED(.Ldo_signal)
545 ssm __LC_SVC_NEW_PSW # reenable interrupts
546 lr %r2,%r11 # pass pointer to pt_regs
547 basr %r14,%r1 # call do_signal
548 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
549 TRACE_IRQS_OFF
550 j io_return
552 #
553 # _TIF_SIGPENDING is set, call do_signal
554 #
555 io_notify_resume:
556 # TRACE_IRQS_ON already done at io_return
557 l %r1,BASED(.Ldo_notify_resume)
558 ssm __LC_SVC_NEW_PSW # reenable interrupts
559 lr %r2,%r11 # pass pointer to pt_regs
560 basr %r14,%r1 # call do_notify_resume
561 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
562 TRACE_IRQS_OFF
563 j io_return
565 /*
566 * External interrupt handler routine
567 */
569 ENTRY(ext_int_handler)
570 stck __LC_INT_CLOCK
571 stpt __LC_ASYNC_ENTER_TIMER
572 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
573 l %r12,__LC_THREAD_INFO
574 l %r13,__LC_SVC_NEW_PSW+4
575 lm %r8,%r9,__LC_EXT_OLD_PSW
576 tmh %r8,0x0001 # interrupting from user ?
577 jz ext_skip
578 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
579 ext_skip:
580 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
581 stm %r0,%r7,__PT_R0(%r11)
582 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
583 stm %r8,%r9,__PT_PSW(%r11)
584 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
585 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
586 TRACE_IRQS_OFF
587 lr %r2,%r11 # pass pointer to pt_regs
588 l %r1,BASED(.Ldo_extint)
589 basr %r14,%r1 # call do_extint
590 j io_return
592 /*
593 * Load idle PSW. The second "half" of this function is in cleanup_idle.
594 */
595 ENTRY(psw_idle)
596 st %r3,__SF_EMPTY(%r15)
597 basr %r1,0
598 la %r1,psw_idle_lpsw+4-.(%r1)
599 st %r1,__SF_EMPTY+4(%r15)
600 oi __SF_EMPTY+4(%r15),0x80
601 stck __CLOCK_IDLE_ENTER(%r2)
602 stpt __TIMER_IDLE_ENTER(%r2)
603 psw_idle_lpsw:
604 lpsw __SF_EMPTY(%r15)
605 br %r14
606 psw_idle_end:
608 __critical_end:
610 /*
611 * Machine check handler routines
612 */
614 ENTRY(mcck_int_handler)
615 stck __LC_MCCK_CLOCK
616 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
617 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
618 l %r12,__LC_THREAD_INFO
619 l %r13,__LC_SVC_NEW_PSW+4
620 lm %r8,%r9,__LC_MCK_OLD_PSW
621 tm __LC_MCCK_CODE,0x80 # system damage?
622 jo mcck_panic # yes -> rest of mcck code invalid
623 la %r14,__LC_CPU_TIMER_SAVE_AREA
624 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
625 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
626 jo 3f
627 la %r14,__LC_SYNC_ENTER_TIMER
628 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
629 jl 0f
630 la %r14,__LC_ASYNC_ENTER_TIMER
631 0: clc 0(8,%r14),__LC_EXIT_TIMER
632 jl 1f
633 la %r14,__LC_EXIT_TIMER
634 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
635 jl 2f
636 la %r14,__LC_LAST_UPDATE_TIMER
637 2: spt 0(%r14)
638 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
639 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
640 jno mcck_panic # no -> skip cleanup critical
641 tm %r8,0x0001 # interrupting from user ?
642 jz mcck_skip
643 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
644 mcck_skip:
645 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
646 stm %r0,%r7,__PT_R0(%r11)
647 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32
648 stm %r8,%r9,__PT_PSW(%r11)
649 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
650 l %r1,BASED(.Ldo_machine_check)
651 lr %r2,%r11 # pass pointer to pt_regs
652 basr %r14,%r1 # call s390_do_machine_check
653 tm __PT_PSW+1(%r11),0x01 # returning to user ?
654 jno mcck_return
655 l %r1,__LC_KERNEL_STACK # switch to kernel stack
656 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
657 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
658 la %r11,STACK_FRAME_OVERHEAD(%r15)
659 lr %r15,%r1
660 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
661 tm __TI_flags+3(%r12),_TIF_MCCK_PENDING
662 jno mcck_return
663 TRACE_IRQS_OFF
664 l %r1,BASED(.Lhandle_mcck)
665 basr %r14,%r1 # call s390_handle_mcck
666 TRACE_IRQS_ON
667 mcck_return:
668 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
669 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
670 jno 0f
671 lm %r0,%r15,__PT_R0(%r11)
672 stpt __LC_EXIT_TIMER
673 lpsw __LC_RETURN_MCCK_PSW
674 0: lm %r0,%r15,__PT_R0(%r11)
675 lpsw __LC_RETURN_MCCK_PSW
677 mcck_panic:
678 l %r14,__LC_PANIC_STACK
679 slr %r14,%r15
680 sra %r14,PAGE_SHIFT
681 jz 0f
682 l %r15,__LC_PANIC_STACK
683 j mcck_skip
684 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
685 j mcck_skip
687 #
688 # PSW restart interrupt handler
689 #
690 ENTRY(restart_int_handler)
691 st %r15,__LC_SAVE_AREA_RESTART
692 l %r15,__LC_RESTART_STACK
693 ahi %r15,-__PT_SIZE # create pt_regs on stack
694 xc 0(__PT_SIZE,%r15),0(%r15)
695 stm %r0,%r14,__PT_R0(%r15)
696 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
697 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
698 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
699 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
700 l %r1,__LC_RESTART_FN # load fn, parm & source cpu
701 l %r2,__LC_RESTART_DATA
702 l %r3,__LC_RESTART_SOURCE
703 ltr %r3,%r3 # test source cpu address
704 jm 1f # negative -> skip source stop
705 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
706 brc 10,0b # wait for status stored
707 1: basr %r14,%r1 # call function
708 stap __SF_EMPTY(%r15) # store cpu address
709 lh %r3,__SF_EMPTY(%r15)
710 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
711 brc 2,2b
712 3: j 3b
714 .section .kprobes.text, "ax"
716 #ifdef CONFIG_CHECK_STACK
717 /*
718 * The synchronous or the asynchronous stack overflowed. We are dead.
719 * No need to properly save the registers, we are going to panic anyway.
720 * Setup a pt_regs so that show_trace can provide a good call trace.
721 */
722 stack_overflow:
723 l %r15,__LC_PANIC_STACK # change to panic stack
724 la %r11,STACK_FRAME_OVERHEAD(%r15)
725 stm %r0,%r7,__PT_R0(%r11)
726 stm %r8,%r9,__PT_PSW(%r11)
727 mvc __PT_R8(32,%r11),0(%r14)
728 l %r1,BASED(1f)
729 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
730 lr %r2,%r11 # pass pointer to pt_regs
731 br %r1 # branch to kernel_stack_overflow
732 1: .long kernel_stack_overflow
733 #endif
735 cleanup_table:
736 .long system_call + 0x80000000
737 .long sysc_do_svc + 0x80000000
738 .long sysc_tif + 0x80000000
739 .long sysc_restore + 0x80000000
740 .long sysc_done + 0x80000000
741 .long io_tif + 0x80000000
742 .long io_restore + 0x80000000
743 .long io_done + 0x80000000
744 .long psw_idle + 0x80000000
745 .long psw_idle_end + 0x80000000
747 cleanup_critical:
748 cl %r9,BASED(cleanup_table) # system_call
749 jl 0f
750 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
751 jl cleanup_system_call
752 cl %r9,BASED(cleanup_table+8) # sysc_tif
753 jl 0f
754 cl %r9,BASED(cleanup_table+12) # sysc_restore
755 jl cleanup_sysc_tif
756 cl %r9,BASED(cleanup_table+16) # sysc_done
757 jl cleanup_sysc_restore
758 cl %r9,BASED(cleanup_table+20) # io_tif
759 jl 0f
760 cl %r9,BASED(cleanup_table+24) # io_restore
761 jl cleanup_io_tif
762 cl %r9,BASED(cleanup_table+28) # io_done
763 jl cleanup_io_restore
764 cl %r9,BASED(cleanup_table+32) # psw_idle
765 jl 0f
766 cl %r9,BASED(cleanup_table+36) # psw_idle_end
767 jl cleanup_idle
768 0: br %r14
770 cleanup_system_call:
771 # check if stpt has been executed
772 cl %r9,BASED(cleanup_system_call_insn)
773 jh 0f
774 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
775 chi %r11,__LC_SAVE_AREA_ASYNC
776 je 0f
777 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
778 0: # check if stm has been executed
779 cl %r9,BASED(cleanup_system_call_insn+4)
780 jh 0f
781 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
782 0: # set up saved registers r12, and r13
783 st %r12,16(%r11) # r12 thread-info pointer
784 st %r13,20(%r11) # r13 literal-pool pointer
785 # check if the user time calculation has been done
786 cl %r9,BASED(cleanup_system_call_insn+8)
787 jh 0f
788 l %r10,__LC_EXIT_TIMER
789 l %r15,__LC_EXIT_TIMER+4
790 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
791 ADD64 %r10,%r15,__LC_USER_TIMER
792 st %r10,__LC_USER_TIMER
793 st %r15,__LC_USER_TIMER+4
794 0: # check if the system time calculation has been done
795 cl %r9,BASED(cleanup_system_call_insn+12)
796 jh 0f
797 l %r10,__LC_LAST_UPDATE_TIMER
798 l %r15,__LC_LAST_UPDATE_TIMER+4
799 SUB64 %r10,%r15,__LC_EXIT_TIMER
800 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
801 st %r10,__LC_SYSTEM_TIMER
802 st %r15,__LC_SYSTEM_TIMER+4
803 0: # update accounting time stamp
804 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
805 # set up saved register 11
806 l %r15,__LC_KERNEL_STACK
807 la %r9,STACK_FRAME_OVERHEAD(%r15)
808 st %r9,12(%r11) # r11 pt_regs pointer
809 # fill pt_regs
810 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
811 stm %r0,%r7,__PT_R0(%r9)
812 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
813 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
814 # setup saved register 15
815 st %r15,28(%r11) # r15 stack pointer
816 # set new psw address and exit
817 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
818 br %r14
819 cleanup_system_call_insn:
820 .long system_call + 0x80000000
821 .long sysc_stm + 0x80000000
822 .long sysc_vtime + 0x80000000 + 36
823 .long sysc_vtime + 0x80000000 + 76
825 cleanup_sysc_tif:
826 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
827 br %r14
829 cleanup_sysc_restore:
830 cl %r9,BASED(cleanup_sysc_restore_insn)
831 jhe 0f
832 l %r9,12(%r11) # get saved pointer to pt_regs
833 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
834 mvc 0(32,%r11),__PT_R8(%r9)
835 lm %r0,%r7,__PT_R0(%r9)
836 0: lm %r8,%r9,__LC_RETURN_PSW
837 br %r14
838 cleanup_sysc_restore_insn:
839 .long sysc_done - 4 + 0x80000000
841 cleanup_io_tif:
842 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
843 br %r14
845 cleanup_io_restore:
846 cl %r9,BASED(cleanup_io_restore_insn)
847 jhe 0f
848 l %r9,12(%r11) # get saved r11 pointer to pt_regs
849 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
850 mvc 0(32,%r11),__PT_R8(%r9)
851 lm %r0,%r7,__PT_R0(%r9)
852 0: lm %r8,%r9,__LC_RETURN_PSW
853 br %r14
854 cleanup_io_restore_insn:
855 .long io_done - 4 + 0x80000000
857 cleanup_idle:
858 # copy interrupt clock & cpu timer
859 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
860 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
861 chi %r11,__LC_SAVE_AREA_ASYNC
862 je 0f
863 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
864 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
865 0: # check if stck has been executed
866 cl %r9,BASED(cleanup_idle_insn)
867 jhe 1f
868 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
869 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3)
870 1: # account system time going idle
871 lm %r9,%r10,__LC_STEAL_TIMER
872 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2)
873 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
874 stm %r9,%r10,__LC_STEAL_TIMER
875 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
876 lm %r9,%r10,__LC_SYSTEM_TIMER
877 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
878 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2)
879 stm %r9,%r10,__LC_SYSTEM_TIMER
880 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
881 # prepare return psw
882 n %r8,BASED(cleanup_idle_wait) # clear wait state bit
883 l %r9,24(%r11) # return from psw_idle
884 br %r14
885 cleanup_idle_insn:
886 .long psw_idle_lpsw + 0x80000000
887 cleanup_idle_wait:
888 .long 0xfffdffff
890 /*
891 * Integer constants
892 */
893 .align 4
894 .Lnr_syscalls:
895 .long NR_syscalls
896 .Lvtimer_max:
897 .quad 0x7fffffffffffffff
899 /*
900 * Symbol constants
901 */
902 .Ldo_machine_check: .long s390_do_machine_check
903 .Lhandle_mcck: .long s390_handle_mcck
904 .Ldo_IRQ: .long do_IRQ
905 .Ldo_extint: .long do_extint
906 .Ldo_signal: .long do_signal
907 .Ldo_notify_resume: .long do_notify_resume
908 .Ldo_per_trap: .long do_per_trap
909 .Ljump_table: .long pgm_check_table
910 .Lschedule: .long schedule
911 #ifdef CONFIG_PREEMPT
912 .Lpreempt_irq: .long preempt_schedule_irq
913 #endif
914 .Ltrace_enter: .long do_syscall_trace_enter
915 .Ltrace_exit: .long do_syscall_trace_exit
916 .Lschedule_tail: .long schedule_tail
917 .Lsysc_per: .long sysc_per + 0x80000000
918 #ifdef CONFIG_TRACE_IRQFLAGS
919 .Lhardirqs_on: .long trace_hardirqs_on_caller
920 .Lhardirqs_off: .long trace_hardirqs_off_caller
921 #endif
922 #ifdef CONFIG_LOCKDEP
923 .Llockdep_sys_exit: .long lockdep_sys_exit
924 #endif
925 .Lcritical_start: .long __critical_start + 0x80000000
926 .Lcritical_length: .long __critical_end - __critical_start
928 .section .rodata, "a"
929 #define SYSCALL(esa,esame,emu) .long esa
930 .globl sys_call_table
931 sys_call_table:
932 #include "syscalls.S"
933 #undef SYSCALL