| blob | 70203265196fa704fcc14723ce1df68f53ead4d6 |
1 /*
2 * S390 low-level entry points.
3 *
4 * Copyright IBM Corp. 1999, 2012
5 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
6 * Hartmut Penner (hp@de.ibm.com),
7 * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8 * Heiko Carstens <heiko.carstens@de.ibm.com>
9 */
11 #include <linux/init.h>
12 #include <linux/linkage.h>
13 #include <asm/processor.h>
14 #include <asm/cache.h>
15 #include <asm/errno.h>
16 #include <asm/ptrace.h>
17 #include <asm/thread_info.h>
18 #include <asm/asm-offsets.h>
19 #include <asm/unistd.h>
20 #include <asm/page.h>
21 #include <asm/sigp.h>
22 #include <asm/irq.h>
24 __PT_R0 = __PT_GPRS
25 __PT_R1 = __PT_GPRS + 4
26 __PT_R2 = __PT_GPRS + 8
27 __PT_R3 = __PT_GPRS + 12
28 __PT_R4 = __PT_GPRS + 16
29 __PT_R5 = __PT_GPRS + 20
30 __PT_R6 = __PT_GPRS + 24
31 __PT_R7 = __PT_GPRS + 28
32 __PT_R8 = __PT_GPRS + 32
33 __PT_R9 = __PT_GPRS + 36
34 __PT_R10 = __PT_GPRS + 40
35 __PT_R11 = __PT_GPRS + 44
36 __PT_R12 = __PT_GPRS + 48
37 __PT_R13 = __PT_GPRS + 524
38 __PT_R14 = __PT_GPRS + 56
39 __PT_R15 = __PT_GPRS + 60
41 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
42 STACK_SIZE = 1 << STACK_SHIFT
43 STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
45 _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED)
46 _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
47 _TIF_SYSCALL_TRACEPOINT)
48 _CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE)
49 _PIF_WORK = (_PIF_PER_TRAP)
51 #define BASED(name) name-system_call(%r13)
53 .macro TRACE_IRQS_ON
54 #ifdef CONFIG_TRACE_IRQFLAGS
55 basr %r2,%r0
56 l %r1,BASED(.Lhardirqs_on)
57 basr %r14,%r1 # call trace_hardirqs_on_caller
58 #endif
59 .endm
61 .macro TRACE_IRQS_OFF
62 #ifdef CONFIG_TRACE_IRQFLAGS
63 basr %r2,%r0
64 l %r1,BASED(.Lhardirqs_off)
65 basr %r14,%r1 # call trace_hardirqs_off_caller
66 #endif
67 .endm
69 .macro LOCKDEP_SYS_EXIT
70 #ifdef CONFIG_LOCKDEP
71 tm __PT_PSW+1(%r11),0x01 # returning to user ?
72 jz .+10
73 l %r1,BASED(.Llockdep_sys_exit)
74 basr %r14,%r1 # call lockdep_sys_exit
75 #endif
76 .endm
78 .macro CHECK_STACK stacksize,savearea
79 #ifdef CONFIG_CHECK_STACK
80 tml %r15,\stacksize - CONFIG_STACK_GUARD
81 la %r14,\savearea
82 jz stack_overflow
83 #endif
84 .endm
86 .macro SWITCH_ASYNC savearea,stack,shift
87 tmh %r8,0x0001 # interrupting from user ?
88 jnz 1f
89 lr %r14,%r9
90 sl %r14,BASED(.Lcritical_start)
91 cl %r14,BASED(.Lcritical_length)
92 jhe 0f
93 la %r11,\savearea # inside critical section, do cleanup
94 bras %r14,cleanup_critical
95 tmh %r8,0x0001 # retest problem state after cleanup
96 jnz 1f
97 0: l %r14,\stack # are we already on the target stack?
98 slr %r14,%r15
99 sra %r14,\shift
100 jnz 1f
101 CHECK_STACK 1<<\shift,\savearea
102 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
103 j 2f
104 1: l %r15,\stack # load target stack
105 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
106 .endm
108 .macro ADD64 high,low,timer
109 al \high,\timer
110 al \low,4+\timer
111 brc 12,.+8
112 ahi \high,1
113 .endm
115 .macro SUB64 high,low,timer
116 sl \high,\timer
117 sl \low,4+\timer
118 brc 3,.+8
119 ahi \high,-1
120 .endm
122 .macro UPDATE_VTIME high,low,enter_timer
123 lm \high,\low,__LC_EXIT_TIMER
124 SUB64 \high,\low,\enter_timer
125 ADD64 \high,\low,__LC_USER_TIMER
126 stm \high,\low,__LC_USER_TIMER
127 lm \high,\low,__LC_LAST_UPDATE_TIMER
128 SUB64 \high,\low,__LC_EXIT_TIMER
129 ADD64 \high,\low,__LC_SYSTEM_TIMER
130 stm \high,\low,__LC_SYSTEM_TIMER
131 mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer
132 .endm
134 .macro REENABLE_IRQS
135 st %r8,__LC_RETURN_PSW
136 ni __LC_RETURN_PSW,0xbf
137 ssm __LC_RETURN_PSW
138 .endm
140 .section .kprobes.text, "ax"
142 /*
143 * Scheduler resume function, called by switch_to
144 * gpr2 = (task_struct *) prev
145 * gpr3 = (task_struct *) next
146 * Returns:
147 * gpr2 = prev
148 */
149 ENTRY(__switch_to)
150 stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
151 st %r15,__THREAD_ksp(%r2) # store kernel stack of prev
152 l %r4,__THREAD_info(%r2) # get thread_info of prev
153 l %r5,__THREAD_info(%r3) # get thread_info of next
154 lr %r15,%r5
155 ahi %r15,STACK_INIT # end of kernel stack of next
156 st %r3,__LC_CURRENT # store task struct of next
157 st %r5,__LC_THREAD_INFO # store thread info of next
158 st %r15,__LC_KERNEL_STACK # store end of kernel stack
159 lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4
160 mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
161 l %r15,__THREAD_ksp(%r3) # load kernel stack of next
162 lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
163 br %r14
165 __critical_start:
166 /*
167 * SVC interrupt handler routine. System calls are synchronous events and
168 * are executed with interrupts enabled.
169 */
171 ENTRY(system_call)
172 stpt __LC_SYNC_ENTER_TIMER
173 sysc_stm:
174 stm %r8,%r15,__LC_SAVE_AREA_SYNC
175 l %r12,__LC_THREAD_INFO
176 l %r13,__LC_SVC_NEW_PSW+4
177 lhi %r14,_PIF_SYSCALL
178 sysc_per:
179 l %r15,__LC_KERNEL_STACK
180 la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
181 sysc_vtime:
182 UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
183 stm %r0,%r7,__PT_R0(%r11)
184 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
185 mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW
186 mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC
187 st %r14,__PT_FLAGS(%r11)
188 sysc_do_svc:
189 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
190 lh %r8,__PT_INT_CODE+2(%r11)
191 sla %r8,2 # shift and test for svc0
192 jnz sysc_nr_ok
193 # svc 0: system call number in %r1
194 cl %r1,BASED(.Lnr_syscalls)
195 jnl sysc_nr_ok
196 sth %r1,__PT_INT_CODE+2(%r11)
197 lr %r8,%r1
198 sla %r8,2
199 sysc_nr_ok:
200 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
201 st %r2,__PT_ORIG_GPR2(%r11)
202 st %r7,STACK_FRAME_OVERHEAD(%r15)
203 l %r9,0(%r8,%r10) # get system call addr.
204 tm __TI_flags+3(%r12),_TIF_TRACE
205 jnz sysc_tracesys
206 basr %r14,%r9 # call sys_xxxx
207 st %r2,__PT_R2(%r11) # store return value
209 sysc_return:
210 LOCKDEP_SYS_EXIT
211 sysc_tif:
212 tm __PT_PSW+1(%r11),0x01 # returning to user ?
213 jno sysc_restore
214 tm __PT_FLAGS+3(%r11),_PIF_WORK
215 jnz sysc_work
216 tm __TI_flags+3(%r12),_TIF_WORK
217 jnz sysc_work # check for thread work
218 tm __LC_CPU_FLAGS+3,_CIF_WORK
219 jnz sysc_work
220 sysc_restore:
221 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
222 stpt __LC_EXIT_TIMER
223 lm %r0,%r15,__PT_R0(%r11)
224 lpsw __LC_RETURN_PSW
225 sysc_done:
227 #
228 # One of the work bits is on. Find out which one.
229 #
230 sysc_work:
231 tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING
232 jo sysc_mcck_pending
233 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
234 jo sysc_reschedule
235 tm __PT_FLAGS+3(%r11),_PIF_PER_TRAP
236 jo sysc_singlestep
237 tm __TI_flags+3(%r12),_TIF_SIGPENDING
238 jo sysc_sigpending
239 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
240 jo sysc_notify_resume
241 tm __LC_CPU_FLAGS+3,_CIF_ASCE
242 jo sysc_uaccess
243 j sysc_return # beware of critical section cleanup
245 #
246 # _TIF_NEED_RESCHED is set, call schedule
247 #
248 sysc_reschedule:
249 l %r1,BASED(.Lschedule)
250 la %r14,BASED(sysc_return)
251 br %r1 # call schedule
253 #
254 # _CIF_MCCK_PENDING is set, call handler
255 #
256 sysc_mcck_pending:
257 l %r1,BASED(.Lhandle_mcck)
258 la %r14,BASED(sysc_return)
259 br %r1 # TIF bit will be cleared by handler
261 #
262 # _CIF_ASCE is set, load user space asce
263 #
264 sysc_uaccess:
265 ni __LC_CPU_FLAGS+3,255-_CIF_ASCE
266 lctl %c1,%c1,__LC_USER_ASCE # load primary asce
267 j sysc_return
269 #
270 # _TIF_SIGPENDING is set, call do_signal
271 #
272 sysc_sigpending:
273 lr %r2,%r11 # pass pointer to pt_regs
274 l %r1,BASED(.Ldo_signal)
275 basr %r14,%r1 # call do_signal
276 tm __PT_FLAGS+3(%r11),_PIF_SYSCALL
277 jno sysc_return
278 lm %r2,%r7,__PT_R2(%r11) # load svc arguments
279 l %r10,__TI_sysc_table(%r12) # 31 bit system call table
280 xr %r8,%r8 # svc 0 returns -ENOSYS
281 clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
282 jnl sysc_nr_ok # invalid svc number -> do svc 0
283 lh %r8,__PT_INT_CODE+2(%r11) # load new svc number
284 sla %r8,2
285 j sysc_nr_ok # restart svc
287 #
288 # _TIF_NOTIFY_RESUME is set, call do_notify_resume
289 #
290 sysc_notify_resume:
291 lr %r2,%r11 # pass pointer to pt_regs
292 l %r1,BASED(.Ldo_notify_resume)
293 la %r14,BASED(sysc_return)
294 br %r1 # call do_notify_resume
296 #
297 # _PIF_PER_TRAP is set, call do_per_trap
298 #
299 sysc_singlestep:
300 ni __PT_FLAGS+3(%r11),255-_PIF_PER_TRAP
301 lr %r2,%r11 # pass pointer to pt_regs
302 l %r1,BASED(.Ldo_per_trap)
303 la %r14,BASED(sysc_return)
304 br %r1 # call do_per_trap
306 #
307 # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
308 # and after the system call
309 #
310 sysc_tracesys:
311 l %r1,BASED(.Ltrace_enter)
312 lr %r2,%r11 # pass pointer to pt_regs
313 la %r3,0
314 xr %r0,%r0
315 icm %r0,3,__PT_INT_CODE+2(%r11)
316 st %r0,__PT_R2(%r11)
317 basr %r14,%r1 # call do_syscall_trace_enter
318 cl %r2,BASED(.Lnr_syscalls)
319 jnl sysc_tracenogo
320 lr %r8,%r2
321 sll %r8,2
322 l %r9,0(%r8,%r10)
323 sysc_tracego:
324 lm %r3,%r7,__PT_R3(%r11)
325 st %r7,STACK_FRAME_OVERHEAD(%r15)
326 l %r2,__PT_ORIG_GPR2(%r11)
327 basr %r14,%r9 # call sys_xxx
328 st %r2,__PT_R2(%r11) # store return value
329 sysc_tracenogo:
330 tm __TI_flags+3(%r12),_TIF_TRACE
331 jz sysc_return
332 l %r1,BASED(.Ltrace_exit)
333 lr %r2,%r11 # pass pointer to pt_regs
334 la %r14,BASED(sysc_return)
335 br %r1 # call do_syscall_trace_exit
337 #
338 # a new process exits the kernel with ret_from_fork
339 #
340 ENTRY(ret_from_fork)
341 la %r11,STACK_FRAME_OVERHEAD(%r15)
342 l %r12,__LC_THREAD_INFO
343 l %r13,__LC_SVC_NEW_PSW+4
344 l %r1,BASED(.Lschedule_tail)
345 basr %r14,%r1 # call schedule_tail
346 TRACE_IRQS_ON
347 ssm __LC_SVC_NEW_PSW # reenable interrupts
348 tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ?
349 jne sysc_tracenogo
350 # it's a kernel thread
351 lm %r9,%r10,__PT_R9(%r11) # load gprs
352 ENTRY(kernel_thread_starter)
353 la %r2,0(%r10)
354 basr %r14,%r9
355 j sysc_tracenogo
357 /*
358 * Program check handler routine
359 */
361 ENTRY(pgm_check_handler)
362 stpt __LC_SYNC_ENTER_TIMER
363 stm %r8,%r15,__LC_SAVE_AREA_SYNC
364 l %r12,__LC_THREAD_INFO
365 l %r13,__LC_SVC_NEW_PSW+4
366 lm %r8,%r9,__LC_PGM_OLD_PSW
367 tmh %r8,0x0001 # test problem state bit
368 jnz 1f # -> fault in user space
369 tmh %r8,0x4000 # PER bit set in old PSW ?
370 jnz 0f # -> enabled, can't be a double fault
371 tm __LC_PGM_ILC+3,0x80 # check for per exception
372 jnz pgm_svcper # -> single stepped svc
373 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
374 ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
375 j 2f
376 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
377 l %r15,__LC_KERNEL_STACK
378 2: la %r11,STACK_FRAME_OVERHEAD(%r15)
379 stm %r0,%r7,__PT_R0(%r11)
380 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
381 stm %r8,%r9,__PT_PSW(%r11)
382 mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC
383 mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
384 xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11)
385 tm __LC_PGM_ILC+3,0x80 # check for per exception
386 jz 0f
387 l %r1,__TI_task(%r12)
388 tmh %r8,0x0001 # kernel per event ?
389 jz pgm_kprobe
390 oi __PT_FLAGS+3(%r11),_PIF_PER_TRAP
391 mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS
392 mvc __THREAD_per_cause(2,%r1),__LC_PER_CODE
393 mvc __THREAD_per_paid(1,%r1),__LC_PER_ACCESS_ID
394 0: REENABLE_IRQS
395 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
396 l %r1,BASED(.Ljump_table)
397 la %r10,0x7f
398 n %r10,__PT_INT_CODE(%r11)
399 je sysc_return
400 sll %r10,2
401 l %r1,0(%r10,%r1) # load address of handler routine
402 lr %r2,%r11 # pass pointer to pt_regs
403 basr %r14,%r1 # branch to interrupt-handler
404 j sysc_return
406 #
407 # PER event in supervisor state, must be kprobes
408 #
409 pgm_kprobe:
410 REENABLE_IRQS
411 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
412 l %r1,BASED(.Ldo_per_trap)
413 lr %r2,%r11 # pass pointer to pt_regs
414 basr %r14,%r1 # call do_per_trap
415 j sysc_return
417 #
418 # single stepped system call
419 #
420 pgm_svcper:
421 mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
422 mvc __LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
423 lhi %r14,_PIF_SYSCALL | _PIF_PER_TRAP
424 lpsw __LC_RETURN_PSW # branch to sysc_per and enable irqs
426 /*
427 * IO interrupt handler routine
428 */
430 ENTRY(io_int_handler)
431 stck __LC_INT_CLOCK
432 stpt __LC_ASYNC_ENTER_TIMER
433 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
434 l %r12,__LC_THREAD_INFO
435 l %r13,__LC_SVC_NEW_PSW+4
436 lm %r8,%r9,__LC_IO_OLD_PSW
437 tmh %r8,0x0001 # interrupting from user ?
438 jz io_skip
439 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
440 io_skip:
441 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
442 stm %r0,%r7,__PT_R0(%r11)
443 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
444 stm %r8,%r9,__PT_PSW(%r11)
445 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
446 xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11)
447 TRACE_IRQS_OFF
448 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
449 io_loop:
450 l %r1,BASED(.Ldo_IRQ)
451 lr %r2,%r11 # pass pointer to pt_regs
452 lhi %r3,IO_INTERRUPT
453 tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ?
454 jz io_call
455 lhi %r3,THIN_INTERRUPT
456 io_call:
457 basr %r14,%r1 # call do_IRQ
458 tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR
459 jz io_return
460 tpi 0
461 jz io_return
462 mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
463 j io_loop
464 io_return:
465 LOCKDEP_SYS_EXIT
466 TRACE_IRQS_ON
467 io_tif:
468 tm __TI_flags+3(%r12),_TIF_WORK
469 jnz io_work # there is work to do (signals etc.)
470 tm __LC_CPU_FLAGS+3,_CIF_WORK
471 jnz io_work
472 io_restore:
473 mvc __LC_RETURN_PSW(8),__PT_PSW(%r11)
474 stpt __LC_EXIT_TIMER
475 lm %r0,%r15,__PT_R0(%r11)
476 lpsw __LC_RETURN_PSW
477 io_done:
479 #
480 # There is work todo, find out in which context we have been interrupted:
481 # 1) if we return to user space we can do all _TIF_WORK work
482 # 2) if we return to kernel code and preemptive scheduling is enabled check
483 # the preemption counter and if it is zero call preempt_schedule_irq
484 # Before any work can be done, a switch to the kernel stack is required.
485 #
486 io_work:
487 tm __PT_PSW+1(%r11),0x01 # returning to user ?
488 jo io_work_user # yes -> do resched & signal
489 #ifdef CONFIG_PREEMPT
490 # check for preemptive scheduling
491 icm %r0,15,__TI_precount(%r12)
492 jnz io_restore # preemption disabled
493 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
494 jno io_restore
495 # switch to kernel stack
496 l %r1,__PT_R15(%r11)
497 ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
498 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
499 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
500 la %r11,STACK_FRAME_OVERHEAD(%r1)
501 lr %r15,%r1
502 # TRACE_IRQS_ON already done at io_return, call
503 # TRACE_IRQS_OFF to keep things symmetrical
504 TRACE_IRQS_OFF
505 l %r1,BASED(.Lpreempt_irq)
506 basr %r14,%r1 # call preempt_schedule_irq
507 j io_return
508 #else
509 j io_restore
510 #endif
512 #
513 # Need to do work before returning to userspace, switch to kernel stack
514 #
515 io_work_user:
516 l %r1,__LC_KERNEL_STACK
517 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
518 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
519 la %r11,STACK_FRAME_OVERHEAD(%r1)
520 lr %r15,%r1
522 #
523 # One of the work bits is on. Find out which one.
524 #
525 io_work_tif:
526 tm __LC_CPU_FLAGS+3(%r12),_CIF_MCCK_PENDING
527 jo io_mcck_pending
528 tm __TI_flags+3(%r12),_TIF_NEED_RESCHED
529 jo io_reschedule
530 tm __TI_flags+3(%r12),_TIF_SIGPENDING
531 jo io_sigpending
532 tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME
533 jo io_notify_resume
534 tm __LC_CPU_FLAGS+3,_CIF_ASCE
535 jo io_uaccess
536 j io_return # beware of critical section cleanup
538 #
539 # _CIF_MCCK_PENDING is set, call handler
540 #
541 io_mcck_pending:
542 # TRACE_IRQS_ON already done at io_return
543 l %r1,BASED(.Lhandle_mcck)
544 basr %r14,%r1 # TIF bit will be cleared by handler
545 TRACE_IRQS_OFF
546 j io_return
548 #
549 # _CIF_ASCE is set, load user space asce
550 #
551 io_uaccess:
552 ni __LC_CPU_FLAGS+3,255-_CIF_ASCE
553 lctl %c1,%c1,__LC_USER_ASCE # load primary asce
554 j io_return
556 #
557 # _TIF_NEED_RESCHED is set, call schedule
558 #
559 io_reschedule:
560 # TRACE_IRQS_ON already done at io_return
561 l %r1,BASED(.Lschedule)
562 ssm __LC_SVC_NEW_PSW # reenable interrupts
563 basr %r14,%r1 # call scheduler
564 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
565 TRACE_IRQS_OFF
566 j io_return
568 #
569 # _TIF_SIGPENDING is set, call do_signal
570 #
571 io_sigpending:
572 # TRACE_IRQS_ON already done at io_return
573 l %r1,BASED(.Ldo_signal)
574 ssm __LC_SVC_NEW_PSW # reenable interrupts
575 lr %r2,%r11 # pass pointer to pt_regs
576 basr %r14,%r1 # call do_signal
577 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
578 TRACE_IRQS_OFF
579 j io_return
581 #
582 # _TIF_SIGPENDING is set, call do_signal
583 #
584 io_notify_resume:
585 # TRACE_IRQS_ON already done at io_return
586 l %r1,BASED(.Ldo_notify_resume)
587 ssm __LC_SVC_NEW_PSW # reenable interrupts
588 lr %r2,%r11 # pass pointer to pt_regs
589 basr %r14,%r1 # call do_notify_resume
590 ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts
591 TRACE_IRQS_OFF
592 j io_return
594 /*
595 * External interrupt handler routine
596 */
598 ENTRY(ext_int_handler)
599 stck __LC_INT_CLOCK
600 stpt __LC_ASYNC_ENTER_TIMER
601 stm %r8,%r15,__LC_SAVE_AREA_ASYNC
602 l %r12,__LC_THREAD_INFO
603 l %r13,__LC_SVC_NEW_PSW+4
604 lm %r8,%r9,__LC_EXT_OLD_PSW
605 tmh %r8,0x0001 # interrupting from user ?
606 jz ext_skip
607 UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
608 ext_skip:
609 SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
610 stm %r0,%r7,__PT_R0(%r11)
611 mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
612 stm %r8,%r9,__PT_PSW(%r11)
613 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
614 mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
615 xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11)
616 TRACE_IRQS_OFF
617 l %r1,BASED(.Ldo_IRQ)
618 lr %r2,%r11 # pass pointer to pt_regs
619 lhi %r3,EXT_INTERRUPT
620 basr %r14,%r1 # call do_IRQ
621 j io_return
623 /*
624 * Load idle PSW. The second "half" of this function is in cleanup_idle.
625 */
626 ENTRY(psw_idle)
627 st %r3,__SF_EMPTY(%r15)
628 basr %r1,0
629 la %r1,psw_idle_lpsw+4-.(%r1)
630 st %r1,__SF_EMPTY+4(%r15)
631 oi __SF_EMPTY+4(%r15),0x80
632 stck __CLOCK_IDLE_ENTER(%r2)
633 stpt __TIMER_IDLE_ENTER(%r2)
634 psw_idle_lpsw:
635 lpsw __SF_EMPTY(%r15)
636 br %r14
637 psw_idle_end:
639 __critical_end:
641 /*
642 * Machine check handler routines
643 */
645 ENTRY(mcck_int_handler)
646 stck __LC_MCCK_CLOCK
647 spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer
648 lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs
649 l %r12,__LC_THREAD_INFO
650 l %r13,__LC_SVC_NEW_PSW+4
651 lm %r8,%r9,__LC_MCK_OLD_PSW
652 tm __LC_MCCK_CODE,0x80 # system damage?
653 jo mcck_panic # yes -> rest of mcck code invalid
654 la %r14,__LC_CPU_TIMER_SAVE_AREA
655 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
656 tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid?
657 jo 3f
658 la %r14,__LC_SYNC_ENTER_TIMER
659 clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER
660 jl 0f
661 la %r14,__LC_ASYNC_ENTER_TIMER
662 0: clc 0(8,%r14),__LC_EXIT_TIMER
663 jl 1f
664 la %r14,__LC_EXIT_TIMER
665 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
666 jl 2f
667 la %r14,__LC_LAST_UPDATE_TIMER
668 2: spt 0(%r14)
669 mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
670 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid?
671 jno mcck_panic # no -> skip cleanup critical
672 tm %r8,0x0001 # interrupting from user ?
673 jz mcck_skip
674 UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
675 mcck_skip:
676 SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
677 stm %r0,%r7,__PT_R0(%r11)
678 mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32
679 stm %r8,%r9,__PT_PSW(%r11)
680 xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11)
681 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
682 l %r1,BASED(.Ldo_machine_check)
683 lr %r2,%r11 # pass pointer to pt_regs
684 basr %r14,%r1 # call s390_do_machine_check
685 tm __PT_PSW+1(%r11),0x01 # returning to user ?
686 jno mcck_return
687 l %r1,__LC_KERNEL_STACK # switch to kernel stack
688 mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
689 xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
690 la %r11,STACK_FRAME_OVERHEAD(%r15)
691 lr %r15,%r1
692 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off
693 tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING
694 jno mcck_return
695 TRACE_IRQS_OFF
696 l %r1,BASED(.Lhandle_mcck)
697 basr %r14,%r1 # call s390_handle_mcck
698 TRACE_IRQS_ON
699 mcck_return:
700 mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
701 tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
702 jno 0f
703 lm %r0,%r15,__PT_R0(%r11)
704 stpt __LC_EXIT_TIMER
705 lpsw __LC_RETURN_MCCK_PSW
706 0: lm %r0,%r15,__PT_R0(%r11)
707 lpsw __LC_RETURN_MCCK_PSW
709 mcck_panic:
710 l %r14,__LC_PANIC_STACK
711 slr %r14,%r15
712 sra %r14,PAGE_SHIFT
713 jz 0f
714 l %r15,__LC_PANIC_STACK
715 j mcck_skip
716 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
717 j mcck_skip
719 #
720 # PSW restart interrupt handler
721 #
722 ENTRY(restart_int_handler)
723 st %r15,__LC_SAVE_AREA_RESTART
724 l %r15,__LC_RESTART_STACK
725 ahi %r15,-__PT_SIZE # create pt_regs on stack
726 xc 0(__PT_SIZE,%r15),0(%r15)
727 stm %r0,%r14,__PT_R0(%r15)
728 mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
729 mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
730 ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack
731 xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
732 l %r1,__LC_RESTART_FN # load fn, parm & source cpu
733 l %r2,__LC_RESTART_DATA
734 l %r3,__LC_RESTART_SOURCE
735 ltr %r3,%r3 # test source cpu address
736 jm 1f # negative -> skip source stop
737 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
738 brc 10,0b # wait for status stored
739 1: basr %r14,%r1 # call function
740 stap __SF_EMPTY(%r15) # store cpu address
741 lh %r3,__SF_EMPTY(%r15)
742 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
743 brc 2,2b
744 3: j 3b
746 .section .kprobes.text, "ax"
748 #ifdef CONFIG_CHECK_STACK
749 /*
750 * The synchronous or the asynchronous stack overflowed. We are dead.
751 * No need to properly save the registers, we are going to panic anyway.
752 * Setup a pt_regs so that show_trace can provide a good call trace.
753 */
754 stack_overflow:
755 l %r15,__LC_PANIC_STACK # change to panic stack
756 la %r11,STACK_FRAME_OVERHEAD(%r15)
757 stm %r0,%r7,__PT_R0(%r11)
758 stm %r8,%r9,__PT_PSW(%r11)
759 mvc __PT_R8(32,%r11),0(%r14)
760 l %r1,BASED(1f)
761 xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
762 lr %r2,%r11 # pass pointer to pt_regs
763 br %r1 # branch to kernel_stack_overflow
764 1: .long kernel_stack_overflow
765 #endif
767 cleanup_table:
768 .long system_call + 0x80000000
769 .long sysc_do_svc + 0x80000000
770 .long sysc_tif + 0x80000000
771 .long sysc_restore + 0x80000000
772 .long sysc_done + 0x80000000
773 .long io_tif + 0x80000000
774 .long io_restore + 0x80000000
775 .long io_done + 0x80000000
776 .long psw_idle + 0x80000000
777 .long psw_idle_end + 0x80000000
779 cleanup_critical:
780 cl %r9,BASED(cleanup_table) # system_call
781 jl 0f
782 cl %r9,BASED(cleanup_table+4) # sysc_do_svc
783 jl cleanup_system_call
784 cl %r9,BASED(cleanup_table+8) # sysc_tif
785 jl 0f
786 cl %r9,BASED(cleanup_table+12) # sysc_restore
787 jl cleanup_sysc_tif
788 cl %r9,BASED(cleanup_table+16) # sysc_done
789 jl cleanup_sysc_restore
790 cl %r9,BASED(cleanup_table+20) # io_tif
791 jl 0f
792 cl %r9,BASED(cleanup_table+24) # io_restore
793 jl cleanup_io_tif
794 cl %r9,BASED(cleanup_table+28) # io_done
795 jl cleanup_io_restore
796 cl %r9,BASED(cleanup_table+32) # psw_idle
797 jl 0f
798 cl %r9,BASED(cleanup_table+36) # psw_idle_end
799 jl cleanup_idle
800 0: br %r14
802 cleanup_system_call:
803 # check if stpt has been executed
804 cl %r9,BASED(cleanup_system_call_insn)
805 jh 0f
806 mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
807 chi %r11,__LC_SAVE_AREA_ASYNC
808 je 0f
809 mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
810 0: # check if stm has been executed
811 cl %r9,BASED(cleanup_system_call_insn+4)
812 jh 0f
813 mvc __LC_SAVE_AREA_SYNC(32),0(%r11)
814 0: # set up saved registers r12, and r13
815 st %r12,16(%r11) # r12 thread-info pointer
816 st %r13,20(%r11) # r13 literal-pool pointer
817 # check if the user time calculation has been done
818 cl %r9,BASED(cleanup_system_call_insn+8)
819 jh 0f
820 l %r10,__LC_EXIT_TIMER
821 l %r15,__LC_EXIT_TIMER+4
822 SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER
823 ADD64 %r10,%r15,__LC_USER_TIMER
824 st %r10,__LC_USER_TIMER
825 st %r15,__LC_USER_TIMER+4
826 0: # check if the system time calculation has been done
827 cl %r9,BASED(cleanup_system_call_insn+12)
828 jh 0f
829 l %r10,__LC_LAST_UPDATE_TIMER
830 l %r15,__LC_LAST_UPDATE_TIMER+4
831 SUB64 %r10,%r15,__LC_EXIT_TIMER
832 ADD64 %r10,%r15,__LC_SYSTEM_TIMER
833 st %r10,__LC_SYSTEM_TIMER
834 st %r15,__LC_SYSTEM_TIMER+4
835 0: # update accounting time stamp
836 mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
837 # set up saved register 11
838 l %r15,__LC_KERNEL_STACK
839 la %r9,STACK_FRAME_OVERHEAD(%r15)
840 st %r9,12(%r11) # r11 pt_regs pointer
841 # fill pt_regs
842 mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
843 stm %r0,%r7,__PT_R0(%r9)
844 mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
845 mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
846 xc __PT_FLAGS(4,%r9),__PT_FLAGS(%r9)
847 mvi __PT_FLAGS+3(%r9),_PIF_SYSCALL
848 # setup saved register 15
849 st %r15,28(%r11) # r15 stack pointer
850 # set new psw address and exit
851 l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000
852 br %r14
853 cleanup_system_call_insn:
854 .long system_call + 0x80000000
855 .long sysc_stm + 0x80000000
856 .long sysc_vtime + 0x80000000 + 36
857 .long sysc_vtime + 0x80000000 + 76
859 cleanup_sysc_tif:
860 l %r9,BASED(cleanup_table+8) # sysc_tif + 0x80000000
861 br %r14
863 cleanup_sysc_restore:
864 cl %r9,BASED(cleanup_sysc_restore_insn)
865 jhe 0f
866 l %r9,12(%r11) # get saved pointer to pt_regs
867 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
868 mvc 0(32,%r11),__PT_R8(%r9)
869 lm %r0,%r7,__PT_R0(%r9)
870 0: lm %r8,%r9,__LC_RETURN_PSW
871 br %r14
872 cleanup_sysc_restore_insn:
873 .long sysc_done - 4 + 0x80000000
875 cleanup_io_tif:
876 l %r9,BASED(cleanup_table+20) # io_tif + 0x80000000
877 br %r14
879 cleanup_io_restore:
880 cl %r9,BASED(cleanup_io_restore_insn)
881 jhe 0f
882 l %r9,12(%r11) # get saved r11 pointer to pt_regs
883 mvc __LC_RETURN_PSW(8),__PT_PSW(%r9)
884 mvc 0(32,%r11),__PT_R8(%r9)
885 lm %r0,%r7,__PT_R0(%r9)
886 0: lm %r8,%r9,__LC_RETURN_PSW
887 br %r14
888 cleanup_io_restore_insn:
889 .long io_done - 4 + 0x80000000
891 cleanup_idle:
892 # copy interrupt clock & cpu timer
893 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
894 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
895 chi %r11,__LC_SAVE_AREA_ASYNC
896 je 0f
897 mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
898 mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
899 0: # check if stck has been executed
900 cl %r9,BASED(cleanup_idle_insn)
901 jhe 1f
902 mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
903 mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3)
904 1: # account system time going idle
905 lm %r9,%r10,__LC_STEAL_TIMER
906 ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2)
907 SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK
908 stm %r9,%r10,__LC_STEAL_TIMER
909 mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
910 lm %r9,%r10,__LC_SYSTEM_TIMER
911 ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER
912 SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2)
913 stm %r9,%r10,__LC_SYSTEM_TIMER
914 mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
915 # prepare return psw
916 n %r8,BASED(cleanup_idle_wait) # clear irq & wait state bits
917 l %r9,24(%r11) # return from psw_idle
918 br %r14
919 cleanup_idle_insn:
920 .long psw_idle_lpsw + 0x80000000
921 cleanup_idle_wait:
922 .long 0xfcfdffff
924 /*
925 * Integer constants
926 */
927 .align 4
928 .Lnr_syscalls:
929 .long NR_syscalls
930 .Lvtimer_max:
931 .quad 0x7fffffffffffffff
933 /*
934 * Symbol constants
935 */
936 .Ldo_machine_check: .long s390_do_machine_check
937 .Lhandle_mcck: .long s390_handle_mcck
938 .Ldo_IRQ: .long do_IRQ
939 .Ldo_signal: .long do_signal
940 .Ldo_notify_resume: .long do_notify_resume
941 .Ldo_per_trap: .long do_per_trap
942 .Ljump_table: .long pgm_check_table
943 .Lschedule: .long schedule
944 #ifdef CONFIG_PREEMPT
945 .Lpreempt_irq: .long preempt_schedule_irq
946 #endif
947 .Ltrace_enter: .long do_syscall_trace_enter
948 .Ltrace_exit: .long do_syscall_trace_exit
949 .Lschedule_tail: .long schedule_tail
950 .Lsysc_per: .long sysc_per + 0x80000000
951 #ifdef CONFIG_TRACE_IRQFLAGS
952 .Lhardirqs_on: .long trace_hardirqs_on_caller
953 .Lhardirqs_off: .long trace_hardirqs_off_caller
954 #endif
955 #ifdef CONFIG_LOCKDEP
956 .Llockdep_sys_exit: .long lockdep_sys_exit
957 #endif
958 .Lcritical_start: .long __critical_start + 0x80000000
959 .Lcritical_length: .long __critical_end - __critical_start
961 .section .rodata, "a"
962 #define SYSCALL(esa,esame,emu) .long esa
963 .globl sys_call_table
964 sys_call_table:
965 #include "syscalls.S"
966 #undef SYSCALL