| blob | d93e396bf5998813114d0ac1cbbd92551d964551 |
1 /*
2 * arch/ia64/kernel/ivt.S
3 *
4 * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger <davidm@hpl.hp.com>
7 * Copyright (C) 2000, 2002-2003 Intel Co
8 * Asit Mallick <asit.k.mallick@intel.com>
9 * Suresh Siddha <suresh.b.siddha@intel.com>
10 * Kenneth Chen <kenneth.w.chen@intel.com>
11 * Fenghua Yu <fenghua.yu@intel.com>
12 *
13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
14 * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
15 *
16 * Copyright (C) 2005 Hewlett-Packard Co
17 * Dan Magenheimer <dan.magenheimer@hp.com>
18 * Xen paravirtualization
19 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
20 * VA Linux Systems Japan K.K.
21 * pv_ops.
22 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
23 */
24 /*
25 * This file defines the interruption vector table used by the CPU.
26 * It does not include one entry per possible cause of interruption.
27 *
28 * The first 20 entries of the table contain 64 bundles each while the
29 * remaining 48 entries contain only 16 bundles each.
30 *
31 * The 64 bundles are used to allow inlining the whole handler for critical
32 * interruptions like TLB misses.
33 *
34 * For each entry, the comment is as follows:
35 *
36 * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
37 * entry offset ----/ / / / /
38 * entry number ---------/ / / /
39 * size of the entry -------------/ / /
40 * vector name -------------------------------------/ /
41 * interruptions triggering this vector ----------------------/
42 *
43 * The table is 32KB in size and must be aligned on 32KB boundary.
44 * (The CPU ignores the 15 lower bits of the address)
45 *
46 * Table is based upon EAS2.6 (Oct 1999)
47 */
50 #include <asm/asmmacro.h>
51 #include <asm/break.h>
52 #include <asm/kregs.h>
53 #include <asm/asm-offsets.h>
54 #include <asm/pgtable.h>
55 #include <asm/processor.h>
56 #include <asm/ptrace.h>
57 #include <asm/system.h>
58 #include <asm/thread_info.h>
59 #include <asm/unistd.h>
60 #include <asm/errno.h>
62 #if 1
63 # define PSR_DEFAULT_BITS psr.ac
64 #else
65 # define PSR_DEFAULT_BITS 0
66 #endif
68 #if 0
69 /*
70 * This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't
71 * needed for something else before enabling this...
72 */
73 # define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16
74 #else
75 # define DBG_FAULT(i)
76 #endif
78 #include "minstate.h"
80 #define FAULT(n) \
81 mov r31=pr; \
82 mov r19=n;; /* prepare to save predicates */ \
83 br.sptk.many dispatch_to_fault_handler
85 .section .text..ivt,"ax"
87 .align 32768 // align on 32KB boundary
88 .global ia64_ivt
89 ia64_ivt:
90 /////////////////////////////////////////////////////////////////////////////////////////
91 // 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
92 ENTRY(vhpt_miss)
93 DBG_FAULT(0)
94 /*
95 * The VHPT vector is invoked when the TLB entry for the virtual page table
96 * is missing. This happens only as a result of a previous
97 * (the "original") TLB miss, which may either be caused by an instruction
98 * fetch or a data access (or non-access).
99 *
100 * What we do here is normal TLB miss handing for the _original_ miss,
101 * followed by inserting the TLB entry for the virtual page table page
102 * that the VHPT walker was attempting to access. The latter gets
103 * inserted as long as page table entry above pte level have valid
104 * mappings for the faulting address. The TLB entry for the original
105 * miss gets inserted only if the pte entry indicates that the page is
106 * present.
107 *
108 * do_page_fault gets invoked in the following cases:
109 * - the faulting virtual address uses unimplemented address bits
110 * - the faulting virtual address has no valid page table mapping
111 */
112 MOV_FROM_IFA(r16) // get address that caused the TLB miss
113 #ifdef CONFIG_HUGETLB_PAGE
114 movl r18=PAGE_SHIFT
115 MOV_FROM_ITIR(r25)
116 #endif
117 ;;
118 RSM_PSR_DT // use physical addressing for data
119 mov r31=pr // save the predicate registers
120 mov r19=IA64_KR(PT_BASE) // get page table base address
121 shl r21=r16,3 // shift bit 60 into sign bit
122 shr.u r17=r16,61 // get the region number into r17
123 ;;
124 shr.u r22=r21,3
125 #ifdef CONFIG_HUGETLB_PAGE
126 extr.u r26=r25,2,6
127 ;;
128 cmp.ne p8,p0=r18,r26
129 sub r27=r26,r18
130 ;;
131 (p8) dep r25=r18,r25,2,6
132 (p8) shr r22=r22,r27
133 #endif
134 ;;
135 cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
136 shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
137 ;;
138 (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
140 srlz.d
141 LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
143 .pred.rel "mutex", p6, p7
144 (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
145 (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
146 ;;
147 (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
148 (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
149 cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
150 #ifdef CONFIG_PGTABLE_4
151 shr.u r28=r22,PUD_SHIFT // shift pud index into position
152 #else
153 shr.u r18=r22,PMD_SHIFT // shift pmd index into position
154 #endif
155 ;;
156 ld8 r17=[r17] // get *pgd (may be 0)
157 ;;
158 (p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
159 #ifdef CONFIG_PGTABLE_4
160 dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
161 ;;
162 shr.u r18=r22,PMD_SHIFT // shift pmd index into position
163 (p7) ld8 r29=[r28] // get *pud (may be 0)
164 ;;
165 (p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
166 dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
167 #else
168 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
169 #endif
170 ;;
171 (p7) ld8 r20=[r17] // get *pmd (may be 0)
172 shr.u r19=r22,PAGE_SHIFT // shift pte index into position
173 ;;
174 (p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
175 dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
176 ;;
177 (p7) ld8 r18=[r21] // read *pte
178 MOV_FROM_ISR(r19) // cr.isr bit 32 tells us if this is an insn miss
179 ;;
180 (p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
181 MOV_FROM_IHA(r22) // get the VHPT address that caused the TLB miss
182 ;; // avoid RAW on p7
183 (p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
184 dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address
185 ;;
186 ITC_I_AND_D(p10, p11, r18, r24) // insert the instruction TLB entry and
187 // insert the data TLB entry
188 (p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault)
189 MOV_TO_IFA(r22, r24)
191 #ifdef CONFIG_HUGETLB_PAGE
192 MOV_TO_ITIR(p8, r25, r24) // change to default page-size for VHPT
193 #endif
195 /*
196 * Now compute and insert the TLB entry for the virtual page table. We never
197 * execute in a page table page so there is no need to set the exception deferral
198 * bit.
199 */
200 adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
201 ;;
202 ITC_D(p7, r24, r25)
203 ;;
204 #ifdef CONFIG_SMP
205 /*
206 * Tell the assemblers dependency-violation checker that the above "itc" instructions
207 * cannot possibly affect the following loads:
208 */
209 dv_serialize_data
211 /*
212 * Re-check pagetable entry. If they changed, we may have received a ptc.g
213 * between reading the pagetable and the "itc". If so, flush the entry we
214 * inserted and retry. At this point, we have:
215 *
216 * r28 = equivalent of pud_offset(pgd, ifa)
217 * r17 = equivalent of pmd_offset(pud, ifa)
218 * r21 = equivalent of pte_offset(pmd, ifa)
219 *
220 * r29 = *pud
221 * r20 = *pmd
222 * r18 = *pte
223 */
224 ld8 r25=[r21] // read *pte again
225 ld8 r26=[r17] // read *pmd again
226 #ifdef CONFIG_PGTABLE_4
227 ld8 r19=[r28] // read *pud again
228 #endif
229 cmp.ne p6,p7=r0,r0
230 ;;
231 cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
232 #ifdef CONFIG_PGTABLE_4
233 cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
234 #endif
235 mov r27=PAGE_SHIFT<<2
236 ;;
237 (p6) ptc.l r22,r27 // purge PTE page translation
238 (p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
239 ;;
240 (p6) ptc.l r16,r27 // purge translation
241 #endif
243 mov pr=r31,-1 // restore predicate registers
244 RFI
245 END(vhpt_miss)
247 .org ia64_ivt+0x400
248 /////////////////////////////////////////////////////////////////////////////////////////
249 // 0x0400 Entry 1 (size 64 bundles) ITLB (21)
250 ENTRY(itlb_miss)
251 DBG_FAULT(1)
252 /*
253 * The ITLB handler accesses the PTE via the virtually mapped linear
254 * page table. If a nested TLB miss occurs, we switch into physical
255 * mode, walk the page table, and then re-execute the PTE read and
256 * go on normally after that.
257 */
258 MOV_FROM_IFA(r16) // get virtual address
259 mov r29=b0 // save b0
260 mov r31=pr // save predicates
261 .itlb_fault:
262 MOV_FROM_IHA(r17) // get virtual address of PTE
263 movl r30=1f // load nested fault continuation point
264 ;;
265 1: ld8 r18=[r17] // read *pte
266 ;;
267 mov b0=r29
268 tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
269 (p6) br.cond.spnt page_fault
270 ;;
271 ITC_I(p0, r18, r19)
272 ;;
273 #ifdef CONFIG_SMP
274 /*
275 * Tell the assemblers dependency-violation checker that the above "itc" instructions
276 * cannot possibly affect the following loads:
277 */
278 dv_serialize_data
280 ld8 r19=[r17] // read *pte again and see if same
281 mov r20=PAGE_SHIFT<<2 // setup page size for purge
282 ;;
283 cmp.ne p7,p0=r18,r19
284 ;;
285 (p7) ptc.l r16,r20
286 #endif
287 mov pr=r31,-1
288 RFI
289 END(itlb_miss)
291 .org ia64_ivt+0x0800
292 /////////////////////////////////////////////////////////////////////////////////////////
293 // 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
294 ENTRY(dtlb_miss)
295 DBG_FAULT(2)
296 /*
297 * The DTLB handler accesses the PTE via the virtually mapped linear
298 * page table. If a nested TLB miss occurs, we switch into physical
299 * mode, walk the page table, and then re-execute the PTE read and
300 * go on normally after that.
301 */
302 MOV_FROM_IFA(r16) // get virtual address
303 mov r29=b0 // save b0
304 mov r31=pr // save predicates
305 dtlb_fault:
306 MOV_FROM_IHA(r17) // get virtual address of PTE
307 movl r30=1f // load nested fault continuation point
308 ;;
309 1: ld8 r18=[r17] // read *pte
310 ;;
311 mov b0=r29
312 tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
313 (p6) br.cond.spnt page_fault
314 ;;
315 ITC_D(p0, r18, r19)
316 ;;
317 #ifdef CONFIG_SMP
318 /*
319 * Tell the assemblers dependency-violation checker that the above "itc" instructions
320 * cannot possibly affect the following loads:
321 */
322 dv_serialize_data
324 ld8 r19=[r17] // read *pte again and see if same
325 mov r20=PAGE_SHIFT<<2 // setup page size for purge
326 ;;
327 cmp.ne p7,p0=r18,r19
328 ;;
329 (p7) ptc.l r16,r20
330 #endif
331 mov pr=r31,-1
332 RFI
333 END(dtlb_miss)
335 .org ia64_ivt+0x0c00
336 /////////////////////////////////////////////////////////////////////////////////////////
337 // 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
338 ENTRY(alt_itlb_miss)
339 DBG_FAULT(3)
340 MOV_FROM_IFA(r16) // get address that caused the TLB miss
341 movl r17=PAGE_KERNEL
342 MOV_FROM_IPSR(p0, r21)
343 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
344 mov r31=pr
345 ;;
346 #ifdef CONFIG_DISABLE_VHPT
347 shr.u r22=r16,61 // get the region number into r21
348 ;;
349 cmp.gt p8,p0=6,r22 // user mode
350 ;;
351 THASH(p8, r17, r16, r23)
352 ;;
353 MOV_TO_IHA(p8, r17, r23)
354 (p8) mov r29=b0 // save b0
355 (p8) br.cond.dptk .itlb_fault
356 #endif
357 extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
358 and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
359 shr.u r18=r16,57 // move address bit 61 to bit 4
360 ;;
361 andcm r18=0x10,r18 // bit 4=~address-bit(61)
362 cmp.ne p8,p0=r0,r23 // psr.cpl != 0?
363 or r19=r17,r19 // insert PTE control bits into r19
364 ;;
365 or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
366 (p8) br.cond.spnt page_fault
367 ;;
368 ITC_I(p0, r19, r18) // insert the TLB entry
369 mov pr=r31,-1
370 RFI
371 END(alt_itlb_miss)
373 .org ia64_ivt+0x1000
374 /////////////////////////////////////////////////////////////////////////////////////////
375 // 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
376 ENTRY(alt_dtlb_miss)
377 DBG_FAULT(4)
378 MOV_FROM_IFA(r16) // get address that caused the TLB miss
379 movl r17=PAGE_KERNEL
380 MOV_FROM_ISR(r20)
381 movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
382 MOV_FROM_IPSR(p0, r21)
383 mov r31=pr
384 mov r24=PERCPU_ADDR
385 ;;
386 #ifdef CONFIG_DISABLE_VHPT
387 shr.u r22=r16,61 // get the region number into r21
388 ;;
389 cmp.gt p8,p0=6,r22 // access to region 0-5
390 ;;
391 THASH(p8, r17, r16, r25)
392 ;;
393 MOV_TO_IHA(p8, r17, r25)
394 (p8) mov r29=b0 // save b0
395 (p8) br.cond.dptk dtlb_fault
396 #endif
397 cmp.ge p10,p11=r16,r24 // access to per_cpu_data?
398 tbit.z p12,p0=r16,61 // access to region 6?
399 mov r25=PERCPU_PAGE_SHIFT << 2
400 mov r26=PERCPU_PAGE_SIZE
401 nop.m 0
402 nop.b 0
403 ;;
404 (p10) mov r19=IA64_KR(PER_CPU_DATA)
405 (p11) and r19=r19,r16 // clear non-ppn fields
406 extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
407 and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
408 tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
409 tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
410 ;;
411 (p10) sub r19=r19,r26
412 MOV_TO_ITIR(p10, r25, r24)
413 cmp.ne p8,p0=r0,r23
414 (p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
415 (p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr
416 (p8) br.cond.spnt page_fault
418 dep r21=-1,r21,IA64_PSR_ED_BIT,1
419 ;;
420 or r19=r19,r17 // insert PTE control bits into r19
421 MOV_TO_IPSR(p6, r21, r24)
422 ;;
423 ITC_D(p7, r19, r18) // insert the TLB entry
424 mov pr=r31,-1
425 RFI
426 END(alt_dtlb_miss)
428 .org ia64_ivt+0x1400
429 /////////////////////////////////////////////////////////////////////////////////////////
430 // 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
431 ENTRY(nested_dtlb_miss)
432 /*
433 * In the absence of kernel bugs, we get here when the virtually mapped linear
434 * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
435 * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
436 * table is missing, a nested TLB miss fault is triggered and control is
437 * transferred to this point. When this happens, we lookup the pte for the
438 * faulting address by walking the page table in physical mode and return to the
439 * continuation point passed in register r30 (or call page_fault if the address is
440 * not mapped).
441 *
442 * Input: r16: faulting address
443 * r29: saved b0
444 * r30: continuation address
445 * r31: saved pr
446 *
447 * Output: r17: physical address of PTE of faulting address
448 * r29: saved b0
449 * r30: continuation address
450 * r31: saved pr
451 *
452 * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared)
453 */
454 RSM_PSR_DT // switch to using physical data addressing
455 mov r19=IA64_KR(PT_BASE) // get the page table base address
456 shl r21=r16,3 // shift bit 60 into sign bit
457 MOV_FROM_ITIR(r18)
458 ;;
459 shr.u r17=r16,61 // get the region number into r17
460 extr.u r18=r18,2,6 // get the faulting page size
461 ;;
462 cmp.eq p6,p7=5,r17 // is faulting address in region 5?
463 add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
464 add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
465 ;;
466 shr.u r22=r16,r22
467 shr.u r18=r16,r18
468 (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
470 srlz.d
471 LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
473 .pred.rel "mutex", p6, p7
474 (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
475 (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
476 ;;
477 (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
478 (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
479 cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
480 #ifdef CONFIG_PGTABLE_4
481 shr.u r18=r22,PUD_SHIFT // shift pud index into position
482 #else
483 shr.u r18=r22,PMD_SHIFT // shift pmd index into position
484 #endif
485 ;;
486 ld8 r17=[r17] // get *pgd (may be 0)
487 ;;
488 (p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
489 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
490 ;;
491 #ifdef CONFIG_PGTABLE_4
492 (p7) ld8 r17=[r17] // get *pud (may be 0)
493 shr.u r18=r22,PMD_SHIFT // shift pmd index into position
494 ;;
495 (p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
496 dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
497 ;;
498 #endif
499 (p7) ld8 r17=[r17] // get *pmd (may be 0)
500 shr.u r19=r22,PAGE_SHIFT // shift pte index into position
501 ;;
502 (p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
503 dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
504 (p6) br.cond.spnt page_fault
505 mov b0=r30
506 br.sptk.many b0 // return to continuation point
507 END(nested_dtlb_miss)
509 .org ia64_ivt+0x1800
510 /////////////////////////////////////////////////////////////////////////////////////////
511 // 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
512 ENTRY(ikey_miss)
513 DBG_FAULT(6)
514 FAULT(6)
515 END(ikey_miss)
517 .org ia64_ivt+0x1c00
518 /////////////////////////////////////////////////////////////////////////////////////////
519 // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
520 ENTRY(dkey_miss)
521 DBG_FAULT(7)
522 FAULT(7)
523 END(dkey_miss)
525 .org ia64_ivt+0x2000
526 /////////////////////////////////////////////////////////////////////////////////////////
527 // 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
528 ENTRY(dirty_bit)
529 DBG_FAULT(8)
530 /*
531 * What we do here is to simply turn on the dirty bit in the PTE. We need to
532 * update both the page-table and the TLB entry. To efficiently access the PTE,
533 * we address it through the virtual page table. Most likely, the TLB entry for
534 * the relevant virtual page table page is still present in the TLB so we can
535 * normally do this without additional TLB misses. In case the necessary virtual
536 * page table TLB entry isn't present, we take a nested TLB miss hit where we look
537 * up the physical address of the L3 PTE and then continue at label 1 below.
538 */
539 MOV_FROM_IFA(r16) // get the address that caused the fault
540 movl r30=1f // load continuation point in case of nested fault
541 ;;
542 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
543 mov r29=b0 // save b0 in case of nested fault
544 mov r31=pr // save pr
545 #ifdef CONFIG_SMP
546 mov r28=ar.ccv // save ar.ccv
547 ;;
548 1: ld8 r18=[r17]
549 ;; // avoid RAW on r18
550 mov ar.ccv=r18 // set compare value for cmpxchg
551 or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
552 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
553 ;;
554 (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
555 mov r24=PAGE_SHIFT<<2
556 ;;
557 (p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
558 ;;
559 ITC_D(p6, r25, r18) // install updated PTE
560 ;;
561 /*
562 * Tell the assemblers dependency-violation checker that the above "itc" instructions
563 * cannot possibly affect the following loads:
564 */
565 dv_serialize_data
567 ld8 r18=[r17] // read PTE again
568 ;;
569 cmp.eq p6,p7=r18,r25 // is it same as the newly installed
570 ;;
571 (p7) ptc.l r16,r24
572 mov b0=r29 // restore b0
573 mov ar.ccv=r28
574 #else
575 ;;
576 1: ld8 r18=[r17]
577 ;; // avoid RAW on r18
578 or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
579 mov b0=r29 // restore b0
580 ;;
581 st8 [r17]=r18 // store back updated PTE
582 ITC_D(p0, r18, r16) // install updated PTE
583 #endif
584 mov pr=r31,-1 // restore pr
585 RFI
586 END(dirty_bit)
588 .org ia64_ivt+0x2400
589 /////////////////////////////////////////////////////////////////////////////////////////
590 // 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
591 ENTRY(iaccess_bit)
592 DBG_FAULT(9)
593 // Like Entry 8, except for instruction access
594 MOV_FROM_IFA(r16) // get the address that caused the fault
595 movl r30=1f // load continuation point in case of nested fault
596 mov r31=pr // save predicates
597 #ifdef CONFIG_ITANIUM
598 /*
599 * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
600 */
601 MOV_FROM_IPSR(p0, r17)
602 ;;
603 MOV_FROM_IIP(r18)
604 tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
605 ;;
606 (p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
607 #endif /* CONFIG_ITANIUM */
608 ;;
609 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
610 mov r29=b0 // save b0 in case of nested fault)
611 #ifdef CONFIG_SMP
612 mov r28=ar.ccv // save ar.ccv
613 ;;
614 1: ld8 r18=[r17]
615 ;;
616 mov ar.ccv=r18 // set compare value for cmpxchg
617 or r25=_PAGE_A,r18 // set the accessed bit
618 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
619 ;;
620 (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
621 mov r24=PAGE_SHIFT<<2
622 ;;
623 (p6) cmp.eq p6,p7=r26,r18 // Only if page present
624 ;;
625 ITC_I(p6, r25, r26) // install updated PTE
626 ;;
627 /*
628 * Tell the assemblers dependency-violation checker that the above "itc" instructions
629 * cannot possibly affect the following loads:
630 */
631 dv_serialize_data
633 ld8 r18=[r17] // read PTE again
634 ;;
635 cmp.eq p6,p7=r18,r25 // is it same as the newly installed
636 ;;
637 (p7) ptc.l r16,r24
638 mov b0=r29 // restore b0
639 mov ar.ccv=r28
640 #else /* !CONFIG_SMP */
641 ;;
642 1: ld8 r18=[r17]
643 ;;
644 or r18=_PAGE_A,r18 // set the accessed bit
645 mov b0=r29 // restore b0
646 ;;
647 st8 [r17]=r18 // store back updated PTE
648 ITC_I(p0, r18, r16) // install updated PTE
649 #endif /* !CONFIG_SMP */
650 mov pr=r31,-1
651 RFI
652 END(iaccess_bit)
654 .org ia64_ivt+0x2800
655 /////////////////////////////////////////////////////////////////////////////////////////
656 // 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
657 ENTRY(daccess_bit)
658 DBG_FAULT(10)
659 // Like Entry 8, except for data access
660 MOV_FROM_IFA(r16) // get the address that caused the fault
661 movl r30=1f // load continuation point in case of nested fault
662 ;;
663 THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
664 mov r31=pr
665 mov r29=b0 // save b0 in case of nested fault)
666 #ifdef CONFIG_SMP
667 mov r28=ar.ccv // save ar.ccv
668 ;;
669 1: ld8 r18=[r17]
670 ;; // avoid RAW on r18
671 mov ar.ccv=r18 // set compare value for cmpxchg
672 or r25=_PAGE_A,r18 // set the dirty bit
673 tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
674 ;;
675 (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
676 mov r24=PAGE_SHIFT<<2
677 ;;
678 (p6) cmp.eq p6,p7=r26,r18 // Only if page is present
679 ;;
680 ITC_D(p6, r25, r26) // install updated PTE
681 /*
682 * Tell the assemblers dependency-violation checker that the above "itc" instructions
683 * cannot possibly affect the following loads:
684 */
685 dv_serialize_data
686 ;;
687 ld8 r18=[r17] // read PTE again
688 ;;
689 cmp.eq p6,p7=r18,r25 // is it same as the newly installed
690 ;;
691 (p7) ptc.l r16,r24
692 mov ar.ccv=r28
693 #else
694 ;;
695 1: ld8 r18=[r17]
696 ;; // avoid RAW on r18
697 or r18=_PAGE_A,r18 // set the accessed bit
698 ;;
699 st8 [r17]=r18 // store back updated PTE
700 ITC_D(p0, r18, r16) // install updated PTE
701 #endif
702 mov b0=r29 // restore b0
703 mov pr=r31,-1
704 RFI
705 END(daccess_bit)
707 .org ia64_ivt+0x2c00
708 /////////////////////////////////////////////////////////////////////////////////////////
709 // 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
710 ENTRY(break_fault)
711 /*
712 * The streamlined system call entry/exit paths only save/restore the initial part
713 * of pt_regs. This implies that the callers of system-calls must adhere to the
714 * normal procedure calling conventions.
715 *
716 * Registers to be saved & restored:
717 * CR registers: cr.ipsr, cr.iip, cr.ifs
718 * AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
719 * others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
720 * Registers to be restored only:
721 * r8-r11: output value from the system call.
722 *
723 * During system call exit, scratch registers (including r15) are modified/cleared
724 * to prevent leaking bits from kernel to user level.
725 */
726 DBG_FAULT(11)
727 mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
728 MOV_FROM_IPSR(p0, r29) // M2 (12 cyc)
729 mov r31=pr // I0 (2 cyc)
731 MOV_FROM_IIM(r17) // M2 (2 cyc)
732 mov.m r27=ar.rsc // M2 (12 cyc)
733 mov r18=__IA64_BREAK_SYSCALL // A
735 mov.m ar.rsc=0 // M2
736 mov.m r21=ar.fpsr // M2 (12 cyc)
737 mov r19=b6 // I0 (2 cyc)
738 ;;
739 mov.m r23=ar.bspstore // M2 (12 cyc)
740 mov.m r24=ar.rnat // M2 (5 cyc)
741 mov.i r26=ar.pfs // I0 (2 cyc)
743 invala // M0|1
744 nop.m 0 // M
745 mov r20=r1 // A save r1
747 nop.m 0
748 movl r30=sys_call_table // X
750 MOV_FROM_IIP(r28) // M2 (2 cyc)
751 cmp.eq p0,p7=r18,r17 // I0 is this a system call?
752 (p7) br.cond.spnt non_syscall // B no ->
753 //
754 // From this point on, we are definitely on the syscall-path
755 // and we can use (non-banked) scratch registers.
756 //
757 ///////////////////////////////////////////////////////////////////////
758 mov r1=r16 // A move task-pointer to "addl"-addressable reg
759 mov r2=r16 // A setup r2 for ia64_syscall_setup
760 add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = ¤t_thread_info()->flags
762 adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
763 adds r15=-1024,r15 // A subtract 1024 from syscall number
764 mov r3=NR_syscalls - 1
765 ;;
766 ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
767 ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
768 extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
770 shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
771 addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
772 cmp.leu p6,p7=r15,r3 // A syscall number in range?
773 ;;
775 lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
776 (p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
777 tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
779 mov.m ar.bspstore=r22 // M2 switch to kernel RBS
780 cmp.eq p8,p9=2,r8 // A isr.ei==2?
781 ;;
783 (p8) mov r8=0 // A clear ei to 0
784 (p7) movl r30=sys_ni_syscall // X
786 (p8) adds r28=16,r28 // A switch cr.iip to next bundle
787 (p9) adds r8=1,r8 // A increment ei to next slot
788 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
789 ;;
790 mov b6=r30 // I0 setup syscall handler branch reg early
791 #else
792 nop.i 0
793 ;;
794 #endif
796 mov.m r25=ar.unat // M2 (5 cyc)
797 dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
798 adds r15=1024,r15 // A restore original syscall number
799 //
800 // If any of the above loads miss in L1D, we'll stall here until
801 // the data arrives.
802 //
803 ///////////////////////////////////////////////////////////////////////
804 st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
805 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
806 MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting
807 #else
808 mov b6=r30 // I0 setup syscall handler branch reg early
809 #endif
810 cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
812 and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
813 mov r18=ar.bsp // M2 (12 cyc)
814 (pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
815 ;;
816 .back_from_break_fixup:
817 (pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
818 cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
819 br.call.sptk.many b7=ia64_syscall_setup // B
820 1:
821 #ifdef CONFIG_VIRT_CPU_ACCOUNTING
822 // mov.m r30=ar.itc is called in advance, and r13 is current
823 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A
824 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
825 (pKStk) br.cond.spnt .skip_accounting // B unlikely skip
826 ;;
827 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp
828 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave
829 ;;
830 ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime
831 ld8 r21=[r17] // M cumulated utime
832 sub r22=r19,r18 // A stime before leave
833 ;;
834 st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp
835 sub r18=r30,r19 // A elapsed time in user
836 ;;
837 add r20=r20,r22 // A sum stime
838 add r21=r21,r18 // A sum utime
839 ;;
840 st8 [r16]=r20 // M update stime
841 st8 [r17]=r21 // M update utime
842 ;;
843 .skip_accounting:
844 #endif
845 mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
846 nop 0
847 BSW_1(r2, r14) // B (6 cyc) regs are saved, switch to bank 1
848 ;;
850 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16) // M2 now it's safe to re-enable intr.-collection
851 // M0 ensure interruption collection is on
852 movl r3=ia64_ret_from_syscall // X
853 ;;
854 mov rp=r3 // I0 set the real return addr
855 (p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
857 SSM_PSR_I(p15, p15, r16) // M2 restore psr.i
858 (p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
859 br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
860 // NOT REACHED
861 ///////////////////////////////////////////////////////////////////////
862 // On entry, we optimistically assumed that we're coming from user-space.
863 // For the rare cases where a system-call is done from within the kernel,
864 // we fix things up at this point:
865 .break_fixup:
866 add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
867 mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
868 ;;
869 mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
870 br.cond.sptk .back_from_break_fixup
871 END(break_fault)
873 .org ia64_ivt+0x3000
874 /////////////////////////////////////////////////////////////////////////////////////////
875 // 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
876 ENTRY(interrupt)
877 /* interrupt handler has become too big to fit this area. */
878 br.sptk.many __interrupt
879 END(interrupt)
881 .org ia64_ivt+0x3400
882 /////////////////////////////////////////////////////////////////////////////////////////
883 // 0x3400 Entry 13 (size 64 bundles) Reserved
884 DBG_FAULT(13)
885 FAULT(13)
887 .org ia64_ivt+0x3800
888 /////////////////////////////////////////////////////////////////////////////////////////
889 // 0x3800 Entry 14 (size 64 bundles) Reserved
890 DBG_FAULT(14)
891 FAULT(14)
893 /*
894 * There is no particular reason for this code to be here, other than that
895 * there happens to be space here that would go unused otherwise. If this
896 * fault ever gets "unreserved", simply moved the following code to a more
897 * suitable spot...
898 *
899 * ia64_syscall_setup() is a separate subroutine so that it can
900 * allocate stacked registers so it can safely demine any
901 * potential NaT values from the input registers.
902 *
903 * On entry:
904 * - executing on bank 0 or bank 1 register set (doesn't matter)
905 * - r1: stack pointer
906 * - r2: current task pointer
907 * - r3: preserved
908 * - r11: original contents (saved ar.pfs to be saved)
909 * - r12: original contents (sp to be saved)
910 * - r13: original contents (tp to be saved)
911 * - r15: original contents (syscall # to be saved)
912 * - r18: saved bsp (after switching to kernel stack)
913 * - r19: saved b6
914 * - r20: saved r1 (gp)
915 * - r21: saved ar.fpsr
916 * - r22: kernel's register backing store base (krbs_base)
917 * - r23: saved ar.bspstore
918 * - r24: saved ar.rnat
919 * - r25: saved ar.unat
920 * - r26: saved ar.pfs
921 * - r27: saved ar.rsc
922 * - r28: saved cr.iip
923 * - r29: saved cr.ipsr
924 * - r30: ar.itc for accounting (don't touch)
925 * - r31: saved pr
926 * - b0: original contents (to be saved)
927 * On exit:
928 * - p10: TRUE if syscall is invoked with more than 8 out
929 * registers or r15's Nat is true
930 * - r1: kernel's gp
931 * - r3: preserved (same as on entry)
932 * - r8: -EINVAL if p10 is true
933 * - r12: points to kernel stack
934 * - r13: points to current task
935 * - r14: preserved (same as on entry)
936 * - p13: preserved
937 * - p15: TRUE if interrupts need to be re-enabled
938 * - ar.fpsr: set to kernel settings
939 * - b6: preserved (same as on entry)
940 */
941 #ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
942 GLOBAL_ENTRY(ia64_syscall_setup)
943 #if PT(B6) != 0
944 # error This code assumes that b6 is the first field in pt_regs.
945 #endif
946 st8 [r1]=r19 // save b6
947 add r16=PT(CR_IPSR),r1 // initialize first base pointer
948 add r17=PT(R11),r1 // initialize second base pointer
949 ;;
950 alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
951 st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
952 tnat.nz p8,p0=in0
954 st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
955 tnat.nz p9,p0=in1
956 (pKStk) mov r18=r0 // make sure r18 isn't NaT
957 ;;
959 st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
960 st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
961 mov r28=b0 // save b0 (2 cyc)
962 ;;
964 st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
965 dep r19=0,r19,38,26 // clear all bits but 0..37 [I0]
966 (p8) mov in0=-1
967 ;;
969 st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
970 extr.u r11=r19,7,7 // I0 // get sol of ar.pfs
971 and r8=0x7f,r19 // A // get sof of ar.pfs
973 st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
974 tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
975 (p9) mov in1=-1
976 ;;
978 (pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8
979 tnat.nz p10,p0=in2
980 add r11=8,r11
981 ;;
982 (pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
983 (pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
984 tnat.nz p11,p0=in3
985 ;;
986 (p10) mov in2=-1
987 tnat.nz p12,p0=in4 // [I0]
988 (p11) mov in3=-1
989 ;;
990 (pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
991 (pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
992 shl r18=r18,16 // compute ar.rsc to be used for "loadrs"
993 ;;
994 st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
995 st8 [r17]=r28,PT(R1)-PT(B0) // save b0
996 tnat.nz p13,p0=in5 // [I0]
997 ;;
998 st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
999 st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
1000 (p12) mov in4=-1
1001 ;;
1003 .mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12
1004 .mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13
1005 (p13) mov in5=-1
1006 ;;
1007 st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
1008 tnat.nz p13,p0=in6
1009 cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
1010 ;;
1011 mov r8=1
1012 (p9) tnat.nz p10,p0=r15
1013 adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
1015 st8.spill [r17]=r15 // save r15
1016 tnat.nz p8,p0=in7
1017 nop.i 0
1019 mov r13=r2 // establish `current'
1020 movl r1=__gp // establish kernel global pointer
1021 ;;
1022 st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error)
1023 (p13) mov in6=-1
1024 (p8) mov in7=-1
1026 cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
1027 movl r17=FPSR_DEFAULT
1028 ;;
1029 mov.m ar.fpsr=r17 // set ar.fpsr to kernel default value
1030 (p10) mov r8=-EINVAL
1031 br.ret.sptk.many b7
1032 END(ia64_syscall_setup)
1033 #endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
1035 .org ia64_ivt+0x3c00
1036 /////////////////////////////////////////////////////////////////////////////////////////
1037 // 0x3c00 Entry 15 (size 64 bundles) Reserved
1038 DBG_FAULT(15)
1039 FAULT(15)
1041 .org ia64_ivt+0x4000
1042 /////////////////////////////////////////////////////////////////////////////////////////
1043 // 0x4000 Entry 16 (size 64 bundles) Reserved
1044 DBG_FAULT(16)
1045 FAULT(16)
1047 #if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE)
1048 /*
1049 * There is no particular reason for this code to be here, other than
1050 * that there happens to be space here that would go unused otherwise.
1051 * If this fault ever gets "unreserved", simply moved the following
1052 * code to a more suitable spot...
1053 *
1054 * account_sys_enter is called from SAVE_MIN* macros if accounting is
1055 * enabled and if the macro is entered from user mode.
1056 */
1057 GLOBAL_ENTRY(account_sys_enter)
1058 // mov.m r20=ar.itc is called in advance, and r13 is current
1059 add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13
1060 add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13
1061 ;;
1062 ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
1063 ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at left from kernel
1064 ;;
1065 ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
1066 ld8 r21=[r17] // cumulated utime
1067 sub r22=r19,r18 // stime before leave kernel
1068 ;;
1069 st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP // update stamp
1070 sub r18=r20,r19 // elapsed time in user mode
1071 ;;
1072 add r23=r23,r22 // sum stime
1073 add r21=r21,r18 // sum utime
1074 ;;
1075 st8 [r16]=r23 // update stime
1076 st8 [r17]=r21 // update utime
1077 ;;
1078 br.ret.sptk.many rp
1079 END(account_sys_enter)
1080 #endif
1082 .org ia64_ivt+0x4400
1083 /////////////////////////////////////////////////////////////////////////////////////////
1084 // 0x4400 Entry 17 (size 64 bundles) Reserved
1085 DBG_FAULT(17)
1086 FAULT(17)
1088 .org ia64_ivt+0x4800
1089 /////////////////////////////////////////////////////////////////////////////////////////
1090 // 0x4800 Entry 18 (size 64 bundles) Reserved
1091 DBG_FAULT(18)
1092 FAULT(18)
1094 .org ia64_ivt+0x4c00
1095 /////////////////////////////////////////////////////////////////////////////////////////
1096 // 0x4c00 Entry 19 (size 64 bundles) Reserved
1097 DBG_FAULT(19)
1098 FAULT(19)
1100 //
1101 // --- End of long entries, Beginning of short entries
1102 //
1104 .org ia64_ivt+0x5000
1105 /////////////////////////////////////////////////////////////////////////////////////////
1106 // 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
1107 ENTRY(page_not_present)
1108 DBG_FAULT(20)
1109 MOV_FROM_IFA(r16)
1110 RSM_PSR_DT
1111 /*
1112 * The Linux page fault handler doesn't expect non-present pages to be in
1113 * the TLB. Flush the existing entry now, so we meet that expectation.
1114 */
1115 mov r17=PAGE_SHIFT<<2
1116 ;;
1117 ptc.l r16,r17
1118 ;;
1119 mov r31=pr
1120 srlz.d
1121 br.sptk.many page_fault
1122 END(page_not_present)
1124 .org ia64_ivt+0x5100
1125 /////////////////////////////////////////////////////////////////////////////////////////
1126 // 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
1127 ENTRY(key_permission)
1128 DBG_FAULT(21)
1129 MOV_FROM_IFA(r16)
1130 RSM_PSR_DT
1131 mov r31=pr
1132 ;;
1133 srlz.d
1134 br.sptk.many page_fault
1135 END(key_permission)
1137 .org ia64_ivt+0x5200
1138 /////////////////////////////////////////////////////////////////////////////////////////
1139 // 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
1140 ENTRY(iaccess_rights)
1141 DBG_FAULT(22)
1142 MOV_FROM_IFA(r16)
1143 RSM_PSR_DT
1144 mov r31=pr
1145 ;;
1146 srlz.d
1147 br.sptk.many page_fault
1148 END(iaccess_rights)
1150 .org ia64_ivt+0x5300
1151 /////////////////////////////////////////////////////////////////////////////////////////
1152 // 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
1153 ENTRY(daccess_rights)
1154 DBG_FAULT(23)
1155 MOV_FROM_IFA(r16)
1156 RSM_PSR_DT
1157 mov r31=pr
1158 ;;
1159 srlz.d
1160 br.sptk.many page_fault
1161 END(daccess_rights)
1163 .org ia64_ivt+0x5400
1164 /////////////////////////////////////////////////////////////////////////////////////////
1165 // 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
1166 ENTRY(general_exception)
1167 DBG_FAULT(24)
1168 MOV_FROM_ISR(r16)
1169 mov r31=pr
1170 ;;
1171 cmp4.eq p6,p0=0,r16
1172 (p6) br.sptk.many dispatch_illegal_op_fault
1173 ;;
1174 mov r19=24 // fault number
1175 br.sptk.many dispatch_to_fault_handler
1176 END(general_exception)
1178 .org ia64_ivt+0x5500
1179 /////////////////////////////////////////////////////////////////////////////////////////
1180 // 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
1181 ENTRY(disabled_fp_reg)
1182 DBG_FAULT(25)
1183 rsm psr.dfh // ensure we can access fph
1184 ;;
1185 srlz.d
1186 mov r31=pr
1187 mov r19=25
1188 br.sptk.many dispatch_to_fault_handler
1189 END(disabled_fp_reg)
1191 .org ia64_ivt+0x5600
1192 /////////////////////////////////////////////////////////////////////////////////////////
1193 // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
1194 ENTRY(nat_consumption)
1195 DBG_FAULT(26)
1197 MOV_FROM_IPSR(p0, r16)
1198 MOV_FROM_ISR(r17)
1199 mov r31=pr // save PR
1200 ;;
1201 and r18=0xf,r17 // r18 = cr.ipsr.code{3:0}
1202 tbit.z p6,p0=r17,IA64_ISR_NA_BIT
1203 ;;
1204 cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
1205 dep r16=-1,r16,IA64_PSR_ED_BIT,1
1206 (p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
1207 ;;
1208 MOV_TO_IPSR(p0, r16, r18)
1209 mov pr=r31,-1
1210 ;;
1211 RFI
1213 1: mov pr=r31,-1
1214 ;;
1215 FAULT(26)
1216 END(nat_consumption)
1218 .org ia64_ivt+0x5700
1219 /////////////////////////////////////////////////////////////////////////////////////////
1220 // 0x5700 Entry 27 (size 16 bundles) Speculation (40)
1221 ENTRY(speculation_vector)
1222 DBG_FAULT(27)
1223 /*
1224 * A [f]chk.[as] instruction needs to take the branch to the recovery code but
1225 * this part of the architecture is not implemented in hardware on some CPUs, such
1226 * as Itanium. Thus, in general we need to emulate the behavior. IIM contains
1227 * the relative target (not yet sign extended). So after sign extending it we
1228 * simply add it to IIP. We also need to reset the EI field of the IPSR to zero,
1229 * i.e., the slot to restart into.
1230 *
1231 * cr.imm contains zero_ext(imm21)
1232 */
1233 MOV_FROM_IIM(r18)
1234 ;;
1235 MOV_FROM_IIP(r17)
1236 shl r18=r18,43 // put sign bit in position (43=64-21)
1237 ;;
1239 MOV_FROM_IPSR(p0, r16)
1240 shr r18=r18,39 // sign extend (39=43-4)
1241 ;;
1243 add r17=r17,r18 // now add the offset
1244 ;;
1245 MOV_TO_IIP(r17, r19)
1246 dep r16=0,r16,41,2 // clear EI
1247 ;;
1249 MOV_TO_IPSR(p0, r16, r19)
1250 ;;
1252 RFI
1253 END(speculation_vector)
1255 .org ia64_ivt+0x5800
1256 /////////////////////////////////////////////////////////////////////////////////////////
1257 // 0x5800 Entry 28 (size 16 bundles) Reserved
1258 DBG_FAULT(28)
1259 FAULT(28)
1261 .org ia64_ivt+0x5900
1262 /////////////////////////////////////////////////////////////////////////////////////////
1263 // 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
1264 ENTRY(debug_vector)
1265 DBG_FAULT(29)
1266 FAULT(29)
1267 END(debug_vector)
1269 .org ia64_ivt+0x5a00
1270 /////////////////////////////////////////////////////////////////////////////////////////
1271 // 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
1272 ENTRY(unaligned_access)
1273 DBG_FAULT(30)
1274 mov r31=pr // prepare to save predicates
1275 ;;
1276 br.sptk.many dispatch_unaligned_handler
1277 END(unaligned_access)
1279 .org ia64_ivt+0x5b00
1280 /////////////////////////////////////////////////////////////////////////////////////////
1281 // 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
1282 ENTRY(unsupported_data_reference)
1283 DBG_FAULT(31)
1284 FAULT(31)
1285 END(unsupported_data_reference)
1287 .org ia64_ivt+0x5c00
1288 /////////////////////////////////////////////////////////////////////////////////////////
1289 // 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
1290 ENTRY(floating_point_fault)
1291 DBG_FAULT(32)
1292 FAULT(32)
1293 END(floating_point_fault)
1295 .org ia64_ivt+0x5d00
1296 /////////////////////////////////////////////////////////////////////////////////////////
1297 // 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
1298 ENTRY(floating_point_trap)
1299 DBG_FAULT(33)
1300 FAULT(33)
1301 END(floating_point_trap)
1303 .org ia64_ivt+0x5e00
1304 /////////////////////////////////////////////////////////////////////////////////////////
1305 // 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
1306 ENTRY(lower_privilege_trap)
1307 DBG_FAULT(34)
1308 FAULT(34)
1309 END(lower_privilege_trap)
1311 .org ia64_ivt+0x5f00
1312 /////////////////////////////////////////////////////////////////////////////////////////
1313 // 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
1314 ENTRY(taken_branch_trap)
1315 DBG_FAULT(35)
1316 FAULT(35)
1317 END(taken_branch_trap)
1319 .org ia64_ivt+0x6000
1320 /////////////////////////////////////////////////////////////////////////////////////////
1321 // 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
1322 ENTRY(single_step_trap)
1323 DBG_FAULT(36)
1324 FAULT(36)
1325 END(single_step_trap)
1327 .org ia64_ivt+0x6100
1328 /////////////////////////////////////////////////////////////////////////////////////////
1329 // 0x6100 Entry 37 (size 16 bundles) Reserved
1330 DBG_FAULT(37)
1331 FAULT(37)
1333 .org ia64_ivt+0x6200
1334 /////////////////////////////////////////////////////////////////////////////////////////
1335 // 0x6200 Entry 38 (size 16 bundles) Reserved
1336 DBG_FAULT(38)
1337 FAULT(38)
1339 .org ia64_ivt+0x6300
1340 /////////////////////////////////////////////////////////////////////////////////////////
1341 // 0x6300 Entry 39 (size 16 bundles) Reserved
1342 DBG_FAULT(39)
1343 FAULT(39)
1345 .org ia64_ivt+0x6400
1346 /////////////////////////////////////////////////////////////////////////////////////////
1347 // 0x6400 Entry 40 (size 16 bundles) Reserved
1348 DBG_FAULT(40)
1349 FAULT(40)
1351 .org ia64_ivt+0x6500
1352 /////////////////////////////////////////////////////////////////////////////////////////
1353 // 0x6500 Entry 41 (size 16 bundles) Reserved
1354 DBG_FAULT(41)
1355 FAULT(41)
1357 .org ia64_ivt+0x6600
1358 /////////////////////////////////////////////////////////////////////////////////////////
1359 // 0x6600 Entry 42 (size 16 bundles) Reserved
1360 DBG_FAULT(42)
1361 FAULT(42)
1363 .org ia64_ivt+0x6700
1364 /////////////////////////////////////////////////////////////////////////////////////////
1365 // 0x6700 Entry 43 (size 16 bundles) Reserved
1366 DBG_FAULT(43)
1367 FAULT(43)
1369 .org ia64_ivt+0x6800
1370 /////////////////////////////////////////////////////////////////////////////////////////
1371 // 0x6800 Entry 44 (size 16 bundles) Reserved
1372 DBG_FAULT(44)
1373 FAULT(44)
1375 .org ia64_ivt+0x6900
1376 /////////////////////////////////////////////////////////////////////////////////////////
1377 // 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77)
1378 ENTRY(ia32_exception)
1379 DBG_FAULT(45)
1380 FAULT(45)
1381 END(ia32_exception)
1383 .org ia64_ivt+0x6a00
1384 /////////////////////////////////////////////////////////////////////////////////////////
1385 // 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71)
1386 ENTRY(ia32_intercept)
1387 DBG_FAULT(46)
1388 FAULT(46)
1389 END(ia32_intercept)
1391 .org ia64_ivt+0x6b00
1392 /////////////////////////////////////////////////////////////////////////////////////////
1393 // 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74)
1394 ENTRY(ia32_interrupt)
1395 DBG_FAULT(47)
1396 FAULT(47)
1397 END(ia32_interrupt)
1399 .org ia64_ivt+0x6c00
1400 /////////////////////////////////////////////////////////////////////////////////////////
1401 // 0x6c00 Entry 48 (size 16 bundles) Reserved
1402 DBG_FAULT(48)
1403 FAULT(48)
1405 .org ia64_ivt+0x6d00
1406 /////////////////////////////////////////////////////////////////////////////////////////
1407 // 0x6d00 Entry 49 (size 16 bundles) Reserved
1408 DBG_FAULT(49)
1409 FAULT(49)
1411 .org ia64_ivt+0x6e00
1412 /////////////////////////////////////////////////////////////////////////////////////////
1413 // 0x6e00 Entry 50 (size 16 bundles) Reserved
1414 DBG_FAULT(50)
1415 FAULT(50)
1417 .org ia64_ivt+0x6f00
1418 /////////////////////////////////////////////////////////////////////////////////////////
1419 // 0x6f00 Entry 51 (size 16 bundles) Reserved
1420 DBG_FAULT(51)
1421 FAULT(51)
1423 .org ia64_ivt+0x7000
1424 /////////////////////////////////////////////////////////////////////////////////////////
1425 // 0x7000 Entry 52 (size 16 bundles) Reserved
1426 DBG_FAULT(52)
1427 FAULT(52)
1429 .org ia64_ivt+0x7100
1430 /////////////////////////////////////////////////////////////////////////////////////////
1431 // 0x7100 Entry 53 (size 16 bundles) Reserved
1432 DBG_FAULT(53)
1433 FAULT(53)
1435 .org ia64_ivt+0x7200
1436 /////////////////////////////////////////////////////////////////////////////////////////
1437 // 0x7200 Entry 54 (size 16 bundles) Reserved
1438 DBG_FAULT(54)
1439 FAULT(54)
1441 .org ia64_ivt+0x7300
1442 /////////////////////////////////////////////////////////////////////////////////////////
1443 // 0x7300 Entry 55 (size 16 bundles) Reserved
1444 DBG_FAULT(55)
1445 FAULT(55)
1447 .org ia64_ivt+0x7400
1448 /////////////////////////////////////////////////////////////////////////////////////////
1449 // 0x7400 Entry 56 (size 16 bundles) Reserved
1450 DBG_FAULT(56)
1451 FAULT(56)
1453 .org ia64_ivt+0x7500
1454 /////////////////////////////////////////////////////////////////////////////////////////
1455 // 0x7500 Entry 57 (size 16 bundles) Reserved
1456 DBG_FAULT(57)
1457 FAULT(57)
1459 .org ia64_ivt+0x7600
1460 /////////////////////////////////////////////////////////////////////////////////////////
1461 // 0x7600 Entry 58 (size 16 bundles) Reserved
1462 DBG_FAULT(58)
1463 FAULT(58)
1465 .org ia64_ivt+0x7700
1466 /////////////////////////////////////////////////////////////////////////////////////////
1467 // 0x7700 Entry 59 (size 16 bundles) Reserved
1468 DBG_FAULT(59)
1469 FAULT(59)
1471 .org ia64_ivt+0x7800
1472 /////////////////////////////////////////////////////////////////////////////////////////
1473 // 0x7800 Entry 60 (size 16 bundles) Reserved
1474 DBG_FAULT(60)
1475 FAULT(60)
1477 .org ia64_ivt+0x7900
1478 /////////////////////////////////////////////////////////////////////////////////////////
1479 // 0x7900 Entry 61 (size 16 bundles) Reserved
1480 DBG_FAULT(61)
1481 FAULT(61)
1483 .org ia64_ivt+0x7a00
1484 /////////////////////////////////////////////////////////////////////////////////////////
1485 // 0x7a00 Entry 62 (size 16 bundles) Reserved
1486 DBG_FAULT(62)
1487 FAULT(62)
1489 .org ia64_ivt+0x7b00
1490 /////////////////////////////////////////////////////////////////////////////////////////
1491 // 0x7b00 Entry 63 (size 16 bundles) Reserved
1492 DBG_FAULT(63)
1493 FAULT(63)
1495 .org ia64_ivt+0x7c00
1496 /////////////////////////////////////////////////////////////////////////////////////////
1497 // 0x7c00 Entry 64 (size 16 bundles) Reserved
1498 DBG_FAULT(64)
1499 FAULT(64)
1501 .org ia64_ivt+0x7d00
1502 /////////////////////////////////////////////////////////////////////////////////////////
1503 // 0x7d00 Entry 65 (size 16 bundles) Reserved
1504 DBG_FAULT(65)
1505 FAULT(65)
1507 .org ia64_ivt+0x7e00
1508 /////////////////////////////////////////////////////////////////////////////////////////
1509 // 0x7e00 Entry 66 (size 16 bundles) Reserved
1510 DBG_FAULT(66)
1511 FAULT(66)
1513 .org ia64_ivt+0x7f00
1514 /////////////////////////////////////////////////////////////////////////////////////////
1515 // 0x7f00 Entry 67 (size 16 bundles) Reserved
1516 DBG_FAULT(67)
1517 FAULT(67)
1519 //-----------------------------------------------------------------------------------
1520 // call do_page_fault (predicates are in r31, psr.dt may be off, r16 is faulting address)
1521 ENTRY(page_fault)
1522 SSM_PSR_DT_AND_SRLZ_I
1523 ;;
1524 SAVE_MIN_WITH_COVER
1525 alloc r15=ar.pfs,0,0,3,0
1526 MOV_FROM_IFA(out0)
1527 MOV_FROM_ISR(out1)
1528 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r14, r3)
1529 adds r3=8,r2 // set up second base pointer
1530 SSM_PSR_I(p15, p15, r14) // restore psr.i
1531 movl r14=ia64_leave_kernel
1532 ;;
1533 SAVE_REST
1534 mov rp=r14
1535 ;;
1536 adds out2=16,r12 // out2 = pointer to pt_regs
1537 br.call.sptk.many b6=ia64_do_page_fault // ignore return address
1538 END(page_fault)
1540 ENTRY(non_syscall)
1541 mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER
1542 ;;
1543 SAVE_MIN_WITH_COVER
1545 // There is no particular reason for this code to be here, other than that
1546 // there happens to be space here that would go unused otherwise. If this
1547 // fault ever gets "unreserved", simply moved the following code to a more
1548 // suitable spot...
1550 alloc r14=ar.pfs,0,0,2,0
1551 MOV_FROM_IIM(out0)
1552 add out1=16,sp
1553 adds r3=8,r2 // set up second base pointer for SAVE_REST
1555 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r15, r24)
1556 // guarantee that interruption collection is on
1557 SSM_PSR_I(p15, p15, r15) // restore psr.i
1558 movl r15=ia64_leave_kernel
1559 ;;
1560 SAVE_REST
1561 mov rp=r15
1562 ;;
1563 br.call.sptk.many b6=ia64_bad_break // avoid WAW on CFM and ignore return addr
1564 END(non_syscall)
1566 ENTRY(__interrupt)
1567 DBG_FAULT(12)
1568 mov r31=pr // prepare to save predicates
1569 ;;
1570 SAVE_MIN_WITH_COVER // uses r31; defines r2 and r3
1571 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r14)
1572 // ensure everybody knows psr.ic is back on
1573 adds r3=8,r2 // set up second base pointer for SAVE_REST
1574 ;;
1575 SAVE_REST
1576 ;;
1577 MCA_RECOVER_RANGE(interrupt)
1578 alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group
1579 MOV_FROM_IVR(out0, r8) // pass cr.ivr as first arg
1580 add out1=16,sp // pass pointer to pt_regs as second arg
1581 ;;
1582 srlz.d // make sure we see the effect of cr.ivr
1583 movl r14=ia64_leave_kernel
1584 ;;
1585 mov rp=r14
1586 br.call.sptk.many b6=ia64_handle_irq
1587 END(__interrupt)
1589 /*
1590 * There is no particular reason for this code to be here, other than that
1591 * there happens to be space here that would go unused otherwise. If this
1592 * fault ever gets "unreserved", simply moved the following code to a more
1593 * suitable spot...
1594 */
1596 ENTRY(dispatch_unaligned_handler)
1597 SAVE_MIN_WITH_COVER
1598 ;;
1599 alloc r14=ar.pfs,0,0,2,0 // now it's safe (must be first in insn group!)
1600 MOV_FROM_IFA(out0)
1601 adds out1=16,sp
1603 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24)
1604 // guarantee that interruption collection is on
1605 SSM_PSR_I(p15, p15, r3) // restore psr.i
1606 adds r3=8,r2 // set up second base pointer
1607 ;;
1608 SAVE_REST
1609 movl r14=ia64_leave_kernel
1610 ;;
1611 mov rp=r14
1612 br.sptk.many ia64_prepare_handle_unaligned
1613 END(dispatch_unaligned_handler)
1615 /*
1616 * There is no particular reason for this code to be here, other than that
1617 * there happens to be space here that would go unused otherwise. If this
1618 * fault ever gets "unreserved", simply moved the following code to a more
1619 * suitable spot...
1620 */
1622 ENTRY(dispatch_to_fault_handler)
1623 /*
1624 * Input:
1625 * psr.ic: off
1626 * r19: fault vector number (e.g., 24 for General Exception)
1627 * r31: contains saved predicates (pr)
1628 */
1629 SAVE_MIN_WITH_COVER_R19
1630 alloc r14=ar.pfs,0,0,5,0
1631 MOV_FROM_ISR(out1)
1632 MOV_FROM_IFA(out2)
1633 MOV_FROM_IIM(out3)
1634 MOV_FROM_ITIR(out4)
1635 ;;
1636 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, out0)
1637 // guarantee that interruption collection is on
1638 mov out0=r15
1639 ;;
1640 SSM_PSR_I(p15, p15, r3) // restore psr.i
1641 adds r3=8,r2 // set up second base pointer for SAVE_REST
1642 ;;
1643 SAVE_REST
1644 movl r14=ia64_leave_kernel
1645 ;;
1646 mov rp=r14
1647 br.call.sptk.many b6=ia64_fault
1648 END(dispatch_to_fault_handler)
1650 /*
1651 * Squatting in this space ...
1652 *
1653 * This special case dispatcher for illegal operation faults allows preserved
1654 * registers to be modified through a callback function (asm only) that is handed
1655 * back from the fault handler in r8. Up to three arguments can be passed to the
1656 * callback function by returning an aggregate with the callback as its first
1657 * element, followed by the arguments.
1658 */
1659 ENTRY(dispatch_illegal_op_fault)
1660 .prologue
1661 .body
1662 SAVE_MIN_WITH_COVER
1663 SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24)
1664 // guarantee that interruption collection is on
1665 ;;
1666 SSM_PSR_I(p15, p15, r3) // restore psr.i
1667 adds r3=8,r2 // set up second base pointer for SAVE_REST
1668 ;;
1669 alloc r14=ar.pfs,0,0,1,0 // must be first in insn group
1670 mov out0=ar.ec
1671 ;;
1672 SAVE_REST
1673 PT_REGS_UNWIND_INFO(0)
1674 ;;
1675 br.call.sptk.many rp=ia64_illegal_op_fault
1676 .ret0: ;;
1677 alloc r14=ar.pfs,0,0,3,0 // must be first in insn group
1678 mov out0=r9
1679 mov out1=r10
1680 mov out2=r11
1681 movl r15=ia64_leave_kernel
1682 ;;
1683 mov rp=r15
1684 mov b6=r8
1685 ;;
1686 cmp.ne p6,p0=0,r8
1687 (p6) br.call.dpnt.many b6=b6 // call returns to ia64_leave_kernel
1688 br.sptk.many ia64_leave_kernel
1689 END(dispatch_illegal_op_fault)