?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.13.16
[linux/fpc-iii.git] / arch / sparc / kernel / traps_64.c
blobad31af1dd726575986c87bba514b345f291342ec
1 /* arch/sparc64/kernel/traps.c
2 *
3 * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net)
4 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
5 */
6
7 /*
8 * I like traps on v9, :))))
9 */
10
11 #include <linux/extable.h>
12 #include <linux/sched/mm.h>
13 #include <linux/sched/debug.h>
14 #include <linux/linkage.h>
15 #include <linux/kernel.h>
16 #include <linux/signal.h>
17 #include <linux/smp.h>
18 #include <linux/mm.h>
19 #include <linux/init.h>
20 #include <linux/kdebug.h>
21 #include <linux/ftrace.h>
22 #include <linux/reboot.h>
23 #include <linux/gfp.h>
24 #include <linux/context_tracking.h>
25
26 #include <asm/smp.h>
27 #include <asm/delay.h>
28 #include <asm/ptrace.h>
29 #include <asm/oplib.h>
30 #include <asm/page.h>
31 #include <asm/pgtable.h>
32 #include <asm/unistd.h>
33 #include <linux/uaccess.h>
34 #include <asm/fpumacro.h>
35 #include <asm/lsu.h>
36 #include <asm/dcu.h>
37 #include <asm/estate.h>
38 #include <asm/chafsr.h>
39 #include <asm/sfafsr.h>
40 #include <asm/psrcompat.h>
41 #include <asm/processor.h>
42 #include <asm/timer.h>
43 #include <asm/head.h>
44 #include <asm/prom.h>
45 #include <asm/memctrl.h>
46 #include <asm/cacheflush.h>
47 #include <asm/setup.h>
48
49 #include "entry.h"
50 #include "kernel.h"
51 #include "kstack.h"
52
53 /* When an irrecoverable trap occurs at tl > 0, the trap entry
54 * code logs the trap state registers at every level in the trap
55 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
56 * is as follows:
57 */
58 struct tl1_traplog {
59 struct {
60 unsigned long tstate;
61 unsigned long tpc;
62 unsigned long tnpc;
63 unsigned long tt;
64 } trapstack[4];
65 unsigned long tl;
66 };
67
68 static void dump_tl1_traplog(struct tl1_traplog *p)
69 {
70 int i, limit;
71
72 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
73 "dumping track stack.\n", p->tl);
74
75 limit = (tlb_type == hypervisor) ? 2 : 4;
76 for (i = 0; i < limit; i++) {
77 printk(KERN_EMERG
78 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
79 "TNPC[%016lx] TT[%lx]\n",
80 i + 1,
81 p->trapstack[i].tstate, p->trapstack[i].tpc,
82 p->trapstack[i].tnpc, p->trapstack[i].tt);
83 printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc);
84 }
85 }
86
87 void bad_trap(struct pt_regs *regs, long lvl)
88 {
89 char buffer[36];
90 siginfo_t info;
91
92 if (notify_die(DIE_TRAP, "bad trap", regs,
93 0, lvl, SIGTRAP) == NOTIFY_STOP)
94 return;
95
96 if (lvl < 0x100) {
97 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
98 die_if_kernel(buffer, regs);
99 }
100
101 lvl -= 0x100;
102 if (regs->tstate & TSTATE_PRIV) {
103 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
104 die_if_kernel(buffer, regs);
105 }
106 if (test_thread_flag(TIF_32BIT)) {
107 regs->tpc &= 0xffffffff;
108 regs->tnpc &= 0xffffffff;
109 }
110 info.si_signo = SIGILL;
111 info.si_errno = 0;
112 info.si_code = ILL_ILLTRP;
113 info.si_addr = (void __user *)regs->tpc;
114 info.si_trapno = lvl;
115 force_sig_info(SIGILL, &info, current);
116 }
117
118 void bad_trap_tl1(struct pt_regs *regs, long lvl)
119 {
120 char buffer[36];
121
122 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
123 0, lvl, SIGTRAP) == NOTIFY_STOP)
124 return;
125
126 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
127
128 sprintf (buffer, "Bad trap %lx at tl>0", lvl);
129 die_if_kernel (buffer, regs);
130 }
131
132 #ifdef CONFIG_DEBUG_BUGVERBOSE
133 void do_BUG(const char *file, int line)
134 {
135 bust_spinlocks(1);
136 printk("kernel BUG at %s:%d!\n", file, line);
137 }
138 EXPORT_SYMBOL(do_BUG);
139 #endif
140
141 static DEFINE_SPINLOCK(dimm_handler_lock);
142 static dimm_printer_t dimm_handler;
143
144 static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen)
145 {
146 unsigned long flags;
147 int ret = -ENODEV;
148
149 spin_lock_irqsave(&dimm_handler_lock, flags);
150 if (dimm_handler) {
151 ret = dimm_handler(synd_code, paddr, buf, buflen);
152 } else if (tlb_type == spitfire) {
153 if (prom_getunumber(synd_code, paddr, buf, buflen) == -1)
154 ret = -EINVAL;
155 else
156 ret = 0;
157 } else
158 ret = -ENODEV;
159 spin_unlock_irqrestore(&dimm_handler_lock, flags);
160
161 return ret;
162 }
163
164 int register_dimm_printer(dimm_printer_t func)
165 {
166 unsigned long flags;
167 int ret = 0;
168
169 spin_lock_irqsave(&dimm_handler_lock, flags);
170 if (!dimm_handler)
171 dimm_handler = func;
172 else
173 ret = -EEXIST;
174 spin_unlock_irqrestore(&dimm_handler_lock, flags);
175
176 return ret;
177 }
178 EXPORT_SYMBOL_GPL(register_dimm_printer);
179
180 void unregister_dimm_printer(dimm_printer_t func)
181 {
182 unsigned long flags;
183
184 spin_lock_irqsave(&dimm_handler_lock, flags);
185 if (dimm_handler == func)
186 dimm_handler = NULL;
187 spin_unlock_irqrestore(&dimm_handler_lock, flags);
188 }
189 EXPORT_SYMBOL_GPL(unregister_dimm_printer);
190
191 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
192 {
193 enum ctx_state prev_state = exception_enter();
194 siginfo_t info;
195
196 if (notify_die(DIE_TRAP, "instruction access exception", regs,
197 0, 0x8, SIGTRAP) == NOTIFY_STOP)
198 goto out;
199
200 if (regs->tstate & TSTATE_PRIV) {
201 printk("spitfire_insn_access_exception: SFSR[%016lx] "
202 "SFAR[%016lx], going.\n", sfsr, sfar);
203 die_if_kernel("Iax", regs);
204 }
205 if (test_thread_flag(TIF_32BIT)) {
206 regs->tpc &= 0xffffffff;
207 regs->tnpc &= 0xffffffff;
208 }
209 info.si_signo = SIGSEGV;
210 info.si_errno = 0;
211 info.si_code = SEGV_MAPERR;
212 info.si_addr = (void __user *)regs->tpc;
213 info.si_trapno = 0;
214 force_sig_info(SIGSEGV, &info, current);
215 out:
216 exception_exit(prev_state);
217 }
218
219 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
220 {
221 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
222 0, 0x8, SIGTRAP) == NOTIFY_STOP)
223 return;
224
225 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
226 spitfire_insn_access_exception(regs, sfsr, sfar);
227 }
228
229 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
230 {
231 unsigned short type = (type_ctx >> 16);
232 unsigned short ctx = (type_ctx & 0xffff);
233 siginfo_t info;
234
235 if (notify_die(DIE_TRAP, "instruction access exception", regs,
236 0, 0x8, SIGTRAP) == NOTIFY_STOP)
237 return;
238
239 if (regs->tstate & TSTATE_PRIV) {
240 printk("sun4v_insn_access_exception: ADDR[%016lx] "
241 "CTX[%04x] TYPE[%04x], going.\n",
242 addr, ctx, type);
243 die_if_kernel("Iax", regs);
244 }
245
246 if (test_thread_flag(TIF_32BIT)) {
247 regs->tpc &= 0xffffffff;
248 regs->tnpc &= 0xffffffff;
249 }
250 info.si_signo = SIGSEGV;
251 info.si_errno = 0;
252 info.si_code = SEGV_MAPERR;
253 info.si_addr = (void __user *) addr;
254 info.si_trapno = 0;
255 force_sig_info(SIGSEGV, &info, current);
256 }
257
258 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
259 {
260 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
261 0, 0x8, SIGTRAP) == NOTIFY_STOP)
262 return;
263
264 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
265 sun4v_insn_access_exception(regs, addr, type_ctx);
266 }
267
268 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
269 {
270 enum ctx_state prev_state = exception_enter();
271 siginfo_t info;
272
273 if (notify_die(DIE_TRAP, "data access exception", regs,
274 0, 0x30, SIGTRAP) == NOTIFY_STOP)
275 goto out;
276
277 if (regs->tstate & TSTATE_PRIV) {
278 /* Test if this comes from uaccess places. */
279 const struct exception_table_entry *entry;
280
281 entry = search_exception_tables(regs->tpc);
282 if (entry) {
283 /* Ouch, somebody is trying VM hole tricks on us... */
284 #ifdef DEBUG_EXCEPTIONS
285 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
286 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
287 regs->tpc, entry->fixup);
288 #endif
289 regs->tpc = entry->fixup;
290 regs->tnpc = regs->tpc + 4;
291 goto out;
292 }
293 /* Shit... */
294 printk("spitfire_data_access_exception: SFSR[%016lx] "
295 "SFAR[%016lx], going.\n", sfsr, sfar);
296 die_if_kernel("Dax", regs);
297 }
298
299 info.si_signo = SIGSEGV;
300 info.si_errno = 0;
301 info.si_code = SEGV_MAPERR;
302 info.si_addr = (void __user *)sfar;
303 info.si_trapno = 0;
304 force_sig_info(SIGSEGV, &info, current);
305 out:
306 exception_exit(prev_state);
307 }
308
309 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
310 {
311 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
312 0, 0x30, SIGTRAP) == NOTIFY_STOP)
313 return;
314
315 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
316 spitfire_data_access_exception(regs, sfsr, sfar);
317 }
318
319 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
320 {
321 unsigned short type = (type_ctx >> 16);
322 unsigned short ctx = (type_ctx & 0xffff);
323 siginfo_t info;
324
325 if (notify_die(DIE_TRAP, "data access exception", regs,
326 0, 0x8, SIGTRAP) == NOTIFY_STOP)
327 return;
328
329 if (regs->tstate & TSTATE_PRIV) {
330 /* Test if this comes from uaccess places. */
331 const struct exception_table_entry *entry;
332
333 entry = search_exception_tables(regs->tpc);
334 if (entry) {
335 /* Ouch, somebody is trying VM hole tricks on us... */
336 #ifdef DEBUG_EXCEPTIONS
337 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
338 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
339 regs->tpc, entry->fixup);
340 #endif
341 regs->tpc = entry->fixup;
342 regs->tnpc = regs->tpc + 4;
343 return;
344 }
345 printk("sun4v_data_access_exception: ADDR[%016lx] "
346 "CTX[%04x] TYPE[%04x], going.\n",
347 addr, ctx, type);
348 die_if_kernel("Dax", regs);
349 }
350
351 if (test_thread_flag(TIF_32BIT)) {
352 regs->tpc &= 0xffffffff;
353 regs->tnpc &= 0xffffffff;
354 }
355 info.si_signo = SIGSEGV;
356 info.si_errno = 0;
357 info.si_code = SEGV_MAPERR;
358 info.si_addr = (void __user *) addr;
359 info.si_trapno = 0;
360 force_sig_info(SIGSEGV, &info, current);
361 }
362
363 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
364 {
365 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
366 0, 0x8, SIGTRAP) == NOTIFY_STOP)
367 return;
368
369 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
370 sun4v_data_access_exception(regs, addr, type_ctx);
371 }
372
373 #ifdef CONFIG_PCI
374 #include "pci_impl.h"
375 #endif
376
377 /* When access exceptions happen, we must do this. */
378 static void spitfire_clean_and_reenable_l1_caches(void)
379 {
380 unsigned long va;
381
382 if (tlb_type != spitfire)
383 BUG();
384
385 /* Clean 'em. */
386 for (va = 0; va < (PAGE_SIZE << 1); va += 32) {
387 spitfire_put_icache_tag(va, 0x0);
388 spitfire_put_dcache_tag(va, 0x0);
389 }
390
391 /* Re-enable in LSU. */
392 __asm__ __volatile__("flush %%g6\n\t"
393 "membar #Sync\n\t"
394 "stxa %0, [%%g0] %1\n\t"
395 "membar #Sync"
396 : /* no outputs */
397 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
398 LSU_CONTROL_IM | LSU_CONTROL_DM),
399 "i" (ASI_LSU_CONTROL)
400 : "memory");
401 }
402
403 static void spitfire_enable_estate_errors(void)
404 {
405 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
406 "membar #Sync"
407 : /* no outputs */
408 : "r" (ESTATE_ERR_ALL),
409 "i" (ASI_ESTATE_ERROR_EN));
410 }
411
412 static char ecc_syndrome_table[] = {
413 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
414 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
415 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
416 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
417 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
418 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
419 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
420 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
421 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
422 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
423 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
424 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
425 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
426 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
427 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
428 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
429 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
430 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
431 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
432 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
433 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
434 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
435 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
436 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
437 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
438 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
439 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
440 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
441 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
442 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
443 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
444 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
445 };
446
447 static char *syndrome_unknown = "<Unknown>";
448
449 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
450 {
451 unsigned short scode;
452 char memmod_str[64], *p;
453
454 if (udbl & bit) {
455 scode = ecc_syndrome_table[udbl & 0xff];
456 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
457 p = syndrome_unknown;
458 else
459 p = memmod_str;
460 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
461 "Memory Module \"%s\"\n",
462 smp_processor_id(), scode, p);
463 }
464
465 if (udbh & bit) {
466 scode = ecc_syndrome_table[udbh & 0xff];
467 if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0)
468 p = syndrome_unknown;
469 else
470 p = memmod_str;
471 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
472 "Memory Module \"%s\"\n",
473 smp_processor_id(), scode, p);
474 }
475
476 }
477
478 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
479 {
480
481 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
482 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
483 smp_processor_id(), afsr, afar, udbl, udbh, tl1);
484
485 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
486
487 /* We always log it, even if someone is listening for this
488 * trap.
489 */
490 notify_die(DIE_TRAP, "Correctable ECC Error", regs,
491 0, TRAP_TYPE_CEE, SIGTRAP);
492
493 /* The Correctable ECC Error trap does not disable I/D caches. So
494 * we only have to restore the ESTATE Error Enable register.
495 */
496 spitfire_enable_estate_errors();
497 }
498
499 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
500 {
501 siginfo_t info;
502
503 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
504 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
505 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
506
507 /* XXX add more human friendly logging of the error status
508 * XXX as is implemented for cheetah
509 */
510
511 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
512
513 /* We always log it, even if someone is listening for this
514 * trap.
515 */
516 notify_die(DIE_TRAP, "Uncorrectable Error", regs,
517 0, tt, SIGTRAP);
518
519 if (regs->tstate & TSTATE_PRIV) {
520 if (tl1)
521 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
522 die_if_kernel("UE", regs);
523 }
524
525 /* XXX need more intelligent processing here, such as is implemented
526 * XXX for cheetah errors, in fact if the E-cache still holds the
527 * XXX line with bad parity this will loop
528 */
529
530 spitfire_clean_and_reenable_l1_caches();
531 spitfire_enable_estate_errors();
532
533 if (test_thread_flag(TIF_32BIT)) {
534 regs->tpc &= 0xffffffff;
535 regs->tnpc &= 0xffffffff;
536 }
537 info.si_signo = SIGBUS;
538 info.si_errno = 0;
539 info.si_code = BUS_OBJERR;
540 info.si_addr = (void *)0;
541 info.si_trapno = 0;
542 force_sig_info(SIGBUS, &info, current);
543 }
544
545 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
546 {
547 unsigned long afsr, tt, udbh, udbl;
548 int tl1;
549
550 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
551 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
552 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
553 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
554 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
555
556 #ifdef CONFIG_PCI
557 if (tt == TRAP_TYPE_DAE &&
558 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
559 spitfire_clean_and_reenable_l1_caches();
560 spitfire_enable_estate_errors();
561
562 pci_poke_faulted = 1;
563 regs->tnpc = regs->tpc + 4;
564 return;
565 }
566 #endif
567
568 if (afsr & SFAFSR_UE)
569 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
570
571 if (tt == TRAP_TYPE_CEE) {
572 /* Handle the case where we took a CEE trap, but ACK'd
573 * only the UE state in the UDB error registers.
574 */
575 if (afsr & SFAFSR_UE) {
576 if (udbh & UDBE_CE) {
577 __asm__ __volatile__(
578 "stxa %0, [%1] %2\n\t"
579 "membar #Sync"
580 : /* no outputs */
581 : "r" (udbh & UDBE_CE),
582 "r" (0x0), "i" (ASI_UDB_ERROR_W));
583 }
584 if (udbl & UDBE_CE) {
585 __asm__ __volatile__(
586 "stxa %0, [%1] %2\n\t"
587 "membar #Sync"
588 : /* no outputs */
589 : "r" (udbl & UDBE_CE),
590 "r" (0x18), "i" (ASI_UDB_ERROR_W));
591 }
592 }
593
594 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
595 }
596 }
597
598 int cheetah_pcache_forced_on;
599
600 void cheetah_enable_pcache(void)
601 {
602 unsigned long dcr;
603
604 printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
605 smp_processor_id());
606
607 __asm__ __volatile__("ldxa [%%g0] %1, %0"
608 : "=r" (dcr)
609 : "i" (ASI_DCU_CONTROL_REG));
610 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
611 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
612 "membar #Sync"
613 : /* no outputs */
614 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
615 }
616
617 /* Cheetah error trap handling. */
618 static unsigned long ecache_flush_physbase;
619 static unsigned long ecache_flush_linesize;
620 static unsigned long ecache_flush_size;
621
622 /* This table is ordered in priority of errors and matches the
623 * AFAR overwrite policy as well.
624 */
625
626 struct afsr_error_table {
627 unsigned long mask;
628 const char *name;
629 };
630
631 static const char CHAFSR_PERR_msg[] =
632 "System interface protocol error";
633 static const char CHAFSR_IERR_msg[] =
634 "Internal processor error";
635 static const char CHAFSR_ISAP_msg[] =
636 "System request parity error on incoming address";
637 static const char CHAFSR_UCU_msg[] =
638 "Uncorrectable E-cache ECC error for ifetch/data";
639 static const char CHAFSR_UCC_msg[] =
640 "SW Correctable E-cache ECC error for ifetch/data";
641 static const char CHAFSR_UE_msg[] =
642 "Uncorrectable system bus data ECC error for read";
643 static const char CHAFSR_EDU_msg[] =
644 "Uncorrectable E-cache ECC error for stmerge/blkld";
645 static const char CHAFSR_EMU_msg[] =
646 "Uncorrectable system bus MTAG error";
647 static const char CHAFSR_WDU_msg[] =
648 "Uncorrectable E-cache ECC error for writeback";
649 static const char CHAFSR_CPU_msg[] =
650 "Uncorrectable ECC error for copyout";
651 static const char CHAFSR_CE_msg[] =
652 "HW corrected system bus data ECC error for read";
653 static const char CHAFSR_EDC_msg[] =
654 "HW corrected E-cache ECC error for stmerge/blkld";
655 static const char CHAFSR_EMC_msg[] =
656 "HW corrected system bus MTAG ECC error";
657 static const char CHAFSR_WDC_msg[] =
658 "HW corrected E-cache ECC error for writeback";
659 static const char CHAFSR_CPC_msg[] =
660 "HW corrected ECC error for copyout";
661 static const char CHAFSR_TO_msg[] =
662 "Unmapped error from system bus";
663 static const char CHAFSR_BERR_msg[] =
664 "Bus error response from system bus";
665 static const char CHAFSR_IVC_msg[] =
666 "HW corrected system bus data ECC error for ivec read";
667 static const char CHAFSR_IVU_msg[] =
668 "Uncorrectable system bus data ECC error for ivec read";
669 static struct afsr_error_table __cheetah_error_table[] = {
670 { CHAFSR_PERR, CHAFSR_PERR_msg },
671 { CHAFSR_IERR, CHAFSR_IERR_msg },
672 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
673 { CHAFSR_UCU, CHAFSR_UCU_msg },
674 { CHAFSR_UCC, CHAFSR_UCC_msg },
675 { CHAFSR_UE, CHAFSR_UE_msg },
676 { CHAFSR_EDU, CHAFSR_EDU_msg },
677 { CHAFSR_EMU, CHAFSR_EMU_msg },
678 { CHAFSR_WDU, CHAFSR_WDU_msg },
679 { CHAFSR_CPU, CHAFSR_CPU_msg },
680 { CHAFSR_CE, CHAFSR_CE_msg },
681 { CHAFSR_EDC, CHAFSR_EDC_msg },
682 { CHAFSR_EMC, CHAFSR_EMC_msg },
683 { CHAFSR_WDC, CHAFSR_WDC_msg },
684 { CHAFSR_CPC, CHAFSR_CPC_msg },
685 { CHAFSR_TO, CHAFSR_TO_msg },
686 { CHAFSR_BERR, CHAFSR_BERR_msg },
687 /* These two do not update the AFAR. */
688 { CHAFSR_IVC, CHAFSR_IVC_msg },
689 { CHAFSR_IVU, CHAFSR_IVU_msg },
690 { 0, NULL },
691 };
692 static const char CHPAFSR_DTO_msg[] =
693 "System bus unmapped error for prefetch/storequeue-read";
694 static const char CHPAFSR_DBERR_msg[] =
695 "System bus error for prefetch/storequeue-read";
696 static const char CHPAFSR_THCE_msg[] =
697 "Hardware corrected E-cache Tag ECC error";
698 static const char CHPAFSR_TSCE_msg[] =
699 "SW handled correctable E-cache Tag ECC error";
700 static const char CHPAFSR_TUE_msg[] =
701 "Uncorrectable E-cache Tag ECC error";
702 static const char CHPAFSR_DUE_msg[] =
703 "System bus uncorrectable data ECC error due to prefetch/store-fill";
704 static struct afsr_error_table __cheetah_plus_error_table[] = {
705 { CHAFSR_PERR, CHAFSR_PERR_msg },
706 { CHAFSR_IERR, CHAFSR_IERR_msg },
707 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
708 { CHAFSR_UCU, CHAFSR_UCU_msg },
709 { CHAFSR_UCC, CHAFSR_UCC_msg },
710 { CHAFSR_UE, CHAFSR_UE_msg },
711 { CHAFSR_EDU, CHAFSR_EDU_msg },
712 { CHAFSR_EMU, CHAFSR_EMU_msg },
713 { CHAFSR_WDU, CHAFSR_WDU_msg },
714 { CHAFSR_CPU, CHAFSR_CPU_msg },
715 { CHAFSR_CE, CHAFSR_CE_msg },
716 { CHAFSR_EDC, CHAFSR_EDC_msg },
717 { CHAFSR_EMC, CHAFSR_EMC_msg },
718 { CHAFSR_WDC, CHAFSR_WDC_msg },
719 { CHAFSR_CPC, CHAFSR_CPC_msg },
720 { CHAFSR_TO, CHAFSR_TO_msg },
721 { CHAFSR_BERR, CHAFSR_BERR_msg },
722 { CHPAFSR_DTO, CHPAFSR_DTO_msg },
723 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg },
724 { CHPAFSR_THCE, CHPAFSR_THCE_msg },
725 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg },
726 { CHPAFSR_TUE, CHPAFSR_TUE_msg },
727 { CHPAFSR_DUE, CHPAFSR_DUE_msg },
728 /* These two do not update the AFAR. */
729 { CHAFSR_IVC, CHAFSR_IVC_msg },
730 { CHAFSR_IVU, CHAFSR_IVU_msg },
731 { 0, NULL },
732 };
733 static const char JPAFSR_JETO_msg[] =
734 "System interface protocol error, hw timeout caused";
735 static const char JPAFSR_SCE_msg[] =
736 "Parity error on system snoop results";
737 static const char JPAFSR_JEIC_msg[] =
738 "System interface protocol error, illegal command detected";
739 static const char JPAFSR_JEIT_msg[] =
740 "System interface protocol error, illegal ADTYPE detected";
741 static const char JPAFSR_OM_msg[] =
742 "Out of range memory error has occurred";
743 static const char JPAFSR_ETP_msg[] =
744 "Parity error on L2 cache tag SRAM";
745 static const char JPAFSR_UMS_msg[] =
746 "Error due to unsupported store";
747 static const char JPAFSR_RUE_msg[] =
748 "Uncorrectable ECC error from remote cache/memory";
749 static const char JPAFSR_RCE_msg[] =
750 "Correctable ECC error from remote cache/memory";
751 static const char JPAFSR_BP_msg[] =
752 "JBUS parity error on returned read data";
753 static const char JPAFSR_WBP_msg[] =
754 "JBUS parity error on data for writeback or block store";
755 static const char JPAFSR_FRC_msg[] =
756 "Foreign read to DRAM incurring correctable ECC error";
757 static const char JPAFSR_FRU_msg[] =
758 "Foreign read to DRAM incurring uncorrectable ECC error";
759 static struct afsr_error_table __jalapeno_error_table[] = {
760 { JPAFSR_JETO, JPAFSR_JETO_msg },
761 { JPAFSR_SCE, JPAFSR_SCE_msg },
762 { JPAFSR_JEIC, JPAFSR_JEIC_msg },
763 { JPAFSR_JEIT, JPAFSR_JEIT_msg },
764 { CHAFSR_PERR, CHAFSR_PERR_msg },
765 { CHAFSR_IERR, CHAFSR_IERR_msg },
766 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
767 { CHAFSR_UCU, CHAFSR_UCU_msg },
768 { CHAFSR_UCC, CHAFSR_UCC_msg },
769 { CHAFSR_UE, CHAFSR_UE_msg },
770 { CHAFSR_EDU, CHAFSR_EDU_msg },
771 { JPAFSR_OM, JPAFSR_OM_msg },
772 { CHAFSR_WDU, CHAFSR_WDU_msg },
773 { CHAFSR_CPU, CHAFSR_CPU_msg },
774 { CHAFSR_CE, CHAFSR_CE_msg },
775 { CHAFSR_EDC, CHAFSR_EDC_msg },
776 { JPAFSR_ETP, JPAFSR_ETP_msg },
777 { CHAFSR_WDC, CHAFSR_WDC_msg },
778 { CHAFSR_CPC, CHAFSR_CPC_msg },
779 { CHAFSR_TO, CHAFSR_TO_msg },
780 { CHAFSR_BERR, CHAFSR_BERR_msg },
781 { JPAFSR_UMS, JPAFSR_UMS_msg },
782 { JPAFSR_RUE, JPAFSR_RUE_msg },
783 { JPAFSR_RCE, JPAFSR_RCE_msg },
784 { JPAFSR_BP, JPAFSR_BP_msg },
785 { JPAFSR_WBP, JPAFSR_WBP_msg },
786 { JPAFSR_FRC, JPAFSR_FRC_msg },
787 { JPAFSR_FRU, JPAFSR_FRU_msg },
788 /* These two do not update the AFAR. */
789 { CHAFSR_IVU, CHAFSR_IVU_msg },
790 { 0, NULL },
791 };
792 static struct afsr_error_table *cheetah_error_table;
793 static unsigned long cheetah_afsr_errors;
794
795 struct cheetah_err_info *cheetah_error_log;
796
797 static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
798 {
799 struct cheetah_err_info *p;
800 int cpu = smp_processor_id();
801
802 if (!cheetah_error_log)
803 return NULL;
804
805 p = cheetah_error_log + (cpu * 2);
806 if ((afsr & CHAFSR_TL1) != 0UL)
807 p++;
808
809 return p;
810 }
811
812 extern unsigned int tl0_icpe[], tl1_icpe[];
813 extern unsigned int tl0_dcpe[], tl1_dcpe[];
814 extern unsigned int tl0_fecc[], tl1_fecc[];
815 extern unsigned int tl0_cee[], tl1_cee[];
816 extern unsigned int tl0_iae[], tl1_iae[];
817 extern unsigned int tl0_dae[], tl1_dae[];
818 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
819 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
820 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
821 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
822 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
823
824 void __init cheetah_ecache_flush_init(void)
825 {
826 unsigned long largest_size, smallest_linesize, order, ver;
827 int i, sz;
828
829 /* Scan all cpu device tree nodes, note two values:
830 * 1) largest E-cache size
831 * 2) smallest E-cache line size
832 */
833 largest_size = 0UL;
834 smallest_linesize = ~0UL;
835
836 for (i = 0; i < NR_CPUS; i++) {
837 unsigned long val;
838
839 val = cpu_data(i).ecache_size;
840 if (!val)
841 continue;
842
843 if (val > largest_size)
844 largest_size = val;
845
846 val = cpu_data(i).ecache_line_size;
847 if (val < smallest_linesize)
848 smallest_linesize = val;
849
850 }
851
852 if (largest_size == 0UL || smallest_linesize == ~0UL) {
853 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
854 "parameters.\n");
855 prom_halt();
856 }
857
858 ecache_flush_size = (2 * largest_size);
859 ecache_flush_linesize = smallest_linesize;
860
861 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
862
863 if (ecache_flush_physbase == ~0UL) {
864 prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte "
865 "contiguous physical memory.\n",
866 ecache_flush_size);
867 prom_halt();
868 }
869
870 /* Now allocate error trap reporting scoreboard. */
871 sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
872 for (order = 0; order < MAX_ORDER; order++) {
873 if ((PAGE_SIZE << order) >= sz)
874 break;
875 }
876 cheetah_error_log = (struct cheetah_err_info *)
877 __get_free_pages(GFP_KERNEL, order);
878 if (!cheetah_error_log) {
879 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
880 "error logging scoreboard (%d bytes).\n", sz);
881 prom_halt();
882 }
883 memset(cheetah_error_log, 0, PAGE_SIZE << order);
884
885 /* Mark all AFSRs as invalid so that the trap handler will
886 * log new new information there.
887 */
888 for (i = 0; i < 2 * NR_CPUS; i++)
889 cheetah_error_log[i].afsr = CHAFSR_INVALID;
890
891 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
892 if ((ver >> 32) == __JALAPENO_ID ||
893 (ver >> 32) == __SERRANO_ID) {
894 cheetah_error_table = &__jalapeno_error_table[0];
895 cheetah_afsr_errors = JPAFSR_ERRORS;
896 } else if ((ver >> 32) == 0x003e0015) {
897 cheetah_error_table = &__cheetah_plus_error_table[0];
898 cheetah_afsr_errors = CHPAFSR_ERRORS;
899 } else {
900 cheetah_error_table = &__cheetah_error_table[0];
901 cheetah_afsr_errors = CHAFSR_ERRORS;
902 }
903
904 /* Now patch trap tables. */
905 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
906 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
907 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
908 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
909 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
910 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
911 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
912 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
913 if (tlb_type == cheetah_plus) {
914 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
915 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
916 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
917 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
918 }
919 flushi(PAGE_OFFSET);
920 }
921
922 static void cheetah_flush_ecache(void)
923 {
924 unsigned long flush_base = ecache_flush_physbase;
925 unsigned long flush_linesize = ecache_flush_linesize;
926 unsigned long flush_size = ecache_flush_size;
927
928 __asm__ __volatile__("1: subcc %0, %4, %0\n\t"
929 " bne,pt %%xcc, 1b\n\t"
930 " ldxa [%2 + %0] %3, %%g0\n\t"
931 : "=&r" (flush_size)
932 : "0" (flush_size), "r" (flush_base),
933 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
934 }
935
936 static void cheetah_flush_ecache_line(unsigned long physaddr)
937 {
938 unsigned long alias;
939
940 physaddr &= ~(8UL - 1UL);
941 physaddr = (ecache_flush_physbase +
942 (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
943 alias = physaddr + (ecache_flush_size >> 1UL);
944 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
945 "ldxa [%1] %2, %%g0\n\t"
946 "membar #Sync"
947 : /* no outputs */
948 : "r" (physaddr), "r" (alias),
949 "i" (ASI_PHYS_USE_EC));
950 }
951
952 /* Unfortunately, the diagnostic access to the I-cache tags we need to
953 * use to clear the thing interferes with I-cache coherency transactions.
954 *
955 * So we must only flush the I-cache when it is disabled.
956 */
957 static void __cheetah_flush_icache(void)
958 {
959 unsigned int icache_size, icache_line_size;
960 unsigned long addr;
961
962 icache_size = local_cpu_data().icache_size;
963 icache_line_size = local_cpu_data().icache_line_size;
964
965 /* Clear the valid bits in all the tags. */
966 for (addr = 0; addr < icache_size; addr += icache_line_size) {
967 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
968 "membar #Sync"
969 : /* no outputs */
970 : "r" (addr | (2 << 3)),
971 "i" (ASI_IC_TAG));
972 }
973 }
974
975 static void cheetah_flush_icache(void)
976 {
977 unsigned long dcu_save;
978
979 /* Save current DCU, disable I-cache. */
980 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
981 "or %0, %2, %%g1\n\t"
982 "stxa %%g1, [%%g0] %1\n\t"
983 "membar #Sync"
984 : "=r" (dcu_save)
985 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
986 : "g1");
987
988 __cheetah_flush_icache();
989
990 /* Restore DCU register */
991 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
992 "membar #Sync"
993 : /* no outputs */
994 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
995 }
996
997 static void cheetah_flush_dcache(void)
998 {
999 unsigned int dcache_size, dcache_line_size;
1000 unsigned long addr;
1001
1002 dcache_size = local_cpu_data().dcache_size;
1003 dcache_line_size = local_cpu_data().dcache_line_size;
1004
1005 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1006 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
1007 "membar #Sync"
1008 : /* no outputs */
1009 : "r" (addr), "i" (ASI_DCACHE_TAG));
1010 }
1011 }
1012
1013 /* In order to make the even parity correct we must do two things.
1014 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
1015 * Next, we clear out all 32-bytes of data for that line. Data of
1016 * all-zero + tag parity value of zero == correct parity.
1017 */
1018 static void cheetah_plus_zap_dcache_parity(void)
1019 {
1020 unsigned int dcache_size, dcache_line_size;
1021 unsigned long addr;
1022
1023 dcache_size = local_cpu_data().dcache_size;
1024 dcache_line_size = local_cpu_data().dcache_line_size;
1025
1026 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1027 unsigned long tag = (addr >> 14);
1028 unsigned long line;
1029
1030 __asm__ __volatile__("membar #Sync\n\t"
1031 "stxa %0, [%1] %2\n\t"
1032 "membar #Sync"
1033 : /* no outputs */
1034 : "r" (tag), "r" (addr),
1035 "i" (ASI_DCACHE_UTAG));
1036 for (line = addr; line < addr + dcache_line_size; line += 8)
1037 __asm__ __volatile__("membar #Sync\n\t"
1038 "stxa %%g0, [%0] %1\n\t"
1039 "membar #Sync"
1040 : /* no outputs */
1041 : "r" (line),
1042 "i" (ASI_DCACHE_DATA));
1043 }
1044 }
1045
1046 /* Conversion tables used to frob Cheetah AFSR syndrome values into
1047 * something palatable to the memory controller driver get_unumber
1048 * routine.
1049 */
1050 #define MT0 137
1051 #define MT1 138
1052 #define MT2 139
1053 #define NONE 254
1054 #define MTC0 140
1055 #define MTC1 141
1056 #define MTC2 142
1057 #define MTC3 143
1058 #define C0 128
1059 #define C1 129
1060 #define C2 130
1061 #define C3 131
1062 #define C4 132
1063 #define C5 133
1064 #define C6 134
1065 #define C7 135
1066 #define C8 136
1067 #define M2 144
1068 #define M3 145
1069 #define M4 146
1070 #define M 147
1071 static unsigned char cheetah_ecc_syntab[] = {
1072 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1073 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1074 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1075 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1076 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1077 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1078 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1079 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1080 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1081 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1082 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1083 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1084 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1085 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1086 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1087 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1088 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1089 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1090 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1091 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1092 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1093 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1094 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1095 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1096 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1097 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1098 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1099 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1100 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1101 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1102 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1103 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1104 };
1105 static unsigned char cheetah_mtag_syntab[] = {
1106 NONE, MTC0,
1107 MTC1, NONE,
1108 MTC2, NONE,
1109 NONE, MT0,
1110 MTC3, NONE,
1111 NONE, MT1,
1112 NONE, MT2,
1113 NONE, NONE
1114 };
1115
1116 /* Return the highest priority error conditon mentioned. */
1117 static inline unsigned long cheetah_get_hipri(unsigned long afsr)
1118 {
1119 unsigned long tmp = 0;
1120 int i;
1121
1122 for (i = 0; cheetah_error_table[i].mask; i++) {
1123 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1124 return tmp;
1125 }
1126 return tmp;
1127 }
1128
1129 static const char *cheetah_get_string(unsigned long bit)
1130 {
1131 int i;
1132
1133 for (i = 0; cheetah_error_table[i].mask; i++) {
1134 if ((bit & cheetah_error_table[i].mask) != 0UL)
1135 return cheetah_error_table[i].name;
1136 }
1137 return "???";
1138 }
1139
1140 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1141 unsigned long afsr, unsigned long afar, int recoverable)
1142 {
1143 unsigned long hipri;
1144 char unum[256];
1145
1146 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1147 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1148 afsr, afar,
1149 (afsr & CHAFSR_TL1) ? 1 : 0);
1150 printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1151 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1152 regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1153 printk("%s" "ERROR(%d): ",
1154 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id());
1155 printk("TPC<%pS>\n", (void *) regs->tpc);
1156 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
1157 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1158 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1159 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1160 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1161 (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1162 hipri = cheetah_get_hipri(afsr);
1163 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1164 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1165 hipri, cheetah_get_string(hipri));
1166
1167 /* Try to get unumber if relevant. */
1168 #define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \
1169 CHAFSR_CPC | CHAFSR_CPU | \
1170 CHAFSR_UE | CHAFSR_CE | \
1171 CHAFSR_EDC | CHAFSR_EDU | \
1172 CHAFSR_UCC | CHAFSR_UCU | \
1173 CHAFSR_WDU | CHAFSR_WDC)
1174 #define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU)
1175 if (afsr & ESYND_ERRORS) {
1176 int syndrome;
1177 int ret;
1178
1179 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1180 syndrome = cheetah_ecc_syntab[syndrome];
1181 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1182 if (ret != -1)
1183 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1184 (recoverable ? KERN_WARNING : KERN_CRIT),
1185 smp_processor_id(), unum);
1186 } else if (afsr & MSYND_ERRORS) {
1187 int syndrome;
1188 int ret;
1189
1190 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1191 syndrome = cheetah_mtag_syntab[syndrome];
1192 ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum));
1193 if (ret != -1)
1194 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1195 (recoverable ? KERN_WARNING : KERN_CRIT),
1196 smp_processor_id(), unum);
1197 }
1198
1199 /* Now dump the cache snapshots. */
1200 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n",
1201 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1202 (int) info->dcache_index,
1203 info->dcache_tag,
1204 info->dcache_utag,
1205 info->dcache_stag);
1206 printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1207 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1208 info->dcache_data[0],
1209 info->dcache_data[1],
1210 info->dcache_data[2],
1211 info->dcache_data[3]);
1212 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] "
1213 "u[%016llx] l[%016llx]\n",
1214 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1215 (int) info->icache_index,
1216 info->icache_tag,
1217 info->icache_utag,
1218 info->icache_stag,
1219 info->icache_upper,
1220 info->icache_lower);
1221 printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n",
1222 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1223 info->icache_data[0],
1224 info->icache_data[1],
1225 info->icache_data[2],
1226 info->icache_data[3]);
1227 printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n",
1228 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1229 info->icache_data[4],
1230 info->icache_data[5],
1231 info->icache_data[6],
1232 info->icache_data[7]);
1233 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n",
1234 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1235 (int) info->ecache_index, info->ecache_tag);
1236 printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n",
1237 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1238 info->ecache_data[0],
1239 info->ecache_data[1],
1240 info->ecache_data[2],
1241 info->ecache_data[3]);
1242
1243 afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1244 while (afsr != 0UL) {
1245 unsigned long bit = cheetah_get_hipri(afsr);
1246
1247 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1248 (recoverable ? KERN_WARNING : KERN_CRIT),
1249 bit, cheetah_get_string(bit));
1250
1251 afsr &= ~bit;
1252 }
1253
1254 if (!recoverable)
1255 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1256 }
1257
1258 static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1259 {
1260 unsigned long afsr, afar;
1261 int ret = 0;
1262
1263 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1264 : "=r" (afsr)
1265 : "i" (ASI_AFSR));
1266 if ((afsr & cheetah_afsr_errors) != 0) {
1267 if (logp != NULL) {
1268 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1269 : "=r" (afar)
1270 : "i" (ASI_AFAR));
1271 logp->afsr = afsr;
1272 logp->afar = afar;
1273 }
1274 ret = 1;
1275 }
1276 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1277 "membar #Sync\n\t"
1278 : : "r" (afsr), "i" (ASI_AFSR));
1279
1280 return ret;
1281 }
1282
1283 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1284 {
1285 struct cheetah_err_info local_snapshot, *p;
1286 int recoverable;
1287
1288 /* Flush E-cache */
1289 cheetah_flush_ecache();
1290
1291 p = cheetah_get_error_log(afsr);
1292 if (!p) {
1293 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1294 afsr, afar);
1295 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1296 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1297 prom_halt();
1298 }
1299
1300 /* Grab snapshot of logged error. */
1301 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1302
1303 /* If the current trap snapshot does not match what the
1304 * trap handler passed along into our args, big trouble.
1305 * In such a case, mark the local copy as invalid.
1306 *
1307 * Else, it matches and we mark the afsr in the non-local
1308 * copy as invalid so we may log new error traps there.
1309 */
1310 if (p->afsr != afsr || p->afar != afar)
1311 local_snapshot.afsr = CHAFSR_INVALID;
1312 else
1313 p->afsr = CHAFSR_INVALID;
1314
1315 cheetah_flush_icache();
1316 cheetah_flush_dcache();
1317
1318 /* Re-enable I-cache/D-cache */
1319 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1320 "or %%g1, %1, %%g1\n\t"
1321 "stxa %%g1, [%%g0] %0\n\t"
1322 "membar #Sync"
1323 : /* no outputs */
1324 : "i" (ASI_DCU_CONTROL_REG),
1325 "i" (DCU_DC | DCU_IC)
1326 : "g1");
1327
1328 /* Re-enable error reporting */
1329 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1330 "or %%g1, %1, %%g1\n\t"
1331 "stxa %%g1, [%%g0] %0\n\t"
1332 "membar #Sync"
1333 : /* no outputs */
1334 : "i" (ASI_ESTATE_ERROR_EN),
1335 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1336 : "g1");
1337
1338 /* Decide if we can continue after handling this trap and
1339 * logging the error.
1340 */
1341 recoverable = 1;
1342 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1343 recoverable = 0;
1344
1345 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1346 * error was logged while we had error reporting traps disabled.
1347 */
1348 if (cheetah_recheck_errors(&local_snapshot)) {
1349 unsigned long new_afsr = local_snapshot.afsr;
1350
1351 /* If we got a new asynchronous error, die... */
1352 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1353 CHAFSR_WDU | CHAFSR_CPU |
1354 CHAFSR_IVU | CHAFSR_UE |
1355 CHAFSR_BERR | CHAFSR_TO))
1356 recoverable = 0;
1357 }
1358
1359 /* Log errors. */
1360 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1361
1362 if (!recoverable)
1363 panic("Irrecoverable Fast-ECC error trap.\n");
1364
1365 /* Flush E-cache to kick the error trap handlers out. */
1366 cheetah_flush_ecache();
1367 }
1368
1369 /* Try to fix a correctable error by pushing the line out from
1370 * the E-cache. Recheck error reporting registers to see if the
1371 * problem is intermittent.
1372 */
1373 static int cheetah_fix_ce(unsigned long physaddr)
1374 {
1375 unsigned long orig_estate;
1376 unsigned long alias1, alias2;
1377 int ret;
1378
1379 /* Make sure correctable error traps are disabled. */
1380 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t"
1381 "andn %0, %1, %%g1\n\t"
1382 "stxa %%g1, [%%g0] %2\n\t"
1383 "membar #Sync"
1384 : "=&r" (orig_estate)
1385 : "i" (ESTATE_ERROR_CEEN),
1386 "i" (ASI_ESTATE_ERROR_EN)
1387 : "g1");
1388
1389 /* We calculate alias addresses that will force the
1390 * cache line in question out of the E-cache. Then
1391 * we bring it back in with an atomic instruction so
1392 * that we get it in some modified/exclusive state,
1393 * then we displace it again to try and get proper ECC
1394 * pushed back into the system.
1395 */
1396 physaddr &= ~(8UL - 1UL);
1397 alias1 = (ecache_flush_physbase +
1398 (physaddr & ((ecache_flush_size >> 1) - 1)));
1399 alias2 = alias1 + (ecache_flush_size >> 1);
1400 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t"
1401 "ldxa [%1] %3, %%g0\n\t"
1402 "casxa [%2] %3, %%g0, %%g0\n\t"
1403 "ldxa [%0] %3, %%g0\n\t"
1404 "ldxa [%1] %3, %%g0\n\t"
1405 "membar #Sync"
1406 : /* no outputs */
1407 : "r" (alias1), "r" (alias2),
1408 "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1409
1410 /* Did that trigger another error? */
1411 if (cheetah_recheck_errors(NULL)) {
1412 /* Try one more time. */
1413 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1414 "membar #Sync"
1415 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1416 if (cheetah_recheck_errors(NULL))
1417 ret = 2;
1418 else
1419 ret = 1;
1420 } else {
1421 /* No new error, intermittent problem. */
1422 ret = 0;
1423 }
1424
1425 /* Restore error enables. */
1426 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1427 "membar #Sync"
1428 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1429
1430 return ret;
1431 }
1432
1433 /* Return non-zero if PADDR is a valid physical memory address. */
1434 static int cheetah_check_main_memory(unsigned long paddr)
1435 {
1436 unsigned long vaddr = PAGE_OFFSET + paddr;
1437
1438 if (vaddr > (unsigned long) high_memory)
1439 return 0;
1440
1441 return kern_addr_valid(vaddr);
1442 }
1443
1444 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1445 {
1446 struct cheetah_err_info local_snapshot, *p;
1447 int recoverable, is_memory;
1448
1449 p = cheetah_get_error_log(afsr);
1450 if (!p) {
1451 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1452 afsr, afar);
1453 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1454 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1455 prom_halt();
1456 }
1457
1458 /* Grab snapshot of logged error. */
1459 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1460
1461 /* If the current trap snapshot does not match what the
1462 * trap handler passed along into our args, big trouble.
1463 * In such a case, mark the local copy as invalid.
1464 *
1465 * Else, it matches and we mark the afsr in the non-local
1466 * copy as invalid so we may log new error traps there.
1467 */
1468 if (p->afsr != afsr || p->afar != afar)
1469 local_snapshot.afsr = CHAFSR_INVALID;
1470 else
1471 p->afsr = CHAFSR_INVALID;
1472
1473 is_memory = cheetah_check_main_memory(afar);
1474
1475 if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1476 /* XXX Might want to log the results of this operation
1477 * XXX somewhere... -DaveM
1478 */
1479 cheetah_fix_ce(afar);
1480 }
1481
1482 {
1483 int flush_all, flush_line;
1484
1485 flush_all = flush_line = 0;
1486 if ((afsr & CHAFSR_EDC) != 0UL) {
1487 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1488 flush_line = 1;
1489 else
1490 flush_all = 1;
1491 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1492 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1493 flush_line = 1;
1494 else
1495 flush_all = 1;
1496 }
1497
1498 /* Trap handler only disabled I-cache, flush it. */
1499 cheetah_flush_icache();
1500
1501 /* Re-enable I-cache */
1502 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1503 "or %%g1, %1, %%g1\n\t"
1504 "stxa %%g1, [%%g0] %0\n\t"
1505 "membar #Sync"
1506 : /* no outputs */
1507 : "i" (ASI_DCU_CONTROL_REG),
1508 "i" (DCU_IC)
1509 : "g1");
1510
1511 if (flush_all)
1512 cheetah_flush_ecache();
1513 else if (flush_line)
1514 cheetah_flush_ecache_line(afar);
1515 }
1516
1517 /* Re-enable error reporting */
1518 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1519 "or %%g1, %1, %%g1\n\t"
1520 "stxa %%g1, [%%g0] %0\n\t"
1521 "membar #Sync"
1522 : /* no outputs */
1523 : "i" (ASI_ESTATE_ERROR_EN),
1524 "i" (ESTATE_ERROR_CEEN)
1525 : "g1");
1526
1527 /* Decide if we can continue after handling this trap and
1528 * logging the error.
1529 */
1530 recoverable = 1;
1531 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1532 recoverable = 0;
1533
1534 /* Re-check AFSR/AFAR */
1535 (void) cheetah_recheck_errors(&local_snapshot);
1536
1537 /* Log errors. */
1538 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1539
1540 if (!recoverable)
1541 panic("Irrecoverable Correctable-ECC error trap.\n");
1542 }
1543
1544 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1545 {
1546 struct cheetah_err_info local_snapshot, *p;
1547 int recoverable, is_memory;
1548
1549 #ifdef CONFIG_PCI
1550 /* Check for the special PCI poke sequence. */
1551 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1552 cheetah_flush_icache();
1553 cheetah_flush_dcache();
1554
1555 /* Re-enable I-cache/D-cache */
1556 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1557 "or %%g1, %1, %%g1\n\t"
1558 "stxa %%g1, [%%g0] %0\n\t"
1559 "membar #Sync"
1560 : /* no outputs */
1561 : "i" (ASI_DCU_CONTROL_REG),
1562 "i" (DCU_DC | DCU_IC)
1563 : "g1");
1564
1565 /* Re-enable error reporting */
1566 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1567 "or %%g1, %1, %%g1\n\t"
1568 "stxa %%g1, [%%g0] %0\n\t"
1569 "membar #Sync"
1570 : /* no outputs */
1571 : "i" (ASI_ESTATE_ERROR_EN),
1572 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1573 : "g1");
1574
1575 (void) cheetah_recheck_errors(NULL);
1576
1577 pci_poke_faulted = 1;
1578 regs->tpc += 4;
1579 regs->tnpc = regs->tpc + 4;
1580 return;
1581 }
1582 #endif
1583
1584 p = cheetah_get_error_log(afsr);
1585 if (!p) {
1586 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1587 afsr, afar);
1588 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1589 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1590 prom_halt();
1591 }
1592
1593 /* Grab snapshot of logged error. */
1594 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1595
1596 /* If the current trap snapshot does not match what the
1597 * trap handler passed along into our args, big trouble.
1598 * In such a case, mark the local copy as invalid.
1599 *
1600 * Else, it matches and we mark the afsr in the non-local
1601 * copy as invalid so we may log new error traps there.
1602 */
1603 if (p->afsr != afsr || p->afar != afar)
1604 local_snapshot.afsr = CHAFSR_INVALID;
1605 else
1606 p->afsr = CHAFSR_INVALID;
1607
1608 is_memory = cheetah_check_main_memory(afar);
1609
1610 {
1611 int flush_all, flush_line;
1612
1613 flush_all = flush_line = 0;
1614 if ((afsr & CHAFSR_EDU) != 0UL) {
1615 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1616 flush_line = 1;
1617 else
1618 flush_all = 1;
1619 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1620 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1621 flush_line = 1;
1622 else
1623 flush_all = 1;
1624 }
1625
1626 cheetah_flush_icache();
1627 cheetah_flush_dcache();
1628
1629 /* Re-enable I/D caches */
1630 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1631 "or %%g1, %1, %%g1\n\t"
1632 "stxa %%g1, [%%g0] %0\n\t"
1633 "membar #Sync"
1634 : /* no outputs */
1635 : "i" (ASI_DCU_CONTROL_REG),
1636 "i" (DCU_IC | DCU_DC)
1637 : "g1");
1638
1639 if (flush_all)
1640 cheetah_flush_ecache();
1641 else if (flush_line)
1642 cheetah_flush_ecache_line(afar);
1643 }
1644
1645 /* Re-enable error reporting */
1646 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1647 "or %%g1, %1, %%g1\n\t"
1648 "stxa %%g1, [%%g0] %0\n\t"
1649 "membar #Sync"
1650 : /* no outputs */
1651 : "i" (ASI_ESTATE_ERROR_EN),
1652 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1653 : "g1");
1654
1655 /* Decide if we can continue after handling this trap and
1656 * logging the error.
1657 */
1658 recoverable = 1;
1659 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1660 recoverable = 0;
1661
1662 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1663 * error was logged while we had error reporting traps disabled.
1664 */
1665 if (cheetah_recheck_errors(&local_snapshot)) {
1666 unsigned long new_afsr = local_snapshot.afsr;
1667
1668 /* If we got a new asynchronous error, die... */
1669 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1670 CHAFSR_WDU | CHAFSR_CPU |
1671 CHAFSR_IVU | CHAFSR_UE |
1672 CHAFSR_BERR | CHAFSR_TO))
1673 recoverable = 0;
1674 }
1675
1676 /* Log errors. */
1677 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1678
1679 /* "Recoverable" here means we try to yank the page from ever
1680 * being newly used again. This depends upon a few things:
1681 * 1) Must be main memory, and AFAR must be valid.
1682 * 2) If we trapped from user, OK.
1683 * 3) Else, if we trapped from kernel we must find exception
1684 * table entry (ie. we have to have been accessing user
1685 * space).
1686 *
1687 * If AFAR is not in main memory, or we trapped from kernel
1688 * and cannot find an exception table entry, it is unacceptable
1689 * to try and continue.
1690 */
1691 if (recoverable && is_memory) {
1692 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1693 /* OK, usermode access. */
1694 recoverable = 1;
1695 } else {
1696 const struct exception_table_entry *entry;
1697
1698 entry = search_exception_tables(regs->tpc);
1699 if (entry) {
1700 /* OK, kernel access to userspace. */
1701 recoverable = 1;
1702
1703 } else {
1704 /* BAD, privileged state is corrupted. */
1705 recoverable = 0;
1706 }
1707
1708 if (recoverable) {
1709 if (pfn_valid(afar >> PAGE_SHIFT))
1710 get_page(pfn_to_page(afar >> PAGE_SHIFT));
1711 else
1712 recoverable = 0;
1713
1714 /* Only perform fixup if we still have a
1715 * recoverable condition.
1716 */
1717 if (recoverable) {
1718 regs->tpc = entry->fixup;
1719 regs->tnpc = regs->tpc + 4;
1720 }
1721 }
1722 }
1723 } else {
1724 recoverable = 0;
1725 }
1726
1727 if (!recoverable)
1728 panic("Irrecoverable deferred error trap.\n");
1729 }
1730
1731 /* Handle a D/I cache parity error trap. TYPE is encoded as:
1732 *
1733 * Bit0: 0=dcache,1=icache
1734 * Bit1: 0=recoverable,1=unrecoverable
1735 *
1736 * The hardware has disabled both the I-cache and D-cache in
1737 * the %dcr register.
1738 */
1739 void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1740 {
1741 if (type & 0x1)
1742 __cheetah_flush_icache();
1743 else
1744 cheetah_plus_zap_dcache_parity();
1745 cheetah_flush_dcache();
1746
1747 /* Re-enable I-cache/D-cache */
1748 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1749 "or %%g1, %1, %%g1\n\t"
1750 "stxa %%g1, [%%g0] %0\n\t"
1751 "membar #Sync"
1752 : /* no outputs */
1753 : "i" (ASI_DCU_CONTROL_REG),
1754 "i" (DCU_DC | DCU_IC)
1755 : "g1");
1756
1757 if (type & 0x2) {
1758 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1759 smp_processor_id(),
1760 (type & 0x1) ? 'I' : 'D',
1761 regs->tpc);
1762 printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc);
1763 panic("Irrecoverable Cheetah+ parity error.");
1764 }
1765
1766 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1767 smp_processor_id(),
1768 (type & 0x1) ? 'I' : 'D',
1769 regs->tpc);
1770 printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc);
1771 }
1772
1773 struct sun4v_error_entry {
1774 /* Unique error handle */
1775 /*0x00*/u64 err_handle;
1776
1777 /* %stick value at the time of the error */
1778 /*0x08*/u64 err_stick;
1779
1780 /*0x10*/u8 reserved_1[3];
1781
1782 /* Error type */
1783 /*0x13*/u8 err_type;
1784 #define SUN4V_ERR_TYPE_UNDEFINED 0
1785 #define SUN4V_ERR_TYPE_UNCORRECTED_RES 1
1786 #define SUN4V_ERR_TYPE_PRECISE_NONRES 2
1787 #define SUN4V_ERR_TYPE_DEFERRED_NONRES 3
1788 #define SUN4V_ERR_TYPE_SHUTDOWN_RQST 4
1789 #define SUN4V_ERR_TYPE_DUMP_CORE 5
1790 #define SUN4V_ERR_TYPE_SP_STATE_CHANGE 6
1791 #define SUN4V_ERR_TYPE_NUM 7
1792
1793 /* Error attributes */
1794 /*0x14*/u32 err_attrs;
1795 #define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001
1796 #define SUN4V_ERR_ATTRS_MEMORY 0x00000002
1797 #define SUN4V_ERR_ATTRS_PIO 0x00000004
1798 #define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008
1799 #define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010
1800 #define SUN4V_ERR_ATTRS_SHUTDOWN_RQST 0x00000020
1801 #define SUN4V_ERR_ATTRS_ASR 0x00000040
1802 #define SUN4V_ERR_ATTRS_ASI 0x00000080
1803 #define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100
1804 #define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600
1805 #define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9
1806 #define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000
1807 #define SUN4V_ERR_ATTRS_MODE_SHFT 24
1808 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000
1809
1810 #define SUN4V_ERR_SPSTATE_FAULTED 0
1811 #define SUN4V_ERR_SPSTATE_AVAILABLE 1
1812 #define SUN4V_ERR_SPSTATE_NOT_PRESENT 2
1813
1814 #define SUN4V_ERR_MODE_USER 1
1815 #define SUN4V_ERR_MODE_PRIV 2
1816
1817 /* Real address of the memory region or PIO transaction */
1818 /*0x18*/u64 err_raddr;
1819
1820 /* Size of the operation triggering the error, in bytes */
1821 /*0x20*/u32 err_size;
1822
1823 /* ID of the CPU */
1824 /*0x24*/u16 err_cpu;
1825
1826 /* Grace periof for shutdown, in seconds */
1827 /*0x26*/u16 err_secs;
1828
1829 /* Value of the %asi register */
1830 /*0x28*/u8 err_asi;
1831
1832 /*0x29*/u8 reserved_2;
1833
1834 /* Value of the ASR register number */
1835 /*0x2a*/u16 err_asr;
1836 #define SUN4V_ERR_ASR_VALID 0x8000
1837
1838 /*0x2c*/u32 reserved_3;
1839 /*0x30*/u64 reserved_4;
1840 /*0x38*/u64 reserved_5;
1841 };
1842
1843 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1844 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1845
1846 static const char *sun4v_err_type_to_str(u8 type)
1847 {
1848 static const char *types[SUN4V_ERR_TYPE_NUM] = {
1849 "undefined",
1850 "uncorrected resumable",
1851 "precise nonresumable",
1852 "deferred nonresumable",
1853 "shutdown request",
1854 "dump core",
1855 "SP state change",
1856 };
1857
1858 if (type < SUN4V_ERR_TYPE_NUM)
1859 return types[type];
1860
1861 return "unknown";
1862 }
1863
1864 static void sun4v_emit_err_attr_strings(u32 attrs)
1865 {
1866 static const char *attr_names[] = {
1867 "processor",
1868 "memory",
1869 "PIO",
1870 "int-registers",
1871 "fpu-registers",
1872 "shutdown-request",
1873 "ASR",
1874 "ASI",
1875 "priv-reg",
1876 };
1877 static const char *sp_states[] = {
1878 "sp-faulted",
1879 "sp-available",
1880 "sp-not-present",
1881 "sp-state-reserved",
1882 };
1883 static const char *modes[] = {
1884 "mode-reserved0",
1885 "user",
1886 "priv",
1887 "mode-reserved1",
1888 };
1889 u32 sp_state, mode;
1890 int i;
1891
1892 for (i = 0; i < ARRAY_SIZE(attr_names); i++) {
1893 if (attrs & (1U << i)) {
1894 const char *s = attr_names[i];
1895
1896 pr_cont("%s ", s);
1897 }
1898 }
1899
1900 sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >>
1901 SUN4V_ERR_ATTRS_SPSTATE_SHFT);
1902 pr_cont("%s ", sp_states[sp_state]);
1903
1904 mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >>
1905 SUN4V_ERR_ATTRS_MODE_SHFT);
1906 pr_cont("%s ", modes[mode]);
1907
1908 if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL)
1909 pr_cont("res-queue-full ");
1910 }
1911
1912 /* When the report contains a real-address of "-1" it means that the
1913 * hardware did not provide the address. So we compute the effective
1914 * address of the load or store instruction at regs->tpc and report
1915 * that. Usually when this happens it's a PIO and in such a case we
1916 * are using physical addresses with bypass ASIs anyways, so what we
1917 * report here is exactly what we want.
1918 */
1919 static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs)
1920 {
1921 unsigned int insn;
1922 u64 addr;
1923
1924 if (!(regs->tstate & TSTATE_PRIV))
1925 return;
1926
1927 insn = *(unsigned int *) regs->tpc;
1928
1929 addr = compute_effective_address(regs, insn, 0);
1930
1931 printk("%s: insn effective address [0x%016llx]\n",
1932 pfx, addr);
1933 }
1934
1935 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent,
1936 int cpu, const char *pfx, atomic_t *ocnt)
1937 {
1938 u64 *raw_ptr = (u64 *) ent;
1939 u32 attrs;
1940 int cnt;
1941
1942 printk("%s: Reporting on cpu %d\n", pfx, cpu);
1943 printk("%s: TPC [0x%016lx] <%pS>\n",
1944 pfx, regs->tpc, (void *) regs->tpc);
1945
1946 printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n",
1947 pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]);
1948 printk("%s: %016llx:%016llx:%016llx:%016llx]\n",
1949 pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]);
1950
1951 printk("%s: handle [0x%016llx] stick [0x%016llx]\n",
1952 pfx, ent->err_handle, ent->err_stick);
1953
1954 printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type));
1955
1956 attrs = ent->err_attrs;
1957 printk("%s: attrs [0x%08x] < ", pfx, attrs);
1958 sun4v_emit_err_attr_strings(attrs);
1959 pr_cont(">\n");
1960
1961 /* Various fields in the error report are only valid if
1962 * certain attribute bits are set.
1963 */
1964 if (attrs & (SUN4V_ERR_ATTRS_MEMORY |
1965 SUN4V_ERR_ATTRS_PIO |
1966 SUN4V_ERR_ATTRS_ASI)) {
1967 printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr);
1968
1969 if (ent->err_raddr == ~(u64)0)
1970 sun4v_report_real_raddr(pfx, regs);
1971 }
1972
1973 if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI))
1974 printk("%s: size [0x%x]\n", pfx, ent->err_size);
1975
1976 if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR |
1977 SUN4V_ERR_ATTRS_INT_REGISTERS |
1978 SUN4V_ERR_ATTRS_FPU_REGISTERS |
1979 SUN4V_ERR_ATTRS_PRIV_REG))
1980 printk("%s: cpu[%u]\n", pfx, ent->err_cpu);
1981
1982 if (attrs & SUN4V_ERR_ATTRS_ASI)
1983 printk("%s: asi [0x%02x]\n", pfx, ent->err_asi);
1984
1985 if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS |
1986 SUN4V_ERR_ATTRS_FPU_REGISTERS |
1987 SUN4V_ERR_ATTRS_PRIV_REG)) &&
1988 (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0)
1989 printk("%s: reg [0x%04x]\n",
1990 pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID);
1991
1992 show_regs(regs);
1993
1994 if ((cnt = atomic_read(ocnt)) != 0) {
1995 atomic_set(ocnt, 0);
1996 wmb();
1997 printk("%s: Queue overflowed %d times.\n",
1998 pfx, cnt);
1999 }
2000 }
2001
2002 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2003 * Log the event and clear the first word of the entry.
2004 */
2005 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
2006 {
2007 enum ctx_state prev_state = exception_enter();
2008 struct sun4v_error_entry *ent, local_copy;
2009 struct trap_per_cpu *tb;
2010 unsigned long paddr;
2011 int cpu;
2012
2013 cpu = get_cpu();
2014
2015 tb = &trap_block[cpu];
2016 paddr = tb->resum_kernel_buf_pa + offset;
2017 ent = __va(paddr);
2018
2019 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2020
2021 /* We have a local copy now, so release the entry. */
2022 ent->err_handle = 0;
2023 wmb();
2024
2025 put_cpu();
2026
2027 if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) {
2028 /* We should really take the seconds field of
2029 * the error report and use it for the shutdown
2030 * invocation, but for now do the same thing we
2031 * do for a DS shutdown request.
2032 */
2033 pr_info("Shutdown request, %u seconds...\n",
2034 local_copy.err_secs);
2035 orderly_poweroff(true);
2036 goto out;
2037 }
2038
2039 sun4v_log_error(regs, &local_copy, cpu,
2040 KERN_ERR "RESUMABLE ERROR",
2041 &sun4v_resum_oflow_cnt);
2042 out:
2043 exception_exit(prev_state);
2044 }
2045
2046 /* If we try to printk() we'll probably make matters worse, by trying
2047 * to retake locks this cpu already holds or causing more errors. So
2048 * just bump a counter, and we'll report these counter bumps above.
2049 */
2050 void sun4v_resum_overflow(struct pt_regs *regs)
2051 {
2052 atomic_inc(&sun4v_resum_oflow_cnt);
2053 }
2054
2055 /* Given a set of registers, get the virtual addressi that was being accessed
2056 * by the faulting instructions at tpc.
2057 */
2058 static unsigned long sun4v_get_vaddr(struct pt_regs *regs)
2059 {
2060 unsigned int insn;
2061
2062 if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) {
2063 return compute_effective_address(regs, insn,
2064 (insn >> 25) & 0x1f);
2065 }
2066 return 0;
2067 }
2068
2069 /* Attempt to handle non-resumable errors generated from userspace.
2070 * Returns true if the signal was handled, false otherwise.
2071 */
2072 bool sun4v_nonresum_error_user_handled(struct pt_regs *regs,
2073 struct sun4v_error_entry *ent) {
2074
2075 unsigned int attrs = ent->err_attrs;
2076
2077 if (attrs & SUN4V_ERR_ATTRS_MEMORY) {
2078 unsigned long addr = ent->err_raddr;
2079 siginfo_t info;
2080
2081 if (addr == ~(u64)0) {
2082 /* This seems highly unlikely to ever occur */
2083 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n");
2084 } else {
2085 unsigned long page_cnt = DIV_ROUND_UP(ent->err_size,
2086 PAGE_SIZE);
2087
2088 /* Break the unfortunate news. */
2089 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n",
2090 addr);
2091 pr_emerg("SUN4V NON-RECOVERABLE ERROR: Claiming %lu ages.\n",
2092 page_cnt);
2093
2094 while (page_cnt-- > 0) {
2095 if (pfn_valid(addr >> PAGE_SHIFT))
2096 get_page(pfn_to_page(addr >> PAGE_SHIFT));
2097 addr += PAGE_SIZE;
2098 }
2099 }
2100 info.si_signo = SIGKILL;
2101 info.si_errno = 0;
2102 info.si_trapno = 0;
2103 force_sig_info(info.si_signo, &info, current);
2104
2105 return true;
2106 }
2107 if (attrs & SUN4V_ERR_ATTRS_PIO) {
2108 siginfo_t info;
2109
2110 info.si_signo = SIGBUS;
2111 info.si_code = BUS_ADRERR;
2112 info.si_addr = (void __user *)sun4v_get_vaddr(regs);
2113 force_sig_info(info.si_signo, &info, current);
2114
2115 return true;
2116 }
2117
2118 /* Default to doing nothing */
2119 return false;
2120 }
2121
2122 /* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate.
2123 * Log the event, clear the first word of the entry, and die.
2124 */
2125 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
2126 {
2127 struct sun4v_error_entry *ent, local_copy;
2128 struct trap_per_cpu *tb;
2129 unsigned long paddr;
2130 int cpu;
2131
2132 cpu = get_cpu();
2133
2134 tb = &trap_block[cpu];
2135 paddr = tb->nonresum_kernel_buf_pa + offset;
2136 ent = __va(paddr);
2137
2138 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
2139
2140 /* We have a local copy now, so release the entry. */
2141 ent->err_handle = 0;
2142 wmb();
2143
2144 put_cpu();
2145
2146 if (!(regs->tstate & TSTATE_PRIV) &&
2147 sun4v_nonresum_error_user_handled(regs, &local_copy)) {
2148 /* DON'T PANIC: This userspace error was handled. */
2149 return;
2150 }
2151
2152 #ifdef CONFIG_PCI
2153 /* Check for the special PCI poke sequence. */
2154 if (pci_poke_in_progress && pci_poke_cpu == cpu) {
2155 pci_poke_faulted = 1;
2156 regs->tpc += 4;
2157 regs->tnpc = regs->tpc + 4;
2158 return;
2159 }
2160 #endif
2161
2162 sun4v_log_error(regs, &local_copy, cpu,
2163 KERN_EMERG "NON-RESUMABLE ERROR",
2164 &sun4v_nonresum_oflow_cnt);
2165
2166 panic("Non-resumable error.");
2167 }
2168
2169 /* If we try to printk() we'll probably make matters worse, by trying
2170 * to retake locks this cpu already holds or causing more errors. So
2171 * just bump a counter, and we'll report these counter bumps above.
2172 */
2173 void sun4v_nonresum_overflow(struct pt_regs *regs)
2174 {
2175 /* XXX Actually even this can make not that much sense. Perhaps
2176 * XXX we should just pull the plug and panic directly from here?
2177 */
2178 atomic_inc(&sun4v_nonresum_oflow_cnt);
2179 }
2180
2181 static void sun4v_tlb_error(struct pt_regs *regs)
2182 {
2183 die_if_kernel("TLB/TSB error", regs);
2184 }
2185
2186 unsigned long sun4v_err_itlb_vaddr;
2187 unsigned long sun4v_err_itlb_ctx;
2188 unsigned long sun4v_err_itlb_pte;
2189 unsigned long sun4v_err_itlb_error;
2190
2191 void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
2192 {
2193 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2194
2195 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
2196 regs->tpc, tl);
2197 printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc);
2198 printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2199 printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n",
2200 (void *) regs->u_regs[UREG_I7]);
2201 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
2202 "pte[%lx] error[%lx]\n",
2203 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
2204 sun4v_err_itlb_pte, sun4v_err_itlb_error);
2205
2206 sun4v_tlb_error(regs);
2207 }
2208
2209 unsigned long sun4v_err_dtlb_vaddr;
2210 unsigned long sun4v_err_dtlb_ctx;
2211 unsigned long sun4v_err_dtlb_pte;
2212 unsigned long sun4v_err_dtlb_error;
2213
2214 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
2215 {
2216 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2217
2218 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
2219 regs->tpc, tl);
2220 printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc);
2221 printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]);
2222 printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n",
2223 (void *) regs->u_regs[UREG_I7]);
2224 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
2225 "pte[%lx] error[%lx]\n",
2226 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
2227 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
2228
2229 sun4v_tlb_error(regs);
2230 }
2231
2232 void hypervisor_tlbop_error(unsigned long err, unsigned long op)
2233 {
2234 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
2235 err, op);
2236 }
2237
2238 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
2239 {
2240 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
2241 err, op);
2242 }
2243
2244 static void do_fpe_common(struct pt_regs *regs)
2245 {
2246 if (regs->tstate & TSTATE_PRIV) {
2247 regs->tpc = regs->tnpc;
2248 regs->tnpc += 4;
2249 } else {
2250 unsigned long fsr = current_thread_info()->xfsr[0];
2251 siginfo_t info;
2252
2253 if (test_thread_flag(TIF_32BIT)) {
2254 regs->tpc &= 0xffffffff;
2255 regs->tnpc &= 0xffffffff;
2256 }
2257 info.si_signo = SIGFPE;
2258 info.si_errno = 0;
2259 info.si_addr = (void __user *)regs->tpc;
2260 info.si_trapno = 0;
2261 info.si_code = __SI_FAULT;
2262 if ((fsr & 0x1c000) == (1 << 14)) {
2263 if (fsr & 0x10)
2264 info.si_code = FPE_FLTINV;
2265 else if (fsr & 0x08)
2266 info.si_code = FPE_FLTOVF;
2267 else if (fsr & 0x04)
2268 info.si_code = FPE_FLTUND;
2269 else if (fsr & 0x02)
2270 info.si_code = FPE_FLTDIV;
2271 else if (fsr & 0x01)
2272 info.si_code = FPE_FLTRES;
2273 }
2274 force_sig_info(SIGFPE, &info, current);
2275 }
2276 }
2277
2278 void do_fpieee(struct pt_regs *regs)
2279 {
2280 enum ctx_state prev_state = exception_enter();
2281
2282 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2283 0, 0x24, SIGFPE) == NOTIFY_STOP)
2284 goto out;
2285
2286 do_fpe_common(regs);
2287 out:
2288 exception_exit(prev_state);
2289 }
2290
2291 void do_fpother(struct pt_regs *regs)
2292 {
2293 enum ctx_state prev_state = exception_enter();
2294 struct fpustate *f = FPUSTATE;
2295 int ret = 0;
2296
2297 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2298 0, 0x25, SIGFPE) == NOTIFY_STOP)
2299 goto out;
2300
2301 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2302 case (2 << 14): /* unfinished_FPop */
2303 case (3 << 14): /* unimplemented_FPop */
2304 ret = do_mathemu(regs, f, false);
2305 break;
2306 }
2307 if (ret)
2308 goto out;
2309 do_fpe_common(regs);
2310 out:
2311 exception_exit(prev_state);
2312 }
2313
2314 void do_tof(struct pt_regs *regs)
2315 {
2316 enum ctx_state prev_state = exception_enter();
2317 siginfo_t info;
2318
2319 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2320 0, 0x26, SIGEMT) == NOTIFY_STOP)
2321 goto out;
2322
2323 if (regs->tstate & TSTATE_PRIV)
2324 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2325 if (test_thread_flag(TIF_32BIT)) {
2326 regs->tpc &= 0xffffffff;
2327 regs->tnpc &= 0xffffffff;
2328 }
2329 info.si_signo = SIGEMT;
2330 info.si_errno = 0;
2331 info.si_code = EMT_TAGOVF;
2332 info.si_addr = (void __user *)regs->tpc;
2333 info.si_trapno = 0;
2334 force_sig_info(SIGEMT, &info, current);
2335 out:
2336 exception_exit(prev_state);
2337 }
2338
2339 void do_div0(struct pt_regs *regs)
2340 {
2341 enum ctx_state prev_state = exception_enter();
2342 siginfo_t info;
2343
2344 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2345 0, 0x28, SIGFPE) == NOTIFY_STOP)
2346 goto out;
2347
2348 if (regs->tstate & TSTATE_PRIV)
2349 die_if_kernel("TL0: Kernel divide by zero.", regs);
2350 if (test_thread_flag(TIF_32BIT)) {
2351 regs->tpc &= 0xffffffff;
2352 regs->tnpc &= 0xffffffff;
2353 }
2354 info.si_signo = SIGFPE;
2355 info.si_errno = 0;
2356 info.si_code = FPE_INTDIV;
2357 info.si_addr = (void __user *)regs->tpc;
2358 info.si_trapno = 0;
2359 force_sig_info(SIGFPE, &info, current);
2360 out:
2361 exception_exit(prev_state);
2362 }
2363
2364 static void instruction_dump(unsigned int *pc)
2365 {
2366 int i;
2367
2368 if ((((unsigned long) pc) & 3))
2369 return;
2370
2371 printk("Instruction DUMP:");
2372 for (i = -3; i < 6; i++)
2373 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2374 printk("\n");
2375 }
2376
2377 static void user_instruction_dump(unsigned int __user *pc)
2378 {
2379 int i;
2380 unsigned int buf[9];
2381
2382 if ((((unsigned long) pc) & 3))
2383 return;
2384
2385 if (copy_from_user(buf, pc - 3, sizeof(buf)))
2386 return;
2387
2388 printk("Instruction DUMP:");
2389 for (i = 0; i < 9; i++)
2390 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2391 printk("\n");
2392 }
2393
2394 void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2395 {
2396 unsigned long fp, ksp;
2397 struct thread_info *tp;
2398 int count = 0;
2399 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2400 int graph = 0;
2401 #endif
2402
2403 ksp = (unsigned long) _ksp;
2404 if (!tsk)
2405 tsk = current;
2406 tp = task_thread_info(tsk);
2407 if (ksp == 0UL) {
2408 if (tsk == current)
2409 asm("mov %%fp, %0" : "=r" (ksp));
2410 else
2411 ksp = tp->ksp;
2412 }
2413 if (tp == current_thread_info())
2414 flushw_all();
2415
2416 fp = ksp + STACK_BIAS;
2417
2418 printk("Call Trace:\n");
2419 do {
2420 struct sparc_stackf *sf;
2421 struct pt_regs *regs;
2422 unsigned long pc;
2423
2424 if (!kstack_valid(tp, fp))
2425 break;
2426 sf = (struct sparc_stackf *) fp;
2427 regs = (struct pt_regs *) (sf + 1);
2428
2429 if (kstack_is_trap_frame(tp, regs)) {
2430 if (!(regs->tstate & TSTATE_PRIV))
2431 break;
2432 pc = regs->tpc;
2433 fp = regs->u_regs[UREG_I6] + STACK_BIAS;
2434 } else {
2435 pc = sf->callers_pc;
2436 fp = (unsigned long)sf->fp + STACK_BIAS;
2437 }
2438
2439 printk(" [%016lx] %pS\n", pc, (void *) pc);
2440 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2441 if ((pc + 8UL) == (unsigned long) &return_to_handler) {
2442 int index = tsk->curr_ret_stack;
2443 if (tsk->ret_stack && index >= graph) {
2444 pc = tsk->ret_stack[index - graph].ret;
2445 printk(" [%016lx] %pS\n", pc, (void *) pc);
2446 graph++;
2447 }
2448 }
2449 #endif
2450 } while (++count < 16);
2451 }
2452
2453 static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2454 {
2455 unsigned long fp = rw->ins[6];
2456
2457 if (!fp)
2458 return NULL;
2459
2460 return (struct reg_window *) (fp + STACK_BIAS);
2461 }
2462
2463 void __noreturn die_if_kernel(char *str, struct pt_regs *regs)
2464 {
2465 static int die_counter;
2466 int count = 0;
2467
2468 /* Amuse the user. */
2469 printk(
2470 " \\|/ ____ \\|/\n"
2471 " \"@'/ .. \\`@\"\n"
2472 " /_| \\__/ |_\\\n"
2473 " \\__U_/\n");
2474
2475 printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter);
2476 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2477 __asm__ __volatile__("flushw");
2478 show_regs(regs);
2479 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
2480 if (regs->tstate & TSTATE_PRIV) {
2481 struct thread_info *tp = current_thread_info();
2482 struct reg_window *rw = (struct reg_window *)
2483 (regs->u_regs[UREG_FP] + STACK_BIAS);
2484
2485 /* Stop the back trace when we hit userland or we
2486 * find some badly aligned kernel stack.
2487 */
2488 while (rw &&
2489 count++ < 30 &&
2490 kstack_valid(tp, (unsigned long) rw)) {
2491 printk("Caller[%016lx]: %pS\n", rw->ins[7],
2492 (void *) rw->ins[7]);
2493
2494 rw = kernel_stack_up(rw);
2495 }
2496 instruction_dump ((unsigned int *) regs->tpc);
2497 } else {
2498 if (test_thread_flag(TIF_32BIT)) {
2499 regs->tpc &= 0xffffffff;
2500 regs->tnpc &= 0xffffffff;
2501 }
2502 user_instruction_dump ((unsigned int __user *) regs->tpc);
2503 }
2504 if (panic_on_oops)
2505 panic("Fatal exception");
2506 if (regs->tstate & TSTATE_PRIV)
2507 do_exit(SIGKILL);
2508 do_exit(SIGSEGV);
2509 }
2510 EXPORT_SYMBOL(die_if_kernel);
2511
2512 #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19))
2513 #define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19))
2514
2515 void do_illegal_instruction(struct pt_regs *regs)
2516 {
2517 enum ctx_state prev_state = exception_enter();
2518 unsigned long pc = regs->tpc;
2519 unsigned long tstate = regs->tstate;
2520 u32 insn;
2521 siginfo_t info;
2522
2523 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2524 0, 0x10, SIGILL) == NOTIFY_STOP)
2525 goto out;
2526
2527 if (tstate & TSTATE_PRIV)
2528 die_if_kernel("Kernel illegal instruction", regs);
2529 if (test_thread_flag(TIF_32BIT))
2530 pc = (u32)pc;
2531 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2532 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2533 if (handle_popc(insn, regs))
2534 goto out;
2535 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2536 if (handle_ldf_stq(insn, regs))
2537 goto out;
2538 } else if (tlb_type == hypervisor) {
2539 if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) {
2540 if (!vis_emul(regs, insn))
2541 goto out;
2542 } else {
2543 struct fpustate *f = FPUSTATE;
2544
2545 /* On UltraSPARC T2 and later, FPU insns which
2546 * are not implemented in HW signal an illegal
2547 * instruction trap and do not set the FP Trap
2548 * Trap in the %fsr to unimplemented_FPop.
2549 */
2550 if (do_mathemu(regs, f, true))
2551 goto out;
2552 }
2553 }
2554 }
2555 info.si_signo = SIGILL;
2556 info.si_errno = 0;
2557 info.si_code = ILL_ILLOPC;
2558 info.si_addr = (void __user *)pc;
2559 info.si_trapno = 0;
2560 force_sig_info(SIGILL, &info, current);
2561 out:
2562 exception_exit(prev_state);
2563 }
2564
2565 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2566 {
2567 enum ctx_state prev_state = exception_enter();
2568 siginfo_t info;
2569
2570 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2571 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2572 goto out;
2573
2574 if (regs->tstate & TSTATE_PRIV) {
2575 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2576 goto out;
2577 }
2578 info.si_signo = SIGBUS;
2579 info.si_errno = 0;
2580 info.si_code = BUS_ADRALN;
2581 info.si_addr = (void __user *)sfar;
2582 info.si_trapno = 0;
2583 force_sig_info(SIGBUS, &info, current);
2584 out:
2585 exception_exit(prev_state);
2586 }
2587
2588 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2589 {
2590 siginfo_t info;
2591
2592 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2593 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2594 return;
2595
2596 if (regs->tstate & TSTATE_PRIV) {
2597 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2598 return;
2599 }
2600 info.si_signo = SIGBUS;
2601 info.si_errno = 0;
2602 info.si_code = BUS_ADRALN;
2603 info.si_addr = (void __user *) addr;
2604 info.si_trapno = 0;
2605 force_sig_info(SIGBUS, &info, current);
2606 }
2607
2608 void do_privop(struct pt_regs *regs)
2609 {
2610 enum ctx_state prev_state = exception_enter();
2611 siginfo_t info;
2612
2613 if (notify_die(DIE_TRAP, "privileged operation", regs,
2614 0, 0x11, SIGILL) == NOTIFY_STOP)
2615 goto out;
2616
2617 if (test_thread_flag(TIF_32BIT)) {
2618 regs->tpc &= 0xffffffff;
2619 regs->tnpc &= 0xffffffff;
2620 }
2621 info.si_signo = SIGILL;
2622 info.si_errno = 0;
2623 info.si_code = ILL_PRVOPC;
2624 info.si_addr = (void __user *)regs->tpc;
2625 info.si_trapno = 0;
2626 force_sig_info(SIGILL, &info, current);
2627 out:
2628 exception_exit(prev_state);
2629 }
2630
2631 void do_privact(struct pt_regs *regs)
2632 {
2633 do_privop(regs);
2634 }
2635
2636 /* Trap level 1 stuff or other traps we should never see... */
2637 void do_cee(struct pt_regs *regs)
2638 {
2639 exception_enter();
2640 die_if_kernel("TL0: Cache Error Exception", regs);
2641 }
2642
2643 void do_div0_tl1(struct pt_regs *regs)
2644 {
2645 exception_enter();
2646 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2647 die_if_kernel("TL1: DIV0 Exception", regs);
2648 }
2649
2650 void do_fpieee_tl1(struct pt_regs *regs)
2651 {
2652 exception_enter();
2653 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2654 die_if_kernel("TL1: FPU IEEE Exception", regs);
2655 }
2656
2657 void do_fpother_tl1(struct pt_regs *regs)
2658 {
2659 exception_enter();
2660 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2661 die_if_kernel("TL1: FPU Other Exception", regs);
2662 }
2663
2664 void do_ill_tl1(struct pt_regs *regs)
2665 {
2666 exception_enter();
2667 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2668 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2669 }
2670
2671 void do_irq_tl1(struct pt_regs *regs)
2672 {
2673 exception_enter();
2674 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2675 die_if_kernel("TL1: IRQ Exception", regs);
2676 }
2677
2678 void do_lddfmna_tl1(struct pt_regs *regs)
2679 {
2680 exception_enter();
2681 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2682 die_if_kernel("TL1: LDDF Exception", regs);
2683 }
2684
2685 void do_stdfmna_tl1(struct pt_regs *regs)
2686 {
2687 exception_enter();
2688 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2689 die_if_kernel("TL1: STDF Exception", regs);
2690 }
2691
2692 void do_paw(struct pt_regs *regs)
2693 {
2694 exception_enter();
2695 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2696 }
2697
2698 void do_paw_tl1(struct pt_regs *regs)
2699 {
2700 exception_enter();
2701 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2702 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2703 }
2704
2705 void do_vaw(struct pt_regs *regs)
2706 {
2707 exception_enter();
2708 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2709 }
2710
2711 void do_vaw_tl1(struct pt_regs *regs)
2712 {
2713 exception_enter();
2714 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2715 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2716 }
2717
2718 void do_tof_tl1(struct pt_regs *regs)
2719 {
2720 exception_enter();
2721 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2722 die_if_kernel("TL1: Tag Overflow Exception", regs);
2723 }
2724
2725 void do_getpsr(struct pt_regs *regs)
2726 {
2727 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2728 regs->tpc = regs->tnpc;
2729 regs->tnpc += 4;
2730 if (test_thread_flag(TIF_32BIT)) {
2731 regs->tpc &= 0xffffffff;
2732 regs->tnpc &= 0xffffffff;
2733 }
2734 }
2735
2736 u64 cpu_mondo_counter[NR_CPUS] = {0};
2737 struct trap_per_cpu trap_block[NR_CPUS];
2738 EXPORT_SYMBOL(trap_block);
2739
2740 /* This can get invoked before sched_init() so play it super safe
2741 * and use hard_smp_processor_id().
2742 */
2743 void notrace init_cur_cpu_trap(struct thread_info *t)
2744 {
2745 int cpu = hard_smp_processor_id();
2746 struct trap_per_cpu *p = &trap_block[cpu];
2747
2748 p->thread = t;
2749 p->pgd_paddr = 0;
2750 }
2751
2752 extern void thread_info_offsets_are_bolixed_dave(void);
2753 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2754 extern void tsb_config_offsets_are_bolixed_dave(void);
2755
2756 /* Only invoked on boot processor. */
2757 void __init trap_init(void)
2758 {
2759 /* Compile time sanity check. */
2760 BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) ||
2761 TI_FLAGS != offsetof(struct thread_info, flags) ||
2762 TI_CPU != offsetof(struct thread_info, cpu) ||
2763 TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2764 TI_KSP != offsetof(struct thread_info, ksp) ||
2765 TI_FAULT_ADDR != offsetof(struct thread_info,
2766 fault_address) ||
2767 TI_KREGS != offsetof(struct thread_info, kregs) ||
2768 TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2769 TI_REG_WINDOW != offsetof(struct thread_info,
2770 reg_window) ||
2771 TI_RWIN_SPTRS != offsetof(struct thread_info,
2772 rwbuf_stkptrs) ||
2773 TI_GSR != offsetof(struct thread_info, gsr) ||
2774 TI_XFSR != offsetof(struct thread_info, xfsr) ||
2775 TI_PRE_COUNT != offsetof(struct thread_info,
2776 preempt_count) ||
2777 TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2778 TI_CURRENT_DS != offsetof(struct thread_info,
2779 current_ds) ||
2780 TI_KUNA_REGS != offsetof(struct thread_info,
2781 kern_una_regs) ||
2782 TI_KUNA_INSN != offsetof(struct thread_info,
2783 kern_una_insn) ||
2784 TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2785 (TI_FPREGS & (64 - 1)));
2786
2787 BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu,
2788 thread) ||
2789 (TRAP_PER_CPU_PGD_PADDR !=
2790 offsetof(struct trap_per_cpu, pgd_paddr)) ||
2791 (TRAP_PER_CPU_CPU_MONDO_PA !=
2792 offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2793 (TRAP_PER_CPU_DEV_MONDO_PA !=
2794 offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2795 (TRAP_PER_CPU_RESUM_MONDO_PA !=
2796 offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2797 (TRAP_PER_CPU_RESUM_KBUF_PA !=
2798 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2799 (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2800 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2801 (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2802 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2803 (TRAP_PER_CPU_FAULT_INFO !=
2804 offsetof(struct trap_per_cpu, fault_info)) ||
2805 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2806 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2807 (TRAP_PER_CPU_CPU_LIST_PA !=
2808 offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2809 (TRAP_PER_CPU_TSB_HUGE !=
2810 offsetof(struct trap_per_cpu, tsb_huge)) ||
2811 (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2812 offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2813 (TRAP_PER_CPU_IRQ_WORKLIST_PA !=
2814 offsetof(struct trap_per_cpu, irq_worklist_pa)) ||
2815 (TRAP_PER_CPU_CPU_MONDO_QMASK !=
2816 offsetof(struct trap_per_cpu, cpu_mondo_qmask)) ||
2817 (TRAP_PER_CPU_DEV_MONDO_QMASK !=
2818 offsetof(struct trap_per_cpu, dev_mondo_qmask)) ||
2819 (TRAP_PER_CPU_RESUM_QMASK !=
2820 offsetof(struct trap_per_cpu, resum_qmask)) ||
2821 (TRAP_PER_CPU_NONRESUM_QMASK !=
2822 offsetof(struct trap_per_cpu, nonresum_qmask)) ||
2823 (TRAP_PER_CPU_PER_CPU_BASE !=
2824 offsetof(struct trap_per_cpu, __per_cpu_base)));
2825
2826 BUILD_BUG_ON((TSB_CONFIG_TSB !=
2827 offsetof(struct tsb_config, tsb)) ||
2828 (TSB_CONFIG_RSS_LIMIT !=
2829 offsetof(struct tsb_config, tsb_rss_limit)) ||
2830 (TSB_CONFIG_NENTRIES !=
2831 offsetof(struct tsb_config, tsb_nentries)) ||
2832 (TSB_CONFIG_REG_VAL !=
2833 offsetof(struct tsb_config, tsb_reg_val)) ||
2834 (TSB_CONFIG_MAP_VADDR !=
2835 offsetof(struct tsb_config, tsb_map_vaddr)) ||
2836 (TSB_CONFIG_MAP_PTE !=
2837 offsetof(struct tsb_config, tsb_map_pte)));
2838
2839 /* Attach to the address space of init_task. On SMP we
2840 * do this in smp.c:smp_callin for other cpus.
2841 */
2842 mmgrab(&init_mm);
2843 current->active_mm = &init_mm;
2844 }
Linux with added FPC-III support