1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* These get patched into the trap table at boot time
3 * once we know we have a cheetah processor.
5 .globl cheetah_fecc_trap_vector
6 .type cheetah_fecc_trap_vector,#function
7 cheetah_fecc_trap_vector:
9 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
10 andn %g1, DCU_DC | DCU_IC, %g1
11 stxa %g1, [%g0] ASI_DCU_CONTROL_REG
13 sethi %hi(cheetah_fast_ecc), %g2
14 jmpl %g2 + %lo(cheetah_fast_ecc), %g0
16 .size cheetah_fecc_trap_vector,.-cheetah_fecc_trap_vector
18 .globl cheetah_fecc_trap_vector_tl1
19 .type cheetah_fecc_trap_vector_tl1,#function
20 cheetah_fecc_trap_vector_tl1:
22 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
23 andn %g1, DCU_DC | DCU_IC, %g1
24 stxa %g1, [%g0] ASI_DCU_CONTROL_REG
26 sethi %hi(cheetah_fast_ecc), %g2
27 jmpl %g2 + %lo(cheetah_fast_ecc), %g0
29 .size cheetah_fecc_trap_vector_tl1,.-cheetah_fecc_trap_vector_tl1
31 .globl cheetah_cee_trap_vector
32 .type cheetah_cee_trap_vector,#function
33 cheetah_cee_trap_vector:
35 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
37 stxa %g1, [%g0] ASI_DCU_CONTROL_REG
39 sethi %hi(cheetah_cee), %g2
40 jmpl %g2 + %lo(cheetah_cee), %g0
42 .size cheetah_cee_trap_vector,.-cheetah_cee_trap_vector
44 .globl cheetah_cee_trap_vector_tl1
45 .type cheetah_cee_trap_vector_tl1,#function
46 cheetah_cee_trap_vector_tl1:
48 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
50 stxa %g1, [%g0] ASI_DCU_CONTROL_REG
52 sethi %hi(cheetah_cee), %g2
53 jmpl %g2 + %lo(cheetah_cee), %g0
55 .size cheetah_cee_trap_vector_tl1,.-cheetah_cee_trap_vector_tl1
57 .globl cheetah_deferred_trap_vector
58 .type cheetah_deferred_trap_vector,#function
59 cheetah_deferred_trap_vector:
61 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1;
62 andn %g1, DCU_DC | DCU_IC, %g1;
63 stxa %g1, [%g0] ASI_DCU_CONTROL_REG;
65 sethi %hi(cheetah_deferred_trap), %g2
66 jmpl %g2 + %lo(cheetah_deferred_trap), %g0
68 .size cheetah_deferred_trap_vector,.-cheetah_deferred_trap_vector
70 .globl cheetah_deferred_trap_vector_tl1
71 .type cheetah_deferred_trap_vector_tl1,#function
72 cheetah_deferred_trap_vector_tl1:
74 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1;
75 andn %g1, DCU_DC | DCU_IC, %g1;
76 stxa %g1, [%g0] ASI_DCU_CONTROL_REG;
78 sethi %hi(cheetah_deferred_trap), %g2
79 jmpl %g2 + %lo(cheetah_deferred_trap), %g0
81 .size cheetah_deferred_trap_vector_tl1,.-cheetah_deferred_trap_vector_tl1
83 /* Cheetah+ specific traps. These are for the new I/D cache parity
84 * error traps. The first argument to cheetah_plus_parity_handler
85 * is encoded as follows:
87 * Bit0: 0=dcache,1=icache
88 * Bit1: 0=recoverable,1=unrecoverable
90 .globl cheetah_plus_dcpe_trap_vector
91 .type cheetah_plus_dcpe_trap_vector,#function
92 cheetah_plus_dcpe_trap_vector:
94 sethi %hi(do_cheetah_plus_data_parity), %g7
95 jmpl %g7 + %lo(do_cheetah_plus_data_parity), %g0
101 .size cheetah_plus_dcpe_trap_vector,.-cheetah_plus_dcpe_trap_vector
103 .type do_cheetah_plus_data_parity,#function
104 do_cheetah_plus_data_parity:
106 wrpr %g0, PIL_NORMAL_MAX, %pil
107 ba,pt %xcc, etrap_irq
109 #ifdef CONFIG_TRACE_IRQFLAGS
110 call trace_hardirqs_off
114 call cheetah_plus_parity_error
115 add %sp, PTREGS_OFF, %o1
116 ba,a,pt %xcc, rtrap_irq
117 .size do_cheetah_plus_data_parity,.-do_cheetah_plus_data_parity
119 .globl cheetah_plus_dcpe_trap_vector_tl1
120 .type cheetah_plus_dcpe_trap_vector_tl1,#function
121 cheetah_plus_dcpe_trap_vector_tl1:
123 wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
124 sethi %hi(do_dcpe_tl1), %g3
125 jmpl %g3 + %lo(do_dcpe_tl1), %g0
130 .size cheetah_plus_dcpe_trap_vector_tl1,.-cheetah_plus_dcpe_trap_vector_tl1
132 .globl cheetah_plus_icpe_trap_vector
133 .type cheetah_plus_icpe_trap_vector,#function
134 cheetah_plus_icpe_trap_vector:
136 sethi %hi(do_cheetah_plus_insn_parity), %g7
137 jmpl %g7 + %lo(do_cheetah_plus_insn_parity), %g0
143 .size cheetah_plus_icpe_trap_vector,.-cheetah_plus_icpe_trap_vector
145 .type do_cheetah_plus_insn_parity,#function
146 do_cheetah_plus_insn_parity:
148 wrpr %g0, PIL_NORMAL_MAX, %pil
149 ba,pt %xcc, etrap_irq
151 #ifdef CONFIG_TRACE_IRQFLAGS
152 call trace_hardirqs_off
156 call cheetah_plus_parity_error
157 add %sp, PTREGS_OFF, %o1
158 ba,a,pt %xcc, rtrap_irq
159 .size do_cheetah_plus_insn_parity,.-do_cheetah_plus_insn_parity
161 .globl cheetah_plus_icpe_trap_vector_tl1
162 .type cheetah_plus_icpe_trap_vector_tl1,#function
163 cheetah_plus_icpe_trap_vector_tl1:
165 wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate
166 sethi %hi(do_icpe_tl1), %g3
167 jmpl %g3 + %lo(do_icpe_tl1), %g0
172 .size cheetah_plus_icpe_trap_vector_tl1,.-cheetah_plus_icpe_trap_vector_tl1
174 /* If we take one of these traps when tl >= 1, then we
175 * jump to interrupt globals. If some trap level above us
176 * was also using interrupt globals, we cannot recover.
177 * We may use all interrupt global registers except %g6.
180 .type do_dcpe_tl1,#function
182 rdpr %tl, %g1 ! Save original trap level
183 mov 1, %g2 ! Setup TSTATE checking loop
184 sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit
185 1: wrpr %g2, %tl ! Set trap level to check
186 rdpr %tstate, %g4 ! Read TSTATE for this level
187 andcc %g4, %g3, %g0 ! Interrupt globals in use?
188 bne,a,pn %xcc, do_dcpe_tl1_fatal ! Yep, irrecoverable
189 wrpr %g1, %tl ! Restore original trap level
190 add %g2, 1, %g2 ! Next trap level
191 cmp %g2, %g1 ! Hit them all yet?
192 ble,pt %icc, 1b ! Not yet
194 wrpr %g1, %tl ! Restore original trap level
195 do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
196 sethi %hi(dcache_parity_tl1_occurred), %g2
197 lduw [%g2 + %lo(dcache_parity_tl1_occurred)], %g1
199 stw %g1, [%g2 + %lo(dcache_parity_tl1_occurred)]
200 /* Reset D-cache parity */
201 sethi %hi(1 << 16), %g1 ! D-cache size
202 mov (1 << 5), %g2 ! D-cache line size
203 sub %g1, %g2, %g1 ! Move down 1 cacheline
204 1: srl %g1, 14, %g3 ! Compute UTAG
206 stxa %g3, [%g1] ASI_DCACHE_UTAG
208 sub %g2, 8, %g3 ! 64-bit data word within line
210 stxa %g0, [%g1 + %g3] ASI_DCACHE_DATA
212 subcc %g3, 8, %g3 ! Next 64-bit data word
215 subcc %g1, %g2, %g1 ! Next cacheline
218 ba,a,pt %xcc, dcpe_icpe_tl1_common
223 1: or %g7, %lo(1b), %g7
225 call cheetah_plus_parity_error
226 add %sp, PTREGS_OFF, %o1
228 .size do_dcpe_tl1,.-do_dcpe_tl1
231 .type do_icpe_tl1,#function
233 rdpr %tl, %g1 ! Save original trap level
234 mov 1, %g2 ! Setup TSTATE checking loop
235 sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit
236 1: wrpr %g2, %tl ! Set trap level to check
237 rdpr %tstate, %g4 ! Read TSTATE for this level
238 andcc %g4, %g3, %g0 ! Interrupt globals in use?
239 bne,a,pn %xcc, do_icpe_tl1_fatal ! Yep, irrecoverable
240 wrpr %g1, %tl ! Restore original trap level
241 add %g2, 1, %g2 ! Next trap level
242 cmp %g2, %g1 ! Hit them all yet?
243 ble,pt %icc, 1b ! Not yet
245 wrpr %g1, %tl ! Restore original trap level
246 do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
247 sethi %hi(icache_parity_tl1_occurred), %g2
248 lduw [%g2 + %lo(icache_parity_tl1_occurred)], %g1
250 stw %g1, [%g2 + %lo(icache_parity_tl1_occurred)]
252 sethi %hi(1 << 15), %g1 ! I-cache size
253 mov (1 << 5), %g2 ! I-cache line size
255 1: or %g1, (2 << 3), %g3
256 stxa %g0, [%g3] ASI_IC_TAG
261 ba,a,pt %xcc, dcpe_icpe_tl1_common
266 1: or %g7, %lo(1b), %g7
268 call cheetah_plus_parity_error
269 add %sp, PTREGS_OFF, %o1
271 .size do_icpe_tl1,.-do_icpe_tl1
273 .type dcpe_icpe_tl1_common,#function
274 dcpe_icpe_tl1_common:
275 /* Flush D-cache, re-enable D/I caches in DCU and finally
276 * retry the trapping instruction.
278 sethi %hi(1 << 16), %g1 ! D-cache size
279 mov (1 << 5), %g2 ! D-cache line size
281 1: stxa %g0, [%g1] ASI_DCACHE_TAG
286 ldxa [%g0] ASI_DCU_CONTROL_REG, %g1
287 or %g1, (DCU_DC | DCU_IC), %g1
288 stxa %g1, [%g0] ASI_DCU_CONTROL_REG
291 .size dcpe_icpe_tl1_common,.-dcpe_icpe_tl1_common
293 /* Capture I/D/E-cache state into per-cpu error scoreboard.
295 * %g1: (TL>=0) ? 1 : 0
300 * %g6: unused, will have current thread ptr after etrap
303 .type __cheetah_log_error,#function
305 /* Put "TL1" software bit into AFSR. */
310 /* Get log entry pointer for this cpu at this trap level. */
311 BRANCH_IF_JALAPENO(g2,g3,50f)
312 ldxa [%g0] ASI_SAFARI_CONFIG, %g2
317 50: ldxa [%g0] ASI_JBUS_CONFIG, %g2
322 sethi %hi(cheetah_error_log), %g3
323 ldx [%g3 + %lo(cheetah_error_log)], %g3
331 /* %g1 holds pointer to the top of the logging scoreboard */
341 /* %g1 now points to D-cache logging area */
342 set 0x3ff8, %g2 /* DC_addr mask */
343 and %g5, %g2, %g2 /* DC_addr bits of AFAR */
345 or %g3, 1, %g3 /* PHYS tag + valid */
347 10: ldxa [%g2] ASI_DCACHE_TAG, %g7
348 cmp %g3, %g7 /* TAG match? */
352 /* Yep, what we want, capture state. */
353 stx %g2, [%g1 + 0x20]
354 stx %g7, [%g1 + 0x28]
356 /* A membar Sync is required before and after utag access. */
358 ldxa [%g2] ASI_DCACHE_UTAG, %g7
360 stx %g7, [%g1 + 0x30]
361 ldxa [%g2] ASI_DCACHE_SNOOP_TAG, %g7
362 stx %g7, [%g1 + 0x38]
365 12: ldxa [%g2 + %g3] ASI_DCACHE_DATA, %g7
367 add %g3, (1 << 5), %g3
375 13: sethi %hi(1 << 14), %g7
384 /* %g1 now points to I-cache logging area */
385 20: set 0x1fe0, %g2 /* IC_addr mask */
386 and %g5, %g2, %g2 /* IC_addr bits of AFAR */
387 sllx %g2, 1, %g2 /* IC_addr[13:6]==VA[12:5] */
388 srlx %g5, (13 - 8), %g3 /* Make PTAG */
389 andn %g3, 0xff, %g3 /* Mask off undefined bits */
391 21: ldxa [%g2] ASI_IC_TAG, %g7
397 /* Yep, what we want, capture state. */
398 stx %g2, [%g1 + 0x40]
399 stx %g7, [%g1 + 0x48]
400 add %g2, (1 << 3), %g2
401 ldxa [%g2] ASI_IC_TAG, %g7
402 add %g2, (1 << 3), %g2
403 stx %g7, [%g1 + 0x50]
404 ldxa [%g2] ASI_IC_TAG, %g7
405 add %g2, (1 << 3), %g2
406 stx %g7, [%g1 + 0x60]
407 ldxa [%g2] ASI_IC_TAG, %g7
408 stx %g7, [%g1 + 0x68]
409 sub %g2, (3 << 3), %g2
410 ldxa [%g2] ASI_IC_STAG, %g7
411 stx %g7, [%g1 + 0x58]
415 22: ldxa [%g2 + %g3] ASI_IC_INSTR, %g7
417 add %g3, (1 << 3), %g3
425 23: sethi %hi(1 << 14), %g7
434 /* %g1 now points to E-cache logging area */
435 30: andn %g5, (32 - 1), %g2
436 stx %g2, [%g1 + 0x20]
437 ldxa [%g2] ASI_EC_TAG_DATA, %g7
438 stx %g7, [%g1 + 0x28]
439 ldxa [%g2] ASI_EC_R, %g0
442 31: ldxa [%g3] ASI_EC_DATA, %g7
456 ba,a,pt %xcc, c_deferred
457 .size __cheetah_log_error,.-__cheetah_log_error
459 /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc
460 * in the trap table. That code has done a memory barrier
461 * and has disabled both the I-cache and D-cache in the DCU
462 * control register. The I-cache is disabled so that we may
463 * capture the corrupted cache line, and the D-cache is disabled
464 * because corrupt data may have been placed there and we don't
465 * want to reference it.
467 * %g1 is one if this trap occurred at %tl >= 1.
469 * Next, we turn off error reporting so that we don't recurse.
471 .globl cheetah_fast_ecc
472 .type cheetah_fast_ecc,#function
474 ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2
475 andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2
476 stxa %g2, [%g0] ASI_ESTATE_ERROR_EN
479 /* Fetch and clear AFSR/AFAR */
480 ldxa [%g0] ASI_AFSR, %g4
481 ldxa [%g0] ASI_AFAR, %g5
482 stxa %g4, [%g0] ASI_AFSR
485 ba,pt %xcc, __cheetah_log_error
487 .size cheetah_fast_ecc,.-cheetah_fast_ecc
489 .type c_fast_ecc,#function
492 wrpr %g0, PIL_NORMAL_MAX, %pil
493 ba,pt %xcc, etrap_irq
495 #ifdef CONFIG_TRACE_IRQFLAGS
496 call trace_hardirqs_off
501 call cheetah_fecc_handler
502 add %sp, PTREGS_OFF, %o0
503 ba,a,pt %xcc, rtrap_irq
504 .size c_fast_ecc,.-c_fast_ecc
506 /* Our caller has disabled I-cache and performed membar Sync. */
508 .type cheetah_cee,#function
510 ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2
511 andn %g2, ESTATE_ERROR_CEEN, %g2
512 stxa %g2, [%g0] ASI_ESTATE_ERROR_EN
515 /* Fetch and clear AFSR/AFAR */
516 ldxa [%g0] ASI_AFSR, %g4
517 ldxa [%g0] ASI_AFAR, %g5
518 stxa %g4, [%g0] ASI_AFSR
521 ba,pt %xcc, __cheetah_log_error
523 .size cheetah_cee,.-cheetah_cee
525 .type c_cee,#function
528 wrpr %g0, PIL_NORMAL_MAX, %pil
529 ba,pt %xcc, etrap_irq
531 #ifdef CONFIG_TRACE_IRQFLAGS
532 call trace_hardirqs_off
537 call cheetah_cee_handler
538 add %sp, PTREGS_OFF, %o0
539 ba,a,pt %xcc, rtrap_irq
542 /* Our caller has disabled I-cache+D-cache and performed membar Sync. */
543 .globl cheetah_deferred_trap
544 .type cheetah_deferred_trap,#function
545 cheetah_deferred_trap:
546 ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2
547 andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2
548 stxa %g2, [%g0] ASI_ESTATE_ERROR_EN
551 /* Fetch and clear AFSR/AFAR */
552 ldxa [%g0] ASI_AFSR, %g4
553 ldxa [%g0] ASI_AFAR, %g5
554 stxa %g4, [%g0] ASI_AFSR
557 ba,pt %xcc, __cheetah_log_error
559 .size cheetah_deferred_trap,.-cheetah_deferred_trap
561 .type c_deferred,#function
564 wrpr %g0, PIL_NORMAL_MAX, %pil
565 ba,pt %xcc, etrap_irq
567 #ifdef CONFIG_TRACE_IRQFLAGS
568 call trace_hardirqs_off
573 call cheetah_deferred_handler
574 add %sp, PTREGS_OFF, %o0
575 ba,a,pt %xcc, rtrap_irq
576 .size c_deferred,.-c_deferred