| blob | 4e020466a831fff835f2298f3a3b745e7a2b99fd |
1 /*
2 * Architecture-specific unaligned trap handling.
3 *
4 * Copyright (C) 1999-2002, 2004 Hewlett-Packard Co
5 * Stephane Eranian <eranian@hpl.hp.com>
6 * David Mosberger-Tang <davidm@hpl.hp.com>
7 *
8 * 2002/12/09 Fix rotating register handling (off-by-1 error, missing fr-rotation). Fix
9 * get_rse_reg() to not leak kernel bits to user-level (reading an out-of-frame
10 * stacked register returns an undefined value; it does NOT trigger a
11 * "rsvd register fault").
12 * 2001/10/11 Fix unaligned access to rotating registers in s/w pipelined loops.
13 * 2001/08/13 Correct size of extended floats (float_fsz) from 16 to 10 bytes.
14 * 2001/01/17 Add support emulation of unaligned kernel accesses.
15 */
16 #include <linux/kernel.h>
17 #include <linux/sched.h>
18 #include <linux/tty.h>
20 #include <asm/intrinsics.h>
21 #include <asm/processor.h>
22 #include <asm/rse.h>
23 #include <asm/uaccess.h>
24 #include <asm/unaligned.h>
28 #undef DEBUG_UNALIGNED_TRAP
30 #ifdef DEBUG_UNALIGNED_TRAP
33 =======
36 # define DDUMP(str,vp,len) dump(str, vp, len)
38 static void
40 {
48 }
49 #else
50 # define DPRINT(a...)
51 # define DDUMP(str,vp,len)
52 #endif
54 #define IA64_FIRST_STACKED_GR 32
55 #define IA64_FIRST_ROTATING_FR 32
56 #define SIGN_EXT9 0xffffffffffffff00ul
58 /*
59 * sysctl settable hook which tells the kernel whether to honor the
60 * IA64_THREAD_UAC_NOPRINT prctl. Because this is user settable, we want
61 * to allow the super user to enable/disable this for security reasons
62 * (i.e. don't allow attacker to fill up logs with unaligned accesses).
63 */
67 /*
68 * For M-unit:
69 *
70 * opcode | m | x6 |
71 * --------|------|---------|
72 * [40-37] | [36] | [35:30] |
73 * --------|------|---------|
74 * 4 | 1 | 6 | = 11 bits
75 * --------------------------
76 * However bits [31:30] are not directly useful to distinguish between
77 * load/store so we can use [35:32] instead, which gives the following
78 * mask ([40:32]) using 9 bits. The 'e' comes from the fact that we defer
79 * checking the m-bit until later in the load/store emulation.
80 */
81 #define IA64_OPCODE_MASK 0x1ef
82 #define IA64_OPCODE_SHIFT 32
84 /*
85 * Table C-28 Integer Load/Store
86 *
87 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
88 *
89 * ld8.fill, st8.fill MUST be aligned because the RNATs are based on
90 * the address (bits [8:3]), so we must failed.
91 */
92 #define LD_OP 0x080
93 #define LDS_OP 0x081
94 #define LDA_OP 0x082
95 #define LDSA_OP 0x083
96 #define LDBIAS_OP 0x084
97 #define LDACQ_OP 0x085
98 /* 0x086, 0x087 are not relevant */
99 #define LDCCLR_OP 0x088
100 #define LDCNC_OP 0x089
101 #define LDCCLRACQ_OP 0x08a
102 #define ST_OP 0x08c
103 #define STREL_OP 0x08d
104 /* 0x08e,0x8f are not relevant */
106 /*
107 * Table C-29 Integer Load +Reg
108 *
109 * we use the ld->m (bit [36:36]) field to determine whether or not we have
110 * a load/store of this form.
111 */
113 /*
114 * Table C-30 Integer Load/Store +Imm
115 *
116 * We ignore [35:32]= 0x6, 0x7, 0xE, 0xF
117 *
118 * ld8.fill, st8.fill must be aligned because the Nat register are based on
119 * the address, so we must fail and the program must be fixed.
120 */
121 #define LD_IMM_OP 0x0a0
122 #define LDS_IMM_OP 0x0a1
123 #define LDA_IMM_OP 0x0a2
124 #define LDSA_IMM_OP 0x0a3
125 #define LDBIAS_IMM_OP 0x0a4
126 #define LDACQ_IMM_OP 0x0a5
127 /* 0x0a6, 0xa7 are not relevant */
128 #define LDCCLR_IMM_OP 0x0a8
129 #define LDCNC_IMM_OP 0x0a9
130 #define LDCCLRACQ_IMM_OP 0x0aa
131 #define ST_IMM_OP 0x0ac
132 #define STREL_IMM_OP 0x0ad
133 /* 0x0ae,0xaf are not relevant */
135 /*
136 * Table C-32 Floating-point Load/Store
137 */
138 #define LDF_OP 0x0c0
139 #define LDFS_OP 0x0c1
140 #define LDFA_OP 0x0c2
141 #define LDFSA_OP 0x0c3
142 /* 0x0c6 is irrelevant */
143 #define LDFCCLR_OP 0x0c8
144 #define LDFCNC_OP 0x0c9
145 /* 0x0cb is irrelevant */
146 #define STF_OP 0x0cc
148 /*
149 * Table C-33 Floating-point Load +Reg
150 *
151 * we use the ld->m (bit [36:36]) field to determine whether or not we have
152 * a load/store of this form.
153 */
155 /*
156 * Table C-34 Floating-point Load/Store +Imm
157 */
158 #define LDF_IMM_OP 0x0e0
159 #define LDFS_IMM_OP 0x0e1
160 #define LDFA_IMM_OP 0x0e2
161 #define LDFSA_IMM_OP 0x0e3
162 /* 0x0e6 is irrelevant */
163 #define LDFCCLR_IMM_OP 0x0e8
164 #define LDFCNC_IMM_OP 0x0e9
165 #define STF_IMM_OP 0x0ec
184 UPD_REG /* ldXZ r1=[r3],r2 */
187 /*
188 * We use tables to keep track of the offsets of registers in the saved state.
189 * This way we save having big switch/case statements.
190 *
191 * We use bit 0 to indicate switch_stack or pt_regs.
192 * The offset is simply shifted by 1 bit.
193 * A 2-byte value should be enough to hold any kind of offset
194 *
195 * In case the calling convention changes (and thus pt_regs/switch_stack)
196 * simply use RSW instead of RPT or vice-versa.
197 */
199 #define RPO(x) ((size_t) &((struct pt_regs *)0)->x)
200 #define RSO(x) ((size_t) &((struct switch_stack *)0)->x)
202 #define RPT(x) (RPO(x) << 1)
203 #define RSW(x) (1| RSO(x)<<1)
205 #define GR_OFFS(x) (gr_info[x]>>1)
206 #define GR_IN_SW(x) (gr_info[x] & 0x1)
208 #define FR_OFFS(x) (fr_info[x]>>1)
209 #define FR_IN_SW(x) (fr_info[x] & 0x1)
225 };
241 };
243 /* Invalidate ALAT entry for integer register REGNO. */
244 static void
246 {
247 # define F(reg) case reg: ia64_invala_gr(reg); break
266 }
267 # undef F
268 }
270 /* Invalidate ALAT entry for floating-point register REGNO. */
271 static void
273 {
274 # define F(reg) case reg: ia64_invala_fr(reg); break
293 }
294 # undef F
295 }
299 {
304 }
306 static void
308 {
320 /* this should never happen, as the "rsvd register fault" has higher priority */
323 }
334 /* the register is on the kernel backing store: easy... */
343 else
346 }
351 }
371 else
376 }
379 static void
381 {
393 /* read of out-of-frame register returns an undefined value; 0 in our case. */
396 }
407 /* the register is on the kernel backing store: easy... */
415 }
417 }
422 }
441 }
444 fail:
449 }
452 static void
454 {
460 /*
461 * First takes care of stacked registers
462 */
466 }
468 /*
469 * Using r0 as a target raises a General Exception fault which has higher priority
470 * than the Unaligned Reference fault.
471 */
473 /*
474 * Now look at registers in [0-31] range and init correct UNAT
475 */
482 }
485 /*
486 * add offset from base of struct
487 * and do it !
488 */
493 /*
494 * We need to clear the corresponding UNAT bit to fully emulate the load
495 * UNAT bit_pos = GR[r3]{8:3} form EAS-2.4
496 */
503 }
505 }
507 /*
508 * Return the (rotated) index for floating point register REGNUM (REGNUM must be in the
509 * range from 32-127, result is in the range from 0-95.
510 */
513 {
516 }
518 static void
520 {
524 /*
525 * From EAS-2.5: FPDisableFault has higher priority than Unaligned
526 * Fault. Thus, when we get here, we know the partition is enabled.
527 * To update f32-f127, there are three choices:
528 *
529 * (1) save f32-f127 to thread.fph and update the values there
530 * (2) use a gigantic switch statement to directly access the registers
531 * (3) generate code on the fly to update the desired register
532 *
533 * For now, we are using approach (1).
534 */
539 /*
540 * pt_regs or switch_stack ?
541 */
546 }
553 /*
554 * mark the low partition as being used now
555 *
556 * It is highly unlikely that this bit is not already set, but
557 * let's do it for safety.
558 */
560 }
561 }
563 /*
564 * Those 2 inline functions generate the spilled versions of the constant floating point
565 * registers which can be used with stfX
566 */
569 {
571 }
575 {
577 }
579 static void
581 {
585 /*
586 * From EAS-2.5: FPDisableFault has higher priority than
587 * Unaligned Fault. Thus, when we get here, we know the partition is
588 * enabled.
589 *
590 * When regnum > 31, the register is still live and we need to force a save
591 * to current->thread.fph to get access to it. See discussion in setfpreg()
592 * for reasons and other ways of doing this.
593 */
598 /*
599 * f0 = 0.0, f1= 1.0. Those registers are constant and are thus
600 * not saved, we must generate their spilled form on the fly
601 */
610 /*
611 * pt_regs or switch_stack ?
612 */
621 }
622 }
623 }
626 static void
628 {
635 }
637 /*
638 * take care of r0 (read-only always evaluate to 0)
639 */
645 }
647 /*
648 * Now look at registers in [0-31] range and init correct UNAT
649 */
656 }
664 /*
665 * do it only when requested
666 */
669 }
671 static void
673 {
674 /*
675 * IMPORTANT:
676 * Given the way we handle unaligned speculative loads, we should
677 * not get to this point in the code but we keep this sanity check,
678 * just in case.
679 */
683 =======
689 }
692 /*
693 * at this point, we know that the base register to update is valid i.e.,
694 * it's not r0
695 */
699 /*
700 * Load +Imm: ldXZ r1=[r3],imm(9)
701 *
702 *
703 * form imm9: [13:19] contain the first 7 bits
704 */
707 /*
708 * sign extend (1+8bits) if m set
709 */
712 /*
713 * ifa == r3 and we know that the NaT bit on r3 was clear so
714 * we can directly use ifa.
715 */
726 /*
727 * Load +Reg Opcode: ldXZ r1=[r3],r2
728 *
729 * Note: that we update r3 even in the case of ldfX.a
730 * (where the load does not happen)
731 *
732 * The way the load algorithm works, we know that r3 does not
733 * have its NaT bit set (would have gotten NaT consumption
734 * before getting the unaligned fault). So we can use ifa
735 * which equals r3 at this point.
736 *
737 * IMPORTANT:
738 * The above statement holds ONLY because we know that we
739 * never reach this code when trying to do a ldX.s.
740 * If we ever make it to here on an ldfX.s then
741 */
746 /*
747 * propagate Nat r2 -> r3
748 */
752 }
753 }
756 static int
758 {
762 /*
763 * r0, as target, doesn't need to be checked because Illegal Instruction
764 * faults have higher priority than unaligned faults.
765 *
766 * r0 cannot be found as the base as it would never generate an
767 * unaligned reference.
768 */
770 /*
771 * ldX.a we will emulate load and also invalidate the ALAT entry.
772 * See comment below for explanation on how we handle ldX.a
773 */
778 }
779 /* this assumes little-endian byte-order: */
784 /*
785 * check for updates on any kind of loads
786 */
790 /*
791 * handling of various loads (based on EAS2.4):
792 *
793 * ldX.acq (ordered load):
794 * - acquire semantics would have been used, so force fence instead.
795 *
796 * ldX.c.clr (check load and clear):
797 * - if we get to this handler, it's because the entry was not in the ALAT.
798 * Therefore the operation reverts to a normal load
799 *
800 * ldX.c.nc (check load no clear):
801 * - same as previous one
802 *
803 * ldX.c.clr.acq (ordered check load and clear):
804 * - same as above for c.clr part. The load needs to have acquire semantics. So
805 * we use the fence semantics which is stronger and thus ensures correctness.
806 *
807 * ldX.a (advanced load):
808 * - suppose ldX.a r1=[r3]. If we get to the unaligned trap it's because the
809 * address doesn't match requested size alignment. This means that we would
810 * possibly need more than one load to get the result.
811 *
812 * The load part can be handled just like a normal load, however the difficult
813 * part is to get the right thing into the ALAT. The critical piece of information
814 * in the base address of the load & size. To do that, a ld.a must be executed,
815 * clearly any address can be pushed into the table by using ld1.a r1=[r3]. Now
816 * if we use the same target register, we will be okay for the check.a instruction.
817 * If we look at the store, basically a stX [r3]=r1 checks the ALAT for any entry
818 * which would overlap within [r3,r3+X] (the size of the load was store in the
819 * ALAT). If such an entry is found the entry is invalidated. But this is not good
820 * enough, take the following example:
821 * r3=3
822 * ld4.a r1=[r3]
823 *
824 * Could be emulated by doing:
825 * ld1.a r1=[r3],1
826 * store to temporary;
827 * ld1.a r1=[r3],1
828 * store & shift to temporary;
829 * ld1.a r1=[r3],1
830 * store & shift to temporary;
831 * ld1.a r1=[r3]
832 * store & shift to temporary;
833 * r1=temporary
834 *
835 * So in this case, you would get the right value is r1 but the wrong info in
836 * the ALAT. Notice that you could do it in reverse to finish with address 3
837 * but you would still get the size wrong. To get the size right, one needs to
838 * execute exactly the same kind of load. You could do it from a aligned
839 * temporary location, but you would get the address wrong.
840 *
841 * So no matter what, it is not possible to emulate an advanced load
842 * correctly. But is that really critical ?
843 *
844 * We will always convert ld.a into a normal load with ALAT invalidated. This
845 * will enable compiler to do optimization where certain code path after ld.a
846 * is not required to have ld.c/chk.a, e.g., code path with no intervening stores.
847 *
848 * If there is a store after the advanced load, one must either do a ld.c.* or
849 * chk.a.* to reuse the value stored in the ALAT. Both can "fail" (meaning no
850 * entry found in ALAT), and that's perfectly ok because:
851 *
852 * - ld.c.*, if the entry is not present a normal load is executed
853 * - chk.a.*, if the entry is not present, execution jumps to recovery code
854 *
855 * In either case, the load can be potentially retried in another form.
856 *
857 * ALAT must be invalidated for the register (so that chk.a or ld.c don't pick
858 * up a stale entry later). The register base update MUST also be performed.
859 */
861 /*
862 * when the load has the .acq completer then
863 * use ordering fence.
864 */
868 /*
869 * invalidate ALAT entry in case of advanced load
870 */
875 }
877 static int
879 {
883 /*
884 * if we get to this handler, Nat bits on both r3 and r2 have already
885 * been checked. so we don't need to do it
886 *
887 * extract the value to be stored
888 */
891 /*
892 * we rely on the macros in unaligned.h for now i.e.,
893 * we let the compiler figure out how to read memory gracefully.
894 *
895 * We need this switch/case because the way the inline function
896 * works. The code is optimized by the compiler and looks like
897 * a single switch/case.
898 */
904 }
906 /* this assumes little-endian byte-order: */
910 /*
911 * stX [r3]=r2,imm(9)
912 *
913 * NOTE:
914 * ld.r3 can never be r0, because r0 would not generate an
915 * unaligned access.
916 */
920 /*
921 * form imm9: [12:6] contain first 7bits
922 */
924 /*
925 * sign extend (8bits) if m set
926 */
928 /*
929 * ifa == r3 (NaT is necessarily cleared)
930 */
936 }
937 /*
938 * we don't have alat_invalidate_multiple() so we need
939 * to do the complete flush :-<<
940 */
943 /*
944 * stX.rel: use fence instead of release
945 */
950 }
952 /*
953 * floating point operations sizes in bytes
954 */
960 };
964 {
968 }
972 {
976 }
980 {
984 }
988 {
992 }
996 {
1000 }
1004 {
1008 }
1012 {
1016 }
1020 {
1024 }
1026 static int
1028 {
1033 /*
1034 * fr0 & fr1 don't need to be checked because Illegal Instruction faults have
1035 * higher priority than unaligned faults.
1036 *
1037 * r0 cannot be found as the base as it would never generate an unaligned
1038 * reference.
1039 */
1041 /*
1042 * make sure we get clean buffers
1043 */
1047 /*
1048 * ldfpX.a: we don't try to emulate anything but we must
1049 * invalidate the ALAT entry and execute updates, if any.
1050 */
1052 /*
1053 * This assumes little-endian byte-order. Note that there is no "ldfpe"
1054 * instruction:
1055 */
1062 /*
1063 * XXX fixme
1064 * Could optimize inlines by using ldfpX & 2 spills
1065 */
1083 }
1085 /*
1086 * XXX fixme
1087 *
1088 * A possible optimization would be to drop fpr_final and directly
1089 * use the storage from the saved context i.e., the actual final
1090 * destination (pt_regs, switch_stack or thread structure).
1091 */
1094 }
1096 /*
1097 * Check for updates: only immediate updates are available for this
1098 * instruction.
1099 */
1101 /*
1102 * the immediate is implicit given the ldsz of the operation:
1103 * single: 8 (2x4) and for all others it's 16 (2x8)
1104 */
1107 /*
1108 * IMPORTANT:
1109 * the fact that we force the NaT of r3 to zero is ONLY valid
1110 * as long as we don't come here with a ldfpX.s.
1111 * For this reason we keep this sanity check
1112 */
1116 __FUNCTION__);
1117 =======
1118 __func__);
1122 }
1124 /*
1125 * Invalidate ALAT entries, if any, for both registers.
1126 */
1130 }
1132 }
1135 static int
1137 {
1142 /*
1143 * fr0 & fr1 don't need to be checked because Illegal Instruction
1144 * faults have higher priority than unaligned faults.
1145 *
1146 * r0 cannot be found as the base as it would never generate an
1147 * unaligned reference.
1148 */
1150 /*
1151 * make sure we get clean buffers
1152 */
1156 /*
1157 * ldfX.a we don't try to emulate anything but we must
1158 * invalidate the ALAT entry.
1159 * See comments in ldX for descriptions on how the various loads are handled.
1160 */
1167 /*
1168 * we only do something for x6_op={0,8,9}
1169 */
1183 }
1185 /*
1186 * XXX fixme
1187 *
1188 * A possible optimization would be to drop fpr_final and directly
1189 * use the storage from the saved context i.e., the actual final
1190 * destination (pt_regs, switch_stack or thread structure).
1191 */
1193 }
1195 /*
1196 * check for updates on any loads
1197 */
1201 /*
1202 * invalidate ALAT entry in case of advanced floating point loads
1203 */
1208 }
1211 static int
1213 {
1218 /*
1219 * make sure we get clean buffers
1220 */
1224 /*
1225 * if we get to this handler, Nat bits on both r3 and r2 have already
1226 * been checked. so we don't need to do it
1227 *
1228 * extract the value to be stored
1229 */
1231 /*
1232 * during this step, we extract the spilled registers from the saved
1233 * context i.e., we refill. Then we store (no spill) to temporary
1234 * aligned location
1235 */
1249 }
1257 /*
1258 * stfX [r3]=r2,imm(9)
1259 *
1260 * NOTE:
1261 * ld.r3 can never be r0, because r0 would not generate an
1262 * unaligned access.
1263 */
1267 /*
1268 * form imm9: [12:6] contain first 7bits
1269 */
1271 /*
1272 * sign extend (8bits) if m set
1273 */
1276 /*
1277 * ifa == r3 (NaT is necessarily cleared)
1278 */
1284 }
1285 /*
1286 * we don't have alat_invalidate_multiple() so we need
1287 * to do the complete flush :-<<
1288 */
1292 }
1294 /*
1295 * Make sure we log the unaligned access, so that user/sysadmin can notice it and
1296 * eventually fix the program. However, we don't want to do that for every access so we
1297 * pace it with jiffies. This isn't really MP-safe, but it doesn't really have to be
1298 * either...
1299 */
1300 static int
1302 {
1309 count++;
1311 }
1314 }
1316 void
1318 {
1327 load_store_t insn;
1332 /* we don't support big-endian accesses */
1336 }
1338 /*
1339 * Treat kernel accesses for which there is an exception handler entry the same as
1340 * user-level unaligned accesses. Otherwise, a clever program could trick this
1341 * handler into reading an arbitrary kernel addresses...
1342 */
1352 {
1360 /*
1361 * Don't call tty_write_message() if we're in the kernel; we might
1362 * be holding locks...
1363 */
1367 /* watch for command names containing %s */
1374 "kernel assistance, which degrades "
1376 "Unaligned exception warnings have "
1377 "been disabled by the system "
1379 "echo 0 > /proc/sys/kernel/ignore-"
1382 }
1383 }
1389 }
1397 /*
1398 * extract the instruction from the bundle given the slot number
1399 */
1404 }
1411 /*
1412 * IMPORTANT:
1413 * Notice that the switch statement DOES not cover all possible instructions
1414 * that DO generate unaligned references. This is made on purpose because for some
1415 * instructions it DOES NOT make sense to try and emulate the access. Sometimes it
1416 * is WRONG to try and emulate. Here is a list of instruction we don't emulate i.e.,
1417 * the program will get a signal and die:
1418 *
1419 * load/store:
1420 * - ldX.spill
1421 * - stX.spill
1422 * Reason: RNATs are based on addresses
1423 * - ld16
1424 * - st16
1425 * Reason: ld16 and st16 are supposed to occur in a single
1426 * memory op
1427 *
1428 * synchronization:
1429 * - cmpxchg
1430 * - fetchadd
1431 * - xchg
1432 * Reason: ATOMIC operations cannot be emulated properly using multiple
1433 * instructions.
1434 *
1435 * speculative loads:
1436 * - ldX.sZ
1437 * Reason: side effects, code must be ready to deal with failure so simpler
1438 * to let the load fail.
1439 * ---------------------------------------------------------------------------------
1440 * XXX fixme
1441 *
1442 * I would like to get rid of this switch case and do something
1443 * more elegant.
1444 */
1449 /* oops, really a semaphore op (cmpxchg, etc) */
1451 /* no break */
1457 /*
1458 * The instruction will be retried with deferred exceptions turned on, and
1459 * we should get Nat bit installed
1460 *
1461 * IMPORTANT: When PSR_ED is set, the register & immediate update forms
1462 * are actually executed even though the operation failed. So we don't
1463 * need to take care of this.
1464 */
1477 /* oops, really a semaphore op (cmpxchg, etc) */
1479 /* no break */
1493 /* oops, really a semaphore op (cmpxchg, etc) */
1495 /* no break */
1507 else
1525 }
1531 /*
1532 * given today's architecture this case is not likely to happen because a
1533 * memory access instruction (M) can never be in the last slot of a
1534 * bundle. But let's keep it for now.
1535 */
1540 done:
1544 failure:
1545 /* something went wrong... */
1550 }
1553 /* NOT_REACHED */
1554 }
1555 force_sigbus:
1565 }