?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
irqchip: Fix dependencies for archs w/o HAS_IOMEM
[linux/fpc-iii.git] / arch / tile / kernel / unaligned.c
blobd075f92ccee04eb5e89e7acab9b38b0558521d66
1 /*
2 * Copyright 2013 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 *
14 * A code-rewriter that handles unaligned exception.
15 */
16
17 #include <linux/smp.h>
18 #include <linux/ptrace.h>
19 #include <linux/slab.h>
20 #include <linux/thread_info.h>
21 #include <linux/uaccess.h>
22 #include <linux/mman.h>
23 #include <linux/types.h>
24 #include <linux/err.h>
25 #include <linux/module.h>
26 #include <linux/compat.h>
27 #include <linux/prctl.h>
28 #include <linux/context_tracking.h>
29 #include <asm/cacheflush.h>
30 #include <asm/traps.h>
31 #include <asm/uaccess.h>
32 #include <asm/unaligned.h>
33 #include <arch/abi.h>
34 #include <arch/spr_def.h>
35 #include <arch/opcode.h>
36
37
38 /*
39 * This file handles unaligned exception for tile-Gx. The tilepro's unaligned
40 * exception is supported out of single_step.c
41 */
42
43 int unaligned_printk;
44
45 static int __init setup_unaligned_printk(char *str)
46 {
47 long val;
48 if (kstrtol(str, 0, &val) != 0)
49 return 0;
50 unaligned_printk = val;
51 pr_info("Printk for each unaligned data accesses is %s\n",
52 unaligned_printk ? "enabled" : "disabled");
53 return 1;
54 }
55 __setup("unaligned_printk=", setup_unaligned_printk);
56
57 unsigned int unaligned_fixup_count;
58
59 #ifdef __tilegx__
60
61 /*
62 * Unalign data jit fixup code fragement. Reserved space is 128 bytes.
63 * The 1st 64-bit word saves fault PC address, 2nd word is the fault
64 * instruction bundle followed by 14 JIT bundles.
65 */
66
67 struct unaligned_jit_fragment {
68 unsigned long pc;
69 tilegx_bundle_bits bundle;
70 tilegx_bundle_bits insn[14];
71 };
72
73 /*
74 * Check if a nop or fnop at bundle's pipeline X0.
75 */
76
77 static bool is_bundle_x0_nop(tilegx_bundle_bits bundle)
78 {
79 return (((get_UnaryOpcodeExtension_X0(bundle) ==
80 NOP_UNARY_OPCODE_X0) &&
81 (get_RRROpcodeExtension_X0(bundle) ==
82 UNARY_RRR_0_OPCODE_X0) &&
83 (get_Opcode_X0(bundle) ==
84 RRR_0_OPCODE_X0)) ||
85 ((get_UnaryOpcodeExtension_X0(bundle) ==
86 FNOP_UNARY_OPCODE_X0) &&
87 (get_RRROpcodeExtension_X0(bundle) ==
88 UNARY_RRR_0_OPCODE_X0) &&
89 (get_Opcode_X0(bundle) ==
90 RRR_0_OPCODE_X0)));
91 }
92
93 /*
94 * Check if nop or fnop at bundle's pipeline X1.
95 */
96
97 static bool is_bundle_x1_nop(tilegx_bundle_bits bundle)
98 {
99 return (((get_UnaryOpcodeExtension_X1(bundle) ==
100 NOP_UNARY_OPCODE_X1) &&
101 (get_RRROpcodeExtension_X1(bundle) ==
102 UNARY_RRR_0_OPCODE_X1) &&
103 (get_Opcode_X1(bundle) ==
104 RRR_0_OPCODE_X1)) ||
105 ((get_UnaryOpcodeExtension_X1(bundle) ==
106 FNOP_UNARY_OPCODE_X1) &&
107 (get_RRROpcodeExtension_X1(bundle) ==
108 UNARY_RRR_0_OPCODE_X1) &&
109 (get_Opcode_X1(bundle) ==
110 RRR_0_OPCODE_X1)));
111 }
112
113 /*
114 * Check if nop or fnop at bundle's Y0 pipeline.
115 */
116
117 static bool is_bundle_y0_nop(tilegx_bundle_bits bundle)
118 {
119 return (((get_UnaryOpcodeExtension_Y0(bundle) ==
120 NOP_UNARY_OPCODE_Y0) &&
121 (get_RRROpcodeExtension_Y0(bundle) ==
122 UNARY_RRR_1_OPCODE_Y0) &&
123 (get_Opcode_Y0(bundle) ==
124 RRR_1_OPCODE_Y0)) ||
125 ((get_UnaryOpcodeExtension_Y0(bundle) ==
126 FNOP_UNARY_OPCODE_Y0) &&
127 (get_RRROpcodeExtension_Y0(bundle) ==
128 UNARY_RRR_1_OPCODE_Y0) &&
129 (get_Opcode_Y0(bundle) ==
130 RRR_1_OPCODE_Y0)));
131 }
132
133 /*
134 * Check if nop or fnop at bundle's pipeline Y1.
135 */
136
137 static bool is_bundle_y1_nop(tilegx_bundle_bits bundle)
138 {
139 return (((get_UnaryOpcodeExtension_Y1(bundle) ==
140 NOP_UNARY_OPCODE_Y1) &&
141 (get_RRROpcodeExtension_Y1(bundle) ==
142 UNARY_RRR_1_OPCODE_Y1) &&
143 (get_Opcode_Y1(bundle) ==
144 RRR_1_OPCODE_Y1)) ||
145 ((get_UnaryOpcodeExtension_Y1(bundle) ==
146 FNOP_UNARY_OPCODE_Y1) &&
147 (get_RRROpcodeExtension_Y1(bundle) ==
148 UNARY_RRR_1_OPCODE_Y1) &&
149 (get_Opcode_Y1(bundle) ==
150 RRR_1_OPCODE_Y1)));
151 }
152
153 /*
154 * Test if a bundle's y0 and y1 pipelines are both nop or fnop.
155 */
156
157 static bool is_y0_y1_nop(tilegx_bundle_bits bundle)
158 {
159 return is_bundle_y0_nop(bundle) && is_bundle_y1_nop(bundle);
160 }
161
162 /*
163 * Test if a bundle's x0 and x1 pipelines are both nop or fnop.
164 */
165
166 static bool is_x0_x1_nop(tilegx_bundle_bits bundle)
167 {
168 return is_bundle_x0_nop(bundle) && is_bundle_x1_nop(bundle);
169 }
170
171 /*
172 * Find the destination, source registers of fault unalign access instruction
173 * at X1 or Y2. Also, allocate up to 3 scratch registers clob1, clob2 and
174 * clob3, which are guaranteed different from any register used in the fault
175 * bundle. r_alias is used to return if the other instructions other than the
176 * unalign load/store shares same register with ra, rb and rd.
177 */
178
179 static void find_regs(tilegx_bundle_bits bundle, uint64_t *rd, uint64_t *ra,
180 uint64_t *rb, uint64_t *clob1, uint64_t *clob2,
181 uint64_t *clob3, bool *r_alias)
182 {
183 int i;
184 uint64_t reg;
185 uint64_t reg_map = 0, alias_reg_map = 0, map;
186 bool alias = false;
187
188 /*
189 * Parse fault bundle, find potential used registers and mark
190 * corresponding bits in reg_map and alias_map. These 2 bit maps
191 * are used to find the scratch registers and determine if there
192 * is register alais.
193 */
194 if (bundle & TILEGX_BUNDLE_MODE_MASK) { /* Y Mode Bundle. */
195
196 reg = get_SrcA_Y2(bundle);
197 reg_map |= 1ULL << reg;
198 *ra = reg;
199 reg = get_SrcBDest_Y2(bundle);
200 reg_map |= 1ULL << reg;
201
202 if (rd) {
203 /* Load. */
204 *rd = reg;
205 alias_reg_map = (1ULL << *rd) | (1ULL << *ra);
206 } else {
207 /* Store. */
208 *rb = reg;
209 alias_reg_map = (1ULL << *ra) | (1ULL << *rb);
210 }
211
212 if (!is_bundle_y1_nop(bundle)) {
213 reg = get_SrcA_Y1(bundle);
214 reg_map |= (1ULL << reg);
215 map = (1ULL << reg);
216
217 reg = get_SrcB_Y1(bundle);
218 reg_map |= (1ULL << reg);
219 map |= (1ULL << reg);
220
221 reg = get_Dest_Y1(bundle);
222 reg_map |= (1ULL << reg);
223 map |= (1ULL << reg);
224
225 if (map & alias_reg_map)
226 alias = true;
227 }
228
229 if (!is_bundle_y0_nop(bundle)) {
230 reg = get_SrcA_Y0(bundle);
231 reg_map |= (1ULL << reg);
232 map = (1ULL << reg);
233
234 reg = get_SrcB_Y0(bundle);
235 reg_map |= (1ULL << reg);
236 map |= (1ULL << reg);
237
238 reg = get_Dest_Y0(bundle);
239 reg_map |= (1ULL << reg);
240 map |= (1ULL << reg);
241
242 if (map & alias_reg_map)
243 alias = true;
244 }
245 } else { /* X Mode Bundle. */
246
247 reg = get_SrcA_X1(bundle);
248 reg_map |= (1ULL << reg);
249 *ra = reg;
250 if (rd) {
251 /* Load. */
252 reg = get_Dest_X1(bundle);
253 reg_map |= (1ULL << reg);
254 *rd = reg;
255 alias_reg_map = (1ULL << *rd) | (1ULL << *ra);
256 } else {
257 /* Store. */
258 reg = get_SrcB_X1(bundle);
259 reg_map |= (1ULL << reg);
260 *rb = reg;
261 alias_reg_map = (1ULL << *ra) | (1ULL << *rb);
262 }
263
264 if (!is_bundle_x0_nop(bundle)) {
265 reg = get_SrcA_X0(bundle);
266 reg_map |= (1ULL << reg);
267 map = (1ULL << reg);
268
269 reg = get_SrcB_X0(bundle);
270 reg_map |= (1ULL << reg);
271 map |= (1ULL << reg);
272
273 reg = get_Dest_X0(bundle);
274 reg_map |= (1ULL << reg);
275 map |= (1ULL << reg);
276
277 if (map & alias_reg_map)
278 alias = true;
279 }
280 }
281
282 /*
283 * "alias" indicates if the unalign access registers have collision
284 * with others in the same bundle. We jsut simply test all register
285 * operands case (RRR), ignored the case with immidate. If a bundle
286 * has no register alias, we may do fixup in a simple or fast manner.
287 * So if an immidata field happens to hit with a register, we may end
288 * up fall back to the generic handling.
289 */
290
291 *r_alias = alias;
292
293 /* Flip bits on reg_map. */
294 reg_map ^= -1ULL;
295
296 /* Scan reg_map lower 54(TREG_SP) bits to find 3 set bits. */
297 for (i = 0; i < TREG_SP; i++) {
298 if (reg_map & (0x1ULL << i)) {
299 if (*clob1 == -1) {
300 *clob1 = i;
301 } else if (*clob2 == -1) {
302 *clob2 = i;
303 } else if (*clob3 == -1) {
304 *clob3 = i;
305 return;
306 }
307 }
308 }
309 }
310
311 /*
312 * Sanity check for register ra, rb, rd, clob1/2/3. Return true if any of them
313 * is unexpected.
314 */
315
316 static bool check_regs(uint64_t rd, uint64_t ra, uint64_t rb,
317 uint64_t clob1, uint64_t clob2, uint64_t clob3)
318 {
319 bool unexpected = false;
320 if ((ra >= 56) && (ra != TREG_ZERO))
321 unexpected = true;
322
323 if ((clob1 >= 56) || (clob2 >= 56) || (clob3 >= 56))
324 unexpected = true;
325
326 if (rd != -1) {
327 if ((rd >= 56) && (rd != TREG_ZERO))
328 unexpected = true;
329 } else {
330 if ((rb >= 56) && (rb != TREG_ZERO))
331 unexpected = true;
332 }
333 return unexpected;
334 }
335
336
337 #define GX_INSN_X0_MASK ((1ULL << 31) - 1)
338 #define GX_INSN_X1_MASK (((1ULL << 31) - 1) << 31)
339 #define GX_INSN_Y0_MASK ((0xFULL << 27) | (0xFFFFFULL))
340 #define GX_INSN_Y1_MASK (GX_INSN_Y0_MASK << 31)
341 #define GX_INSN_Y2_MASK ((0x7FULL << 51) | (0x7FULL << 20))
342
343 #ifdef __LITTLE_ENDIAN
344 #define GX_INSN_BSWAP(_bundle_) (_bundle_)
345 #else
346 #define GX_INSN_BSWAP(_bundle_) swab64(_bundle_)
347 #endif /* __LITTLE_ENDIAN */
348
349 /*
350 * __JIT_CODE(.) creates template bundles in .rodata.unalign_data section.
351 * The corresponding static function jix_x#_###(.) generates partial or
352 * whole bundle based on the template and given arguments.
353 */
354
355 #define __JIT_CODE(_X_) \
356 asm (".pushsection .rodata.unalign_data, \"a\"\n" \
357 _X_"\n" \
358 ".popsection\n")
359
360 __JIT_CODE("__unalign_jit_x1_mtspr: {mtspr 0, r0}");
361 static tilegx_bundle_bits jit_x1_mtspr(int spr, int reg)
362 {
363 extern tilegx_bundle_bits __unalign_jit_x1_mtspr;
364 return (GX_INSN_BSWAP(__unalign_jit_x1_mtspr) & GX_INSN_X1_MASK) |
365 create_MT_Imm14_X1(spr) | create_SrcA_X1(reg);
366 }
367
368 __JIT_CODE("__unalign_jit_x1_mfspr: {mfspr r0, 0}");
369 static tilegx_bundle_bits jit_x1_mfspr(int reg, int spr)
370 {
371 extern tilegx_bundle_bits __unalign_jit_x1_mfspr;
372 return (GX_INSN_BSWAP(__unalign_jit_x1_mfspr) & GX_INSN_X1_MASK) |
373 create_MF_Imm14_X1(spr) | create_Dest_X1(reg);
374 }
375
376 __JIT_CODE("__unalign_jit_x0_addi: {addi r0, r0, 0; iret}");
377 static tilegx_bundle_bits jit_x0_addi(int rd, int ra, int imm8)
378 {
379 extern tilegx_bundle_bits __unalign_jit_x0_addi;
380 return (GX_INSN_BSWAP(__unalign_jit_x0_addi) & GX_INSN_X0_MASK) |
381 create_Dest_X0(rd) | create_SrcA_X0(ra) |
382 create_Imm8_X0(imm8);
383 }
384
385 __JIT_CODE("__unalign_jit_x1_ldna: {ldna r0, r0}");
386 static tilegx_bundle_bits jit_x1_ldna(int rd, int ra)
387 {
388 extern tilegx_bundle_bits __unalign_jit_x1_ldna;
389 return (GX_INSN_BSWAP(__unalign_jit_x1_ldna) & GX_INSN_X1_MASK) |
390 create_Dest_X1(rd) | create_SrcA_X1(ra);
391 }
392
393 __JIT_CODE("__unalign_jit_x0_dblalign: {dblalign r0, r0 ,r0}");
394 static tilegx_bundle_bits jit_x0_dblalign(int rd, int ra, int rb)
395 {
396 extern tilegx_bundle_bits __unalign_jit_x0_dblalign;
397 return (GX_INSN_BSWAP(__unalign_jit_x0_dblalign) & GX_INSN_X0_MASK) |
398 create_Dest_X0(rd) | create_SrcA_X0(ra) |
399 create_SrcB_X0(rb);
400 }
401
402 __JIT_CODE("__unalign_jit_x1_iret: {iret}");
403 static tilegx_bundle_bits jit_x1_iret(void)
404 {
405 extern tilegx_bundle_bits __unalign_jit_x1_iret;
406 return GX_INSN_BSWAP(__unalign_jit_x1_iret) & GX_INSN_X1_MASK;
407 }
408
409 __JIT_CODE("__unalign_jit_x01_fnop: {fnop;fnop}");
410 static tilegx_bundle_bits jit_x0_fnop(void)
411 {
412 extern tilegx_bundle_bits __unalign_jit_x01_fnop;
413 return GX_INSN_BSWAP(__unalign_jit_x01_fnop) & GX_INSN_X0_MASK;
414 }
415
416 static tilegx_bundle_bits jit_x1_fnop(void)
417 {
418 extern tilegx_bundle_bits __unalign_jit_x01_fnop;
419 return GX_INSN_BSWAP(__unalign_jit_x01_fnop) & GX_INSN_X1_MASK;
420 }
421
422 __JIT_CODE("__unalign_jit_y2_dummy: {fnop; fnop; ld zero, sp}");
423 static tilegx_bundle_bits jit_y2_dummy(void)
424 {
425 extern tilegx_bundle_bits __unalign_jit_y2_dummy;
426 return GX_INSN_BSWAP(__unalign_jit_y2_dummy) & GX_INSN_Y2_MASK;
427 }
428
429 static tilegx_bundle_bits jit_y1_fnop(void)
430 {
431 extern tilegx_bundle_bits __unalign_jit_y2_dummy;
432 return GX_INSN_BSWAP(__unalign_jit_y2_dummy) & GX_INSN_Y1_MASK;
433 }
434
435 __JIT_CODE("__unalign_jit_x1_st1_add: {st1_add r1, r0, 0}");
436 static tilegx_bundle_bits jit_x1_st1_add(int ra, int rb, int imm8)
437 {
438 extern tilegx_bundle_bits __unalign_jit_x1_st1_add;
439 return (GX_INSN_BSWAP(__unalign_jit_x1_st1_add) &
440 (~create_SrcA_X1(-1)) &
441 GX_INSN_X1_MASK) | create_SrcA_X1(ra) |
442 create_SrcB_X1(rb) | create_Dest_Imm8_X1(imm8);
443 }
444
445 __JIT_CODE("__unalign_jit_x1_st: {crc32_8 r1, r0, r0; st r0, r0}");
446 static tilegx_bundle_bits jit_x1_st(int ra, int rb)
447 {
448 extern tilegx_bundle_bits __unalign_jit_x1_st;
449 return (GX_INSN_BSWAP(__unalign_jit_x1_st) & GX_INSN_X1_MASK) |
450 create_SrcA_X1(ra) | create_SrcB_X1(rb);
451 }
452
453 __JIT_CODE("__unalign_jit_x1_st_add: {st_add r1, r0, 0}");
454 static tilegx_bundle_bits jit_x1_st_add(int ra, int rb, int imm8)
455 {
456 extern tilegx_bundle_bits __unalign_jit_x1_st_add;
457 return (GX_INSN_BSWAP(__unalign_jit_x1_st_add) &
458 (~create_SrcA_X1(-1)) &
459 GX_INSN_X1_MASK) | create_SrcA_X1(ra) |
460 create_SrcB_X1(rb) | create_Dest_Imm8_X1(imm8);
461 }
462
463 __JIT_CODE("__unalign_jit_x1_ld: {crc32_8 r1, r0, r0; ld r0, r0}");
464 static tilegx_bundle_bits jit_x1_ld(int rd, int ra)
465 {
466 extern tilegx_bundle_bits __unalign_jit_x1_ld;
467 return (GX_INSN_BSWAP(__unalign_jit_x1_ld) & GX_INSN_X1_MASK) |
468 create_Dest_X1(rd) | create_SrcA_X1(ra);
469 }
470
471 __JIT_CODE("__unalign_jit_x1_ld_add: {ld_add r1, r0, 0}");
472 static tilegx_bundle_bits jit_x1_ld_add(int rd, int ra, int imm8)
473 {
474 extern tilegx_bundle_bits __unalign_jit_x1_ld_add;
475 return (GX_INSN_BSWAP(__unalign_jit_x1_ld_add) &
476 (~create_Dest_X1(-1)) &
477 GX_INSN_X1_MASK) | create_Dest_X1(rd) |
478 create_SrcA_X1(ra) | create_Imm8_X1(imm8);
479 }
480
481 __JIT_CODE("__unalign_jit_x0_bfexts: {bfexts r0, r0, 0, 0}");
482 static tilegx_bundle_bits jit_x0_bfexts(int rd, int ra, int bfs, int bfe)
483 {
484 extern tilegx_bundle_bits __unalign_jit_x0_bfexts;
485 return (GX_INSN_BSWAP(__unalign_jit_x0_bfexts) &
486 GX_INSN_X0_MASK) |
487 create_Dest_X0(rd) | create_SrcA_X0(ra) |
488 create_BFStart_X0(bfs) | create_BFEnd_X0(bfe);
489 }
490
491 __JIT_CODE("__unalign_jit_x0_bfextu: {bfextu r0, r0, 0, 0}");
492 static tilegx_bundle_bits jit_x0_bfextu(int rd, int ra, int bfs, int bfe)
493 {
494 extern tilegx_bundle_bits __unalign_jit_x0_bfextu;
495 return (GX_INSN_BSWAP(__unalign_jit_x0_bfextu) &
496 GX_INSN_X0_MASK) |
497 create_Dest_X0(rd) | create_SrcA_X0(ra) |
498 create_BFStart_X0(bfs) | create_BFEnd_X0(bfe);
499 }
500
501 __JIT_CODE("__unalign_jit_x1_addi: {bfextu r1, r1, 0, 0; addi r0, r0, 0}");
502 static tilegx_bundle_bits jit_x1_addi(int rd, int ra, int imm8)
503 {
504 extern tilegx_bundle_bits __unalign_jit_x1_addi;
505 return (GX_INSN_BSWAP(__unalign_jit_x1_addi) & GX_INSN_X1_MASK) |
506 create_Dest_X1(rd) | create_SrcA_X1(ra) |
507 create_Imm8_X1(imm8);
508 }
509
510 __JIT_CODE("__unalign_jit_x0_shrui: {shrui r0, r0, 0; iret}");
511 static tilegx_bundle_bits jit_x0_shrui(int rd, int ra, int imm6)
512 {
513 extern tilegx_bundle_bits __unalign_jit_x0_shrui;
514 return (GX_INSN_BSWAP(__unalign_jit_x0_shrui) &
515 GX_INSN_X0_MASK) |
516 create_Dest_X0(rd) | create_SrcA_X0(ra) |
517 create_ShAmt_X0(imm6);
518 }
519
520 __JIT_CODE("__unalign_jit_x0_rotli: {rotli r0, r0, 0; iret}");
521 static tilegx_bundle_bits jit_x0_rotli(int rd, int ra, int imm6)
522 {
523 extern tilegx_bundle_bits __unalign_jit_x0_rotli;
524 return (GX_INSN_BSWAP(__unalign_jit_x0_rotli) &
525 GX_INSN_X0_MASK) |
526 create_Dest_X0(rd) | create_SrcA_X0(ra) |
527 create_ShAmt_X0(imm6);
528 }
529
530 __JIT_CODE("__unalign_jit_x1_bnezt: {bnezt r0, __unalign_jit_x1_bnezt}");
531 static tilegx_bundle_bits jit_x1_bnezt(int ra, int broff)
532 {
533 extern tilegx_bundle_bits __unalign_jit_x1_bnezt;
534 return (GX_INSN_BSWAP(__unalign_jit_x1_bnezt) &
535 GX_INSN_X1_MASK) |
536 create_SrcA_X1(ra) | create_BrOff_X1(broff);
537 }
538
539 #undef __JIT_CODE
540
541 /*
542 * This function generates unalign fixup JIT.
543 *
544 * We first find unalign load/store instruction's destination, source
545 * registers: ra, rb and rd. and 3 scratch registers by calling
546 * find_regs(...). 3 scratch clobbers should not alias with any register
547 * used in the fault bundle. Then analyze the fault bundle to determine
548 * if it's a load or store, operand width, branch or address increment etc.
549 * At last generated JIT is copied into JIT code area in user space.
550 */
551
552 static
553 void jit_bundle_gen(struct pt_regs *regs, tilegx_bundle_bits bundle,
554 int align_ctl)
555 {
556 struct thread_info *info = current_thread_info();
557 struct unaligned_jit_fragment frag;
558 struct unaligned_jit_fragment *jit_code_area;
559 tilegx_bundle_bits bundle_2 = 0;
560 /* If bundle_2_enable = false, bundle_2 is fnop/nop operation. */
561 bool bundle_2_enable = true;
562 uint64_t ra = -1, rb = -1, rd = -1, clob1 = -1, clob2 = -1, clob3 = -1;
563 /*
564 * Indicate if the unalign access
565 * instruction's registers hit with
566 * others in the same bundle.
567 */
568 bool alias = false;
569 bool load_n_store = true;
570 bool load_store_signed = false;
571 unsigned int load_store_size = 8;
572 bool y1_br = false; /* True, for a branch in same bundle at Y1.*/
573 int y1_br_reg = 0;
574 /* True for link operation. i.e. jalr or lnk at Y1 */
575 bool y1_lr = false;
576 int y1_lr_reg = 0;
577 bool x1_add = false;/* True, for load/store ADD instruction at X1*/
578 int x1_add_imm8 = 0;
579 bool unexpected = false;
580 int n = 0, k;
581
582 jit_code_area =
583 (struct unaligned_jit_fragment *)(info->unalign_jit_base);
584
585 memset((void *)&frag, 0, sizeof(frag));
586
587 /* 0: X mode, Otherwise: Y mode. */
588 if (bundle & TILEGX_BUNDLE_MODE_MASK) {
589 unsigned int mod, opcode;
590
591 if (get_Opcode_Y1(bundle) == RRR_1_OPCODE_Y1 &&
592 get_RRROpcodeExtension_Y1(bundle) ==
593 UNARY_RRR_1_OPCODE_Y1) {
594
595 opcode = get_UnaryOpcodeExtension_Y1(bundle);
596
597 /*
598 * Test "jalr", "jalrp", "jr", "jrp" instruction at Y1
599 * pipeline.
600 */
601 switch (opcode) {
602 case JALR_UNARY_OPCODE_Y1:
603 case JALRP_UNARY_OPCODE_Y1:
604 y1_lr = true;
605 y1_lr_reg = 55; /* Link register. */
606 /* FALLTHROUGH */
607 case JR_UNARY_OPCODE_Y1:
608 case JRP_UNARY_OPCODE_Y1:
609 y1_br = true;
610 y1_br_reg = get_SrcA_Y1(bundle);
611 break;
612 case LNK_UNARY_OPCODE_Y1:
613 /* "lnk" at Y1 pipeline. */
614 y1_lr = true;
615 y1_lr_reg = get_Dest_Y1(bundle);
616 break;
617 }
618 }
619
620 opcode = get_Opcode_Y2(bundle);
621 mod = get_Mode(bundle);
622
623 /*
624 * bundle_2 is bundle after making Y2 as a dummy operation
625 * - ld zero, sp
626 */
627 bundle_2 = (bundle & (~GX_INSN_Y2_MASK)) | jit_y2_dummy();
628
629 /* Make Y1 as fnop if Y1 is a branch or lnk operation. */
630 if (y1_br || y1_lr) {
631 bundle_2 &= ~(GX_INSN_Y1_MASK);
632 bundle_2 |= jit_y1_fnop();
633 }
634
635 if (is_y0_y1_nop(bundle_2))
636 bundle_2_enable = false;
637
638 if (mod == MODE_OPCODE_YC2) {
639 /* Store. */
640 load_n_store = false;
641 load_store_size = 1 << opcode;
642 load_store_signed = false;
643 find_regs(bundle, 0, &ra, &rb, &clob1, &clob2,
644 &clob3, &alias);
645 if (load_store_size > 8)
646 unexpected = true;
647 } else {
648 /* Load. */
649 load_n_store = true;
650 if (mod == MODE_OPCODE_YB2) {
651 switch (opcode) {
652 case LD_OPCODE_Y2:
653 load_store_signed = false;
654 load_store_size = 8;
655 break;
656 case LD4S_OPCODE_Y2:
657 load_store_signed = true;
658 load_store_size = 4;
659 break;
660 case LD4U_OPCODE_Y2:
661 load_store_signed = false;
662 load_store_size = 4;
663 break;
664 default:
665 unexpected = true;
666 }
667 } else if (mod == MODE_OPCODE_YA2) {
668 if (opcode == LD2S_OPCODE_Y2) {
669 load_store_signed = true;
670 load_store_size = 2;
671 } else if (opcode == LD2U_OPCODE_Y2) {
672 load_store_signed = false;
673 load_store_size = 2;
674 } else
675 unexpected = true;
676 } else
677 unexpected = true;
678 find_regs(bundle, &rd, &ra, &rb, &clob1, &clob2,
679 &clob3, &alias);
680 }
681 } else {
682 unsigned int opcode;
683
684 /* bundle_2 is bundle after making X1 as "fnop". */
685 bundle_2 = (bundle & (~GX_INSN_X1_MASK)) | jit_x1_fnop();
686
687 if (is_x0_x1_nop(bundle_2))
688 bundle_2_enable = false;
689
690 if (get_Opcode_X1(bundle) == RRR_0_OPCODE_X1) {
691 opcode = get_UnaryOpcodeExtension_X1(bundle);
692
693 if (get_RRROpcodeExtension_X1(bundle) ==
694 UNARY_RRR_0_OPCODE_X1) {
695 load_n_store = true;
696 find_regs(bundle, &rd, &ra, &rb, &clob1,
697 &clob2, &clob3, &alias);
698
699 switch (opcode) {
700 case LD_UNARY_OPCODE_X1:
701 load_store_signed = false;
702 load_store_size = 8;
703 break;
704 case LD4S_UNARY_OPCODE_X1:
705 load_store_signed = true;
706 /* FALLTHROUGH */
707 case LD4U_UNARY_OPCODE_X1:
708 load_store_size = 4;
709 break;
710
711 case LD2S_UNARY_OPCODE_X1:
712 load_store_signed = true;
713 /* FALLTHROUGH */
714 case LD2U_UNARY_OPCODE_X1:
715 load_store_size = 2;
716 break;
717 default:
718 unexpected = true;
719 }
720 } else {
721 load_n_store = false;
722 load_store_signed = false;
723 find_regs(bundle, 0, &ra, &rb,
724 &clob1, &clob2, &clob3,
725 &alias);
726
727 opcode = get_RRROpcodeExtension_X1(bundle);
728 switch (opcode) {
729 case ST_RRR_0_OPCODE_X1:
730 load_store_size = 8;
731 break;
732 case ST4_RRR_0_OPCODE_X1:
733 load_store_size = 4;
734 break;
735 case ST2_RRR_0_OPCODE_X1:
736 load_store_size = 2;
737 break;
738 default:
739 unexpected = true;
740 }
741 }
742 } else if (get_Opcode_X1(bundle) == IMM8_OPCODE_X1) {
743 load_n_store = true;
744 opcode = get_Imm8OpcodeExtension_X1(bundle);
745 switch (opcode) {
746 case LD_ADD_IMM8_OPCODE_X1:
747 load_store_size = 8;
748 break;
749
750 case LD4S_ADD_IMM8_OPCODE_X1:
751 load_store_signed = true;
752 /* FALLTHROUGH */
753 case LD4U_ADD_IMM8_OPCODE_X1:
754 load_store_size = 4;
755 break;
756
757 case LD2S_ADD_IMM8_OPCODE_X1:
758 load_store_signed = true;
759 /* FALLTHROUGH */
760 case LD2U_ADD_IMM8_OPCODE_X1:
761 load_store_size = 2;
762 break;
763
764 case ST_ADD_IMM8_OPCODE_X1:
765 load_n_store = false;
766 load_store_size = 8;
767 break;
768 case ST4_ADD_IMM8_OPCODE_X1:
769 load_n_store = false;
770 load_store_size = 4;
771 break;
772 case ST2_ADD_IMM8_OPCODE_X1:
773 load_n_store = false;
774 load_store_size = 2;
775 break;
776 default:
777 unexpected = true;
778 }
779
780 if (!unexpected) {
781 x1_add = true;
782 if (load_n_store)
783 x1_add_imm8 = get_Imm8_X1(bundle);
784 else
785 x1_add_imm8 = get_Dest_Imm8_X1(bundle);
786 }
787
788 find_regs(bundle, load_n_store ? (&rd) : NULL,
789 &ra, &rb, &clob1, &clob2, &clob3, &alias);
790 } else
791 unexpected = true;
792 }
793
794 /*
795 * Some sanity check for register numbers extracted from fault bundle.
796 */
797 if (check_regs(rd, ra, rb, clob1, clob2, clob3) == true)
798 unexpected = true;
799
800 /* Give warning if register ra has an aligned address. */
801 if (!unexpected)
802 WARN_ON(!((load_store_size - 1) & (regs->regs[ra])));
803
804
805 /*
806 * Fault came from kernel space, here we only need take care of
807 * unaligned "get_user/put_user" macros defined in "uaccess.h".
808 * Basically, we will handle bundle like this:
809 * {ld/2u/4s rd, ra; movei rx, 0} or {st/2/4 ra, rb; movei rx, 0}
810 * (Refer to file "arch/tile/include/asm/uaccess.h" for details).
811 * For either load or store, byte-wise operation is performed by calling
812 * get_user() or put_user(). If the macro returns non-zero value,
813 * set the value to rx, otherwise set zero to rx. Finally make pc point
814 * to next bundle and return.
815 */
816
817 if (EX1_PL(regs->ex1) != USER_PL) {
818
819 unsigned long rx = 0;
820 unsigned long x = 0, ret = 0;
821
822 if (y1_br || y1_lr || x1_add ||
823 (load_store_signed !=
824 (load_n_store && load_store_size == 4))) {
825 /* No branch, link, wrong sign-ext or load/store add. */
826 unexpected = true;
827 } else if (!unexpected) {
828 if (bundle & TILEGX_BUNDLE_MODE_MASK) {
829 /*
830 * Fault bundle is Y mode.
831 * Check if the Y1 and Y0 is the form of
832 * { movei rx, 0; nop/fnop }, if yes,
833 * find the rx.
834 */
835
836 if ((get_Opcode_Y1(bundle) == ADDI_OPCODE_Y1)
837 && (get_SrcA_Y1(bundle) == TREG_ZERO) &&
838 (get_Imm8_Y1(bundle) == 0) &&
839 is_bundle_y0_nop(bundle)) {
840 rx = get_Dest_Y1(bundle);
841 } else if ((get_Opcode_Y0(bundle) ==
842 ADDI_OPCODE_Y0) &&
843 (get_SrcA_Y0(bundle) == TREG_ZERO) &&
844 (get_Imm8_Y0(bundle) == 0) &&
845 is_bundle_y1_nop(bundle)) {
846 rx = get_Dest_Y0(bundle);
847 } else {
848 unexpected = true;
849 }
850 } else {
851 /*
852 * Fault bundle is X mode.
853 * Check if the X0 is 'movei rx, 0',
854 * if yes, find the rx.
855 */
856
857 if ((get_Opcode_X0(bundle) == IMM8_OPCODE_X0)
858 && (get_Imm8OpcodeExtension_X0(bundle) ==
859 ADDI_IMM8_OPCODE_X0) &&
860 (get_SrcA_X0(bundle) == TREG_ZERO) &&
861 (get_Imm8_X0(bundle) == 0)) {
862 rx = get_Dest_X0(bundle);
863 } else {
864 unexpected = true;
865 }
866 }
867
868 /* rx should be less than 56. */
869 if (!unexpected && (rx >= 56))
870 unexpected = true;
871 }
872
873 if (!search_exception_tables(regs->pc)) {
874 /* No fixup in the exception tables for the pc. */
875 unexpected = true;
876 }
877
878 if (unexpected) {
879 /* Unexpected unalign kernel fault. */
880 struct task_struct *tsk = validate_current();
881
882 bust_spinlocks(1);
883
884 show_regs(regs);
885
886 if (unlikely(tsk->pid < 2)) {
887 panic("Kernel unalign fault running %s!",
888 tsk->pid ? "init" : "the idle task");
889 }
890 #ifdef SUPPORT_DIE
891 die("Oops", regs);
892 #endif
893 bust_spinlocks(1);
894
895 do_group_exit(SIGKILL);
896
897 } else {
898 unsigned long i, b = 0;
899 unsigned char *ptr =
900 (unsigned char *)regs->regs[ra];
901 if (load_n_store) {
902 /* handle get_user(x, ptr) */
903 for (i = 0; i < load_store_size; i++) {
904 ret = get_user(b, ptr++);
905 if (!ret) {
906 /* Success! update x. */
907 #ifdef __LITTLE_ENDIAN
908 x |= (b << (8 * i));
909 #else
910 x <<= 8;
911 x |= b;
912 #endif /* __LITTLE_ENDIAN */
913 } else {
914 x = 0;
915 break;
916 }
917 }
918
919 /* Sign-extend 4-byte loads. */
920 if (load_store_size == 4)
921 x = (long)(int)x;
922
923 /* Set register rd. */
924 regs->regs[rd] = x;
925
926 /* Set register rx. */
927 regs->regs[rx] = ret;
928
929 /* Bump pc. */
930 regs->pc += 8;
931
932 } else {
933 /* Handle put_user(x, ptr) */
934 x = regs->regs[rb];
935 #ifdef __LITTLE_ENDIAN
936 b = x;
937 #else
938 /*
939 * Swap x in order to store x from low
940 * to high memory same as the
941 * little-endian case.
942 */
943 switch (load_store_size) {
944 case 8:
945 b = swab64(x);
946 break;
947 case 4:
948 b = swab32(x);
949 break;
950 case 2:
951 b = swab16(x);
952 break;
953 }
954 #endif /* __LITTLE_ENDIAN */
955 for (i = 0; i < load_store_size; i++) {
956 ret = put_user(b, ptr++);
957 if (ret)
958 break;
959 /* Success! shift 1 byte. */
960 b >>= 8;
961 }
962 /* Set register rx. */
963 regs->regs[rx] = ret;
964
965 /* Bump pc. */
966 regs->pc += 8;
967 }
968 }
969
970 unaligned_fixup_count++;
971
972 if (unaligned_printk) {
973 pr_info("%s/%d - Unalign fixup for kernel access to userspace %lx\n",
974 current->comm, current->pid, regs->regs[ra]);
975 }
976
977 /* Done! Return to the exception handler. */
978 return;
979 }
980
981 if ((align_ctl == 0) || unexpected) {
982 siginfo_t info = {
983 .si_signo = SIGBUS,
984 .si_code = BUS_ADRALN,
985 .si_addr = (unsigned char __user *)0
986 };
987 if (unaligned_printk)
988 pr_info("Unalign bundle: unexp @%llx, %llx\n",
989 (unsigned long long)regs->pc,
990 (unsigned long long)bundle);
991
992 if (ra < 56) {
993 unsigned long uaa = (unsigned long)regs->regs[ra];
994 /* Set bus Address. */
995 info.si_addr = (unsigned char __user *)uaa;
996 }
997
998 unaligned_fixup_count++;
999
1000 trace_unhandled_signal("unaligned fixup trap", regs,
1001 (unsigned long)info.si_addr, SIGBUS);
1002 force_sig_info(info.si_signo, &info, current);
1003 return;
1004 }
1005
1006 #ifdef __LITTLE_ENDIAN
1007 #define UA_FIXUP_ADDR_DELTA 1
1008 #define UA_FIXUP_BFEXT_START(_B_) 0
1009 #define UA_FIXUP_BFEXT_END(_B_) (8 * (_B_) - 1)
1010 #else /* __BIG_ENDIAN */
1011 #define UA_FIXUP_ADDR_DELTA -1
1012 #define UA_FIXUP_BFEXT_START(_B_) (64 - 8 * (_B_))
1013 #define UA_FIXUP_BFEXT_END(_B_) 63
1014 #endif /* __LITTLE_ENDIAN */
1015
1016
1017
1018 if ((ra != rb) && (rd != TREG_SP) && !alias &&
1019 !y1_br && !y1_lr && !x1_add) {
1020 /*
1021 * Simple case: ra != rb and no register alias found,
1022 * and no branch or link. This will be the majority.
1023 * We can do a little better for simplae case than the
1024 * generic scheme below.
1025 */
1026 if (!load_n_store) {
1027 /*
1028 * Simple store: ra != rb, no need for scratch register.
1029 * Just store and rotate to right bytewise.
1030 */
1031 #ifdef __BIG_ENDIAN
1032 frag.insn[n++] =
1033 jit_x0_addi(ra, ra, load_store_size - 1) |
1034 jit_x1_fnop();
1035 #endif /* __BIG_ENDIAN */
1036 for (k = 0; k < load_store_size; k++) {
1037 /* Store a byte. */
1038 frag.insn[n++] =
1039 jit_x0_rotli(rb, rb, 56) |
1040 jit_x1_st1_add(ra, rb,
1041 UA_FIXUP_ADDR_DELTA);
1042 }
1043 #ifdef __BIG_ENDIAN
1044 frag.insn[n] = jit_x1_addi(ra, ra, 1);
1045 #else
1046 frag.insn[n] = jit_x1_addi(ra, ra,
1047 -1 * load_store_size);
1048 #endif /* __LITTLE_ENDIAN */
1049
1050 if (load_store_size == 8) {
1051 frag.insn[n] |= jit_x0_fnop();
1052 } else if (load_store_size == 4) {
1053 frag.insn[n] |= jit_x0_rotli(rb, rb, 32);
1054 } else { /* = 2 */
1055 frag.insn[n] |= jit_x0_rotli(rb, rb, 16);
1056 }
1057 n++;
1058 if (bundle_2_enable)
1059 frag.insn[n++] = bundle_2;
1060 frag.insn[n++] = jit_x0_fnop() | jit_x1_iret();
1061 } else {
1062 if (rd == ra) {
1063 /* Use two clobber registers: clob1/2. */
1064 frag.insn[n++] =
1065 jit_x0_addi(TREG_SP, TREG_SP, -16) |
1066 jit_x1_fnop();
1067 frag.insn[n++] =
1068 jit_x0_addi(clob1, ra, 7) |
1069 jit_x1_st_add(TREG_SP, clob1, -8);
1070 frag.insn[n++] =
1071 jit_x0_addi(clob2, ra, 0) |
1072 jit_x1_st(TREG_SP, clob2);
1073 frag.insn[n++] =
1074 jit_x0_fnop() |
1075 jit_x1_ldna(rd, ra);
1076 frag.insn[n++] =
1077 jit_x0_fnop() |
1078 jit_x1_ldna(clob1, clob1);
1079 /*
1080 * Note: we must make sure that rd must not
1081 * be sp. Recover clob1/2 from stack.
1082 */
1083 frag.insn[n++] =
1084 jit_x0_dblalign(rd, clob1, clob2) |
1085 jit_x1_ld_add(clob2, TREG_SP, 8);
1086 frag.insn[n++] =
1087 jit_x0_fnop() |
1088 jit_x1_ld_add(clob1, TREG_SP, 16);
1089 } else {
1090 /* Use one clobber register: clob1 only. */
1091 frag.insn[n++] =
1092 jit_x0_addi(TREG_SP, TREG_SP, -16) |
1093 jit_x1_fnop();
1094 frag.insn[n++] =
1095 jit_x0_addi(clob1, ra, 7) |
1096 jit_x1_st(TREG_SP, clob1);
1097 frag.insn[n++] =
1098 jit_x0_fnop() |
1099 jit_x1_ldna(rd, ra);
1100 frag.insn[n++] =
1101 jit_x0_fnop() |
1102 jit_x1_ldna(clob1, clob1);
1103 /*
1104 * Note: we must make sure that rd must not
1105 * be sp. Recover clob1 from stack.
1106 */
1107 frag.insn[n++] =
1108 jit_x0_dblalign(rd, clob1, ra) |
1109 jit_x1_ld_add(clob1, TREG_SP, 16);
1110 }
1111
1112 if (bundle_2_enable)
1113 frag.insn[n++] = bundle_2;
1114 /*
1115 * For non 8-byte load, extract corresponding bytes and
1116 * signed extension.
1117 */
1118 if (load_store_size == 4) {
1119 if (load_store_signed)
1120 frag.insn[n++] =
1121 jit_x0_bfexts(
1122 rd, rd,
1123 UA_FIXUP_BFEXT_START(4),
1124 UA_FIXUP_BFEXT_END(4)) |
1125 jit_x1_fnop();
1126 else
1127 frag.insn[n++] =
1128 jit_x0_bfextu(
1129 rd, rd,
1130 UA_FIXUP_BFEXT_START(4),
1131 UA_FIXUP_BFEXT_END(4)) |
1132 jit_x1_fnop();
1133 } else if (load_store_size == 2) {
1134 if (load_store_signed)
1135 frag.insn[n++] =
1136 jit_x0_bfexts(
1137 rd, rd,
1138 UA_FIXUP_BFEXT_START(2),
1139 UA_FIXUP_BFEXT_END(2)) |
1140 jit_x1_fnop();
1141 else
1142 frag.insn[n++] =
1143 jit_x0_bfextu(
1144 rd, rd,
1145 UA_FIXUP_BFEXT_START(2),
1146 UA_FIXUP_BFEXT_END(2)) |
1147 jit_x1_fnop();
1148 }
1149
1150 frag.insn[n++] =
1151 jit_x0_fnop() |
1152 jit_x1_iret();
1153 }
1154 } else if (!load_n_store) {
1155
1156 /*
1157 * Generic memory store cases: use 3 clobber registers.
1158 *
1159 * Alloc space for saveing clob2,1,3 on user's stack.
1160 * register clob3 points to where clob2 saved, followed by
1161 * clob1 and 3 from high to low memory.
1162 */
1163 frag.insn[n++] =
1164 jit_x0_addi(TREG_SP, TREG_SP, -32) |
1165 jit_x1_fnop();
1166 frag.insn[n++] =
1167 jit_x0_addi(clob3, TREG_SP, 16) |
1168 jit_x1_st_add(TREG_SP, clob3, 8);
1169 #ifdef __LITTLE_ENDIAN
1170 frag.insn[n++] =
1171 jit_x0_addi(clob1, ra, 0) |
1172 jit_x1_st_add(TREG_SP, clob1, 8);
1173 #else
1174 frag.insn[n++] =
1175 jit_x0_addi(clob1, ra, load_store_size - 1) |
1176 jit_x1_st_add(TREG_SP, clob1, 8);
1177 #endif
1178 if (load_store_size == 8) {
1179 /*
1180 * We save one byte a time, not for fast, but compact
1181 * code. After each store, data source register shift
1182 * right one byte. unchanged after 8 stores.
1183 */
1184 frag.insn[n++] =
1185 jit_x0_addi(clob2, TREG_ZERO, 7) |
1186 jit_x1_st_add(TREG_SP, clob2, 16);
1187 frag.insn[n++] =
1188 jit_x0_rotli(rb, rb, 56) |
1189 jit_x1_st1_add(clob1, rb, UA_FIXUP_ADDR_DELTA);
1190 frag.insn[n++] =
1191 jit_x0_addi(clob2, clob2, -1) |
1192 jit_x1_bnezt(clob2, -1);
1193 frag.insn[n++] =
1194 jit_x0_fnop() |
1195 jit_x1_addi(clob2, y1_br_reg, 0);
1196 } else if (load_store_size == 4) {
1197 frag.insn[n++] =
1198 jit_x0_addi(clob2, TREG_ZERO, 3) |
1199 jit_x1_st_add(TREG_SP, clob2, 16);
1200 frag.insn[n++] =
1201 jit_x0_rotli(rb, rb, 56) |
1202 jit_x1_st1_add(clob1, rb, UA_FIXUP_ADDR_DELTA);
1203 frag.insn[n++] =
1204 jit_x0_addi(clob2, clob2, -1) |
1205 jit_x1_bnezt(clob2, -1);
1206 /*
1207 * same as 8-byte case, but need shift another 4
1208 * byte to recover rb for 4-byte store.
1209 */
1210 frag.insn[n++] = jit_x0_rotli(rb, rb, 32) |
1211 jit_x1_addi(clob2, y1_br_reg, 0);
1212 } else { /* =2 */
1213 frag.insn[n++] =
1214 jit_x0_addi(clob2, rb, 0) |
1215 jit_x1_st_add(TREG_SP, clob2, 16);
1216 for (k = 0; k < 2; k++) {
1217 frag.insn[n++] =
1218 jit_x0_shrui(rb, rb, 8) |
1219 jit_x1_st1_add(clob1, rb,
1220 UA_FIXUP_ADDR_DELTA);
1221 }
1222 frag.insn[n++] =
1223 jit_x0_addi(rb, clob2, 0) |
1224 jit_x1_addi(clob2, y1_br_reg, 0);
1225 }
1226
1227 if (bundle_2_enable)
1228 frag.insn[n++] = bundle_2;
1229
1230 if (y1_lr) {
1231 frag.insn[n++] =
1232 jit_x0_fnop() |
1233 jit_x1_mfspr(y1_lr_reg,
1234 SPR_EX_CONTEXT_0_0);
1235 }
1236 if (y1_br) {
1237 frag.insn[n++] =
1238 jit_x0_fnop() |
1239 jit_x1_mtspr(SPR_EX_CONTEXT_0_0,
1240 clob2);
1241 }
1242 if (x1_add) {
1243 frag.insn[n++] =
1244 jit_x0_addi(ra, ra, x1_add_imm8) |
1245 jit_x1_ld_add(clob2, clob3, -8);
1246 } else {
1247 frag.insn[n++] =
1248 jit_x0_fnop() |
1249 jit_x1_ld_add(clob2, clob3, -8);
1250 }
1251 frag.insn[n++] =
1252 jit_x0_fnop() |
1253 jit_x1_ld_add(clob1, clob3, -8);
1254 frag.insn[n++] = jit_x0_fnop() | jit_x1_ld(clob3, clob3);
1255 frag.insn[n++] = jit_x0_fnop() | jit_x1_iret();
1256
1257 } else {
1258 /*
1259 * Generic memory load cases.
1260 *
1261 * Alloc space for saveing clob1,2,3 on user's stack.
1262 * register clob3 points to where clob1 saved, followed
1263 * by clob2 and 3 from high to low memory.
1264 */
1265
1266 frag.insn[n++] =
1267 jit_x0_addi(TREG_SP, TREG_SP, -32) |
1268 jit_x1_fnop();
1269 frag.insn[n++] =
1270 jit_x0_addi(clob3, TREG_SP, 16) |
1271 jit_x1_st_add(TREG_SP, clob3, 8);
1272 frag.insn[n++] =
1273 jit_x0_addi(clob2, ra, 0) |
1274 jit_x1_st_add(TREG_SP, clob2, 8);
1275
1276 if (y1_br) {
1277 frag.insn[n++] =
1278 jit_x0_addi(clob1, y1_br_reg, 0) |
1279 jit_x1_st_add(TREG_SP, clob1, 16);
1280 } else {
1281 frag.insn[n++] =
1282 jit_x0_fnop() |
1283 jit_x1_st_add(TREG_SP, clob1, 16);
1284 }
1285
1286 if (bundle_2_enable)
1287 frag.insn[n++] = bundle_2;
1288
1289 if (y1_lr) {
1290 frag.insn[n++] =
1291 jit_x0_fnop() |
1292 jit_x1_mfspr(y1_lr_reg,
1293 SPR_EX_CONTEXT_0_0);
1294 }
1295
1296 if (y1_br) {
1297 frag.insn[n++] =
1298 jit_x0_fnop() |
1299 jit_x1_mtspr(SPR_EX_CONTEXT_0_0,
1300 clob1);
1301 }
1302
1303 frag.insn[n++] =
1304 jit_x0_addi(clob1, clob2, 7) |
1305 jit_x1_ldna(rd, clob2);
1306 frag.insn[n++] =
1307 jit_x0_fnop() |
1308 jit_x1_ldna(clob1, clob1);
1309 frag.insn[n++] =
1310 jit_x0_dblalign(rd, clob1, clob2) |
1311 jit_x1_ld_add(clob1, clob3, -8);
1312 if (x1_add) {
1313 frag.insn[n++] =
1314 jit_x0_addi(ra, ra, x1_add_imm8) |
1315 jit_x1_ld_add(clob2, clob3, -8);
1316 } else {
1317 frag.insn[n++] =
1318 jit_x0_fnop() |
1319 jit_x1_ld_add(clob2, clob3, -8);
1320 }
1321
1322 frag.insn[n++] =
1323 jit_x0_fnop() |
1324 jit_x1_ld(clob3, clob3);
1325
1326 if (load_store_size == 4) {
1327 if (load_store_signed)
1328 frag.insn[n++] =
1329 jit_x0_bfexts(
1330 rd, rd,
1331 UA_FIXUP_BFEXT_START(4),
1332 UA_FIXUP_BFEXT_END(4)) |
1333 jit_x1_fnop();
1334 else
1335 frag.insn[n++] =
1336 jit_x0_bfextu(
1337 rd, rd,
1338 UA_FIXUP_BFEXT_START(4),
1339 UA_FIXUP_BFEXT_END(4)) |
1340 jit_x1_fnop();
1341 } else if (load_store_size == 2) {
1342 if (load_store_signed)
1343 frag.insn[n++] =
1344 jit_x0_bfexts(
1345 rd, rd,
1346 UA_FIXUP_BFEXT_START(2),
1347 UA_FIXUP_BFEXT_END(2)) |
1348 jit_x1_fnop();
1349 else
1350 frag.insn[n++] =
1351 jit_x0_bfextu(
1352 rd, rd,
1353 UA_FIXUP_BFEXT_START(2),
1354 UA_FIXUP_BFEXT_END(2)) |
1355 jit_x1_fnop();
1356 }
1357
1358 frag.insn[n++] = jit_x0_fnop() | jit_x1_iret();
1359 }
1360
1361 /* Max JIT bundle count is 14. */
1362 WARN_ON(n > 14);
1363
1364 if (!unexpected) {
1365 int status = 0;
1366 int idx = (regs->pc >> 3) &
1367 ((1ULL << (PAGE_SHIFT - UNALIGN_JIT_SHIFT)) - 1);
1368
1369 frag.pc = regs->pc;
1370 frag.bundle = bundle;
1371
1372 if (unaligned_printk) {
1373 pr_info("%s/%d, Unalign fixup: pc=%lx bundle=%lx %d %d %d %d %d %d %d %d\n",
1374 current->comm, current->pid,
1375 (unsigned long)frag.pc,
1376 (unsigned long)frag.bundle,
1377 (int)alias, (int)rd, (int)ra,
1378 (int)rb, (int)bundle_2_enable,
1379 (int)y1_lr, (int)y1_br, (int)x1_add);
1380
1381 for (k = 0; k < n; k += 2)
1382 pr_info("[%d] %016llx %016llx\n",
1383 k, (unsigned long long)frag.insn[k],
1384 (unsigned long long)frag.insn[k+1]);
1385 }
1386
1387 /* Swap bundle byte order for big endian sys. */
1388 #ifdef __BIG_ENDIAN
1389 frag.bundle = GX_INSN_BSWAP(frag.bundle);
1390 for (k = 0; k < n; k++)
1391 frag.insn[k] = GX_INSN_BSWAP(frag.insn[k]);
1392 #endif /* __BIG_ENDIAN */
1393
1394 status = copy_to_user((void __user *)&jit_code_area[idx],
1395 &frag, sizeof(frag));
1396 if (status) {
1397 /* Fail to copy JIT into user land. send SIGSEGV. */
1398 siginfo_t info = {
1399 .si_signo = SIGSEGV,
1400 .si_code = SEGV_MAPERR,
1401 .si_addr = (void __user *)&jit_code_area[idx]
1402 };
1403
1404 pr_warn("Unalign fixup: pid=%d %s jit_code_area=%llx\n",
1405 current->pid, current->comm,
1406 (unsigned long long)&jit_code_area[idx]);
1407
1408 trace_unhandled_signal("segfault in unalign fixup",
1409 regs,
1410 (unsigned long)info.si_addr,
1411 SIGSEGV);
1412 force_sig_info(info.si_signo, &info, current);
1413 return;
1414 }
1415
1416
1417 /* Do a cheaper increment, not accurate. */
1418 unaligned_fixup_count++;
1419 __flush_icache_range((unsigned long)&jit_code_area[idx],
1420 (unsigned long)&jit_code_area[idx] +
1421 sizeof(frag));
1422
1423 /* Setup SPR_EX_CONTEXT_0_0/1 for returning to user program.*/
1424 __insn_mtspr(SPR_EX_CONTEXT_0_0, regs->pc + 8);
1425 __insn_mtspr(SPR_EX_CONTEXT_0_1, PL_ICS_EX1(USER_PL, 0));
1426
1427 /* Modify pc at the start of new JIT. */
1428 regs->pc = (unsigned long)&jit_code_area[idx].insn[0];
1429 /* Set ICS in SPR_EX_CONTEXT_K_1. */
1430 regs->ex1 = PL_ICS_EX1(USER_PL, 1);
1431 }
1432 }
1433
1434
1435 /*
1436 * C function to generate unalign data JIT. Called from unalign data
1437 * interrupt handler.
1438 *
1439 * First check if unalign fix is disabled or exception did not not come from
1440 * user space or sp register points to unalign address, if true, generate a
1441 * SIGBUS. Then map a page into user space as JIT area if it is not mapped
1442 * yet. Genenerate JIT code by calling jit_bundle_gen(). After that return
1443 * back to exception handler.
1444 *
1445 * The exception handler will "iret" to new generated JIT code after
1446 * restoring caller saved registers. In theory, the JIT code will perform
1447 * another "iret" to resume user's program.
1448 */
1449
1450 void do_unaligned(struct pt_regs *regs, int vecnum)
1451 {
1452 enum ctx_state prev_state = exception_enter();
1453 tilegx_bundle_bits __user *pc;
1454 tilegx_bundle_bits bundle;
1455 struct thread_info *info = current_thread_info();
1456 int align_ctl;
1457
1458 /* Checks the per-process unaligned JIT flags */
1459 align_ctl = unaligned_fixup;
1460 switch (task_thread_info(current)->align_ctl) {
1461 case PR_UNALIGN_NOPRINT:
1462 align_ctl = 1;
1463 break;
1464 case PR_UNALIGN_SIGBUS:
1465 align_ctl = 0;
1466 break;
1467 }
1468
1469 /* Enable iterrupt in order to access user land. */
1470 local_irq_enable();
1471
1472 /*
1473 * The fault came from kernel space. Two choices:
1474 * (a) unaligned_fixup < 1, we will first call get/put_user fixup
1475 * to return -EFAULT. If no fixup, simply panic the kernel.
1476 * (b) unaligned_fixup >=1, we will try to fix the unaligned access
1477 * if it was triggered by get_user/put_user() macros. Panic the
1478 * kernel if it is not fixable.
1479 */
1480
1481 if (EX1_PL(regs->ex1) != USER_PL) {
1482
1483 if (align_ctl < 1) {
1484 unaligned_fixup_count++;
1485 /* If exception came from kernel, try fix it up. */
1486 if (fixup_exception(regs)) {
1487 if (unaligned_printk)
1488 pr_info("Unalign fixup: %d %llx @%llx\n",
1489 (int)unaligned_fixup,
1490 (unsigned long long)regs->ex1,
1491 (unsigned long long)regs->pc);
1492 } else {
1493 /* Not fixable. Go panic. */
1494 panic("Unalign exception in Kernel. pc=%lx",
1495 regs->pc);
1496 }
1497 } else {
1498 /*
1499 * Try to fix the exception. If we can't, panic the
1500 * kernel.
1501 */
1502 bundle = GX_INSN_BSWAP(
1503 *((tilegx_bundle_bits *)(regs->pc)));
1504 jit_bundle_gen(regs, bundle, align_ctl);
1505 }
1506 goto done;
1507 }
1508
1509 /*
1510 * Fault came from user with ICS or stack is not aligned.
1511 * If so, we will trigger SIGBUS.
1512 */
1513 if ((regs->sp & 0x7) || (regs->ex1) || (align_ctl < 0)) {
1514 siginfo_t info = {
1515 .si_signo = SIGBUS,
1516 .si_code = BUS_ADRALN,
1517 .si_addr = (unsigned char __user *)0
1518 };
1519
1520 if (unaligned_printk)
1521 pr_info("Unalign fixup: %d %llx @%llx\n",
1522 (int)unaligned_fixup,
1523 (unsigned long long)regs->ex1,
1524 (unsigned long long)regs->pc);
1525
1526 unaligned_fixup_count++;
1527
1528 trace_unhandled_signal("unaligned fixup trap", regs, 0, SIGBUS);
1529 force_sig_info(info.si_signo, &info, current);
1530 goto done;
1531 }
1532
1533
1534 /* Read the bundle casued the exception! */
1535 pc = (tilegx_bundle_bits __user *)(regs->pc);
1536 if (get_user(bundle, pc) != 0) {
1537 /* Probably never be here since pc is valid user address.*/
1538 siginfo_t info = {
1539 .si_signo = SIGSEGV,
1540 .si_code = SEGV_MAPERR,
1541 .si_addr = (void __user *)pc
1542 };
1543 pr_err("Couldn't read instruction at %p trying to step\n", pc);
1544 trace_unhandled_signal("segfault in unalign fixup", regs,
1545 (unsigned long)info.si_addr, SIGSEGV);
1546 force_sig_info(info.si_signo, &info, current);
1547 goto done;
1548 }
1549
1550 if (!info->unalign_jit_base) {
1551 void __user *user_page;
1552
1553 /*
1554 * Allocate a page in userland.
1555 * For 64-bit processes we try to place the mapping far
1556 * from anything else that might be going on (specifically
1557 * 64 GB below the top of the user address space). If it
1558 * happens not to be possible to put it there, it's OK;
1559 * the kernel will choose another location and we'll
1560 * remember it for later.
1561 */
1562 if (is_compat_task())
1563 user_page = NULL;
1564 else
1565 user_page = (void __user *)(TASK_SIZE - (1UL << 36)) +
1566 (current->pid << PAGE_SHIFT);
1567
1568 user_page = (void __user *) vm_mmap(NULL,
1569 (unsigned long)user_page,
1570 PAGE_SIZE,
1571 PROT_EXEC | PROT_READ |
1572 PROT_WRITE,
1573 #ifdef CONFIG_HOMECACHE
1574 MAP_CACHE_HOME_TASK |
1575 #endif
1576 MAP_PRIVATE |
1577 MAP_ANONYMOUS,
1578 0);
1579
1580 if (IS_ERR((void __force *)user_page)) {
1581 pr_err("Out of kernel pages trying do_mmap\n");
1582 goto done;
1583 }
1584
1585 /* Save the address in the thread_info struct */
1586 info->unalign_jit_base = user_page;
1587 if (unaligned_printk)
1588 pr_info("Unalign bundle: %d:%d, allocate page @%llx\n",
1589 raw_smp_processor_id(), current->pid,
1590 (unsigned long long)user_page);
1591 }
1592
1593 /* Generate unalign JIT */
1594 jit_bundle_gen(regs, GX_INSN_BSWAP(bundle), align_ctl);
1595
1596 done:
1597 exception_exit(prev_state);
1598 }
1599
1600 #endif /* __tilegx__ */
Linux with added FPC-III support