?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[TG3]: Set minimal hw interrupt mitigation.
[linux-2.6/verdex.git] / arch / ia64 / kernel / unwind.c
blob2776a074c6f10e92970c86b14c40af2689e80996
1 /*
2 * Copyright (C) 1999-2004 Hewlett-Packard Co
3 * David Mosberger-Tang <davidm@hpl.hp.com>
4 * Copyright (C) 2003 Fenghua Yu <fenghua.yu@intel.com>
5 * - Change pt_regs_off() to make it less dependant on pt_regs structure.
6 */
7 /*
8 * This file implements call frame unwind support for the Linux
9 * kernel. Parsing and processing the unwind information is
10 * time-consuming, so this implementation translates the unwind
11 * descriptors into unwind scripts. These scripts are very simple
12 * (basically a sequence of assignments) and efficient to execute.
13 * They are cached for later re-use. Each script is specific for a
14 * given instruction pointer address and the set of predicate values
15 * that the script depends on (most unwind descriptors are
16 * unconditional and scripts often do not depend on predicates at
17 * all). This code is based on the unwind conventions described in
18 * the "IA-64 Software Conventions and Runtime Architecture" manual.
19 *
20 * SMP conventions:
21 * o updates to the global unwind data (in structure "unw") are serialized
22 * by the unw.lock spinlock
23 * o each unwind script has its own read-write lock; a thread must acquire
24 * a read lock before executing a script and must acquire a write lock
25 * before modifying a script
26 * o if both the unw.lock spinlock and a script's read-write lock must be
27 * acquired, then the read-write lock must be acquired first.
28 */
29 #include <linux/module.h>
30 #include <linux/bootmem.h>
31 #include <linux/elf.h>
32 #include <linux/kernel.h>
33 #include <linux/sched.h>
34 #include <linux/slab.h>
35
36 #include <asm/unwind.h>
37
38 #include <asm/delay.h>
39 #include <asm/page.h>
40 #include <asm/ptrace.h>
41 #include <asm/ptrace_offsets.h>
42 #include <asm/rse.h>
43 #include <asm/sections.h>
44 #include <asm/system.h>
45 #include <asm/uaccess.h>
46
47 #include "entry.h"
48 #include "unwind_i.h"
49
50 #define UNW_LOG_CACHE_SIZE 7 /* each unw_script is ~256 bytes in size */
51 #define UNW_CACHE_SIZE (1 << UNW_LOG_CACHE_SIZE)
52
53 #define UNW_LOG_HASH_SIZE (UNW_LOG_CACHE_SIZE + 1)
54 #define UNW_HASH_SIZE (1 << UNW_LOG_HASH_SIZE)
55
56 #define UNW_STATS 0 /* WARNING: this disabled interrupts for long time-spans!! */
57
58 #ifdef UNW_DEBUG
59 static unsigned int unw_debug_level = UNW_DEBUG;
60 # define UNW_DEBUG_ON(n) unw_debug_level >= n
61 /* Do not code a printk level, not all debug lines end in newline */
62 # define UNW_DPRINT(n, ...) if (UNW_DEBUG_ON(n)) printk(__VA_ARGS__)
63 # define inline
64 #else /* !UNW_DEBUG */
65 # define UNW_DEBUG_ON(n) 0
66 # define UNW_DPRINT(n, ...)
67 #endif /* UNW_DEBUG */
68
69 #if UNW_STATS
70 # define STAT(x...) x
71 #else
72 # define STAT(x...)
73 #endif
74
75 #define alloc_reg_state() kmalloc(sizeof(struct unw_reg_state), GFP_ATOMIC)
76 #define free_reg_state(usr) kfree(usr)
77 #define alloc_labeled_state() kmalloc(sizeof(struct unw_labeled_state), GFP_ATOMIC)
78 #define free_labeled_state(usr) kfree(usr)
79
80 typedef unsigned long unw_word;
81 typedef unsigned char unw_hash_index_t;
82
83 static struct {
84 spinlock_t lock; /* spinlock for unwind data */
85
86 /* list of unwind tables (one per load-module) */
87 struct unw_table *tables;
88
89 unsigned long r0; /* constant 0 for r0 */
90
91 /* table of registers that prologues can save (and order in which they're saved): */
92 const unsigned char save_order[8];
93
94 /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
95 unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
96
97 unsigned short lru_head; /* index of lead-recently used script */
98 unsigned short lru_tail; /* index of most-recently used script */
99
100 /* index into unw_frame_info for preserved register i */
101 unsigned short preg_index[UNW_NUM_REGS];
102
103 short pt_regs_offsets[32];
104
105 /* unwind table for the kernel: */
106 struct unw_table kernel_table;
107
108 /* unwind table describing the gate page (kernel code that is mapped into user space): */
109 size_t gate_table_size;
110 unsigned long *gate_table;
111
112 /* hash table that maps instruction pointer to script index: */
113 unsigned short hash[UNW_HASH_SIZE];
114
115 /* script cache: */
116 struct unw_script cache[UNW_CACHE_SIZE];
117
118 # ifdef UNW_DEBUG
119 const char *preg_name[UNW_NUM_REGS];
120 # endif
121 # if UNW_STATS
122 struct {
123 struct {
124 int lookups;
125 int hinted_hits;
126 int normal_hits;
127 int collision_chain_traversals;
128 } cache;
129 struct {
130 unsigned long build_time;
131 unsigned long run_time;
132 unsigned long parse_time;
133 int builds;
134 int news;
135 int collisions;
136 int runs;
137 } script;
138 struct {
139 unsigned long init_time;
140 unsigned long unwind_time;
141 int inits;
142 int unwinds;
143 } api;
144 } stat;
145 # endif
146 } unw = {
147 .tables = &unw.kernel_table,
148 .lock = SPIN_LOCK_UNLOCKED,
149 .save_order = {
150 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
151 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
152 },
153 .preg_index = {
154 offsetof(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_GR */
155 offsetof(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_MEM */
156 offsetof(struct unw_frame_info, bsp_loc)/8,
157 offsetof(struct unw_frame_info, bspstore_loc)/8,
158 offsetof(struct unw_frame_info, pfs_loc)/8,
159 offsetof(struct unw_frame_info, rnat_loc)/8,
160 offsetof(struct unw_frame_info, psp)/8,
161 offsetof(struct unw_frame_info, rp_loc)/8,
162 offsetof(struct unw_frame_info, r4)/8,
163 offsetof(struct unw_frame_info, r5)/8,
164 offsetof(struct unw_frame_info, r6)/8,
165 offsetof(struct unw_frame_info, r7)/8,
166 offsetof(struct unw_frame_info, unat_loc)/8,
167 offsetof(struct unw_frame_info, pr_loc)/8,
168 offsetof(struct unw_frame_info, lc_loc)/8,
169 offsetof(struct unw_frame_info, fpsr_loc)/8,
170 offsetof(struct unw_frame_info, b1_loc)/8,
171 offsetof(struct unw_frame_info, b2_loc)/8,
172 offsetof(struct unw_frame_info, b3_loc)/8,
173 offsetof(struct unw_frame_info, b4_loc)/8,
174 offsetof(struct unw_frame_info, b5_loc)/8,
175 offsetof(struct unw_frame_info, f2_loc)/8,
176 offsetof(struct unw_frame_info, f3_loc)/8,
177 offsetof(struct unw_frame_info, f4_loc)/8,
178 offsetof(struct unw_frame_info, f5_loc)/8,
179 offsetof(struct unw_frame_info, fr_loc[16 - 16])/8,
180 offsetof(struct unw_frame_info, fr_loc[17 - 16])/8,
181 offsetof(struct unw_frame_info, fr_loc[18 - 16])/8,
182 offsetof(struct unw_frame_info, fr_loc[19 - 16])/8,
183 offsetof(struct unw_frame_info, fr_loc[20 - 16])/8,
184 offsetof(struct unw_frame_info, fr_loc[21 - 16])/8,
185 offsetof(struct unw_frame_info, fr_loc[22 - 16])/8,
186 offsetof(struct unw_frame_info, fr_loc[23 - 16])/8,
187 offsetof(struct unw_frame_info, fr_loc[24 - 16])/8,
188 offsetof(struct unw_frame_info, fr_loc[25 - 16])/8,
189 offsetof(struct unw_frame_info, fr_loc[26 - 16])/8,
190 offsetof(struct unw_frame_info, fr_loc[27 - 16])/8,
191 offsetof(struct unw_frame_info, fr_loc[28 - 16])/8,
192 offsetof(struct unw_frame_info, fr_loc[29 - 16])/8,
193 offsetof(struct unw_frame_info, fr_loc[30 - 16])/8,
194 offsetof(struct unw_frame_info, fr_loc[31 - 16])/8,
195 },
196 .pt_regs_offsets = {
197 [0] = -1,
198 offsetof(struct pt_regs, r1),
199 offsetof(struct pt_regs, r2),
200 offsetof(struct pt_regs, r3),
201 [4] = -1, [5] = -1, [6] = -1, [7] = -1,
202 offsetof(struct pt_regs, r8),
203 offsetof(struct pt_regs, r9),
204 offsetof(struct pt_regs, r10),
205 offsetof(struct pt_regs, r11),
206 offsetof(struct pt_regs, r12),
207 offsetof(struct pt_regs, r13),
208 offsetof(struct pt_regs, r14),
209 offsetof(struct pt_regs, r15),
210 offsetof(struct pt_regs, r16),
211 offsetof(struct pt_regs, r17),
212 offsetof(struct pt_regs, r18),
213 offsetof(struct pt_regs, r19),
214 offsetof(struct pt_regs, r20),
215 offsetof(struct pt_regs, r21),
216 offsetof(struct pt_regs, r22),
217 offsetof(struct pt_regs, r23),
218 offsetof(struct pt_regs, r24),
219 offsetof(struct pt_regs, r25),
220 offsetof(struct pt_regs, r26),
221 offsetof(struct pt_regs, r27),
222 offsetof(struct pt_regs, r28),
223 offsetof(struct pt_regs, r29),
224 offsetof(struct pt_regs, r30),
225 offsetof(struct pt_regs, r31),
226 },
227 .hash = { [0 ... UNW_HASH_SIZE - 1] = -1 },
228 #ifdef UNW_DEBUG
229 .preg_name = {
230 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
231 "r4", "r5", "r6", "r7",
232 "ar.unat", "pr", "ar.lc", "ar.fpsr",
233 "b1", "b2", "b3", "b4", "b5",
234 "f2", "f3", "f4", "f5",
235 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
236 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
237 }
238 #endif
239 };
240
241 static inline int
242 read_only (void *addr)
243 {
244 return (unsigned long) ((char *) addr - (char *) &unw.r0) < sizeof(unw.r0);
245 }
246
247 /*
248 * Returns offset of rREG in struct pt_regs.
249 */
250 static inline unsigned long
251 pt_regs_off (unsigned long reg)
252 {
253 short off = -1;
254
255 if (reg < ARRAY_SIZE(unw.pt_regs_offsets))
256 off = unw.pt_regs_offsets[reg];
257
258 if (off < 0) {
259 UNW_DPRINT(0, "unwind.%s: bad scratch reg r%lu\n", __FUNCTION__, reg);
260 off = 0;
261 }
262 return (unsigned long) off;
263 }
264
265 static inline struct pt_regs *
266 get_scratch_regs (struct unw_frame_info *info)
267 {
268 if (!info->pt) {
269 /* This should not happen with valid unwind info. */
270 UNW_DPRINT(0, "unwind.%s: bad unwind info: resetting info->pt\n", __FUNCTION__);
271 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
272 info->pt = (unsigned long) ((struct pt_regs *) info->psp - 1);
273 else
274 info->pt = info->sp - 16;
275 }
276 UNW_DPRINT(3, "unwind.%s: sp 0x%lx pt 0x%lx\n", __FUNCTION__, info->sp, info->pt);
277 return (struct pt_regs *) info->pt;
278 }
279
280 /* Unwind accessors. */
281
282 int
283 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
284 {
285 unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
286 struct unw_ireg *ireg;
287 struct pt_regs *pt;
288
289 if ((unsigned) regnum - 1 >= 127) {
290 if (regnum == 0 && !write) {
291 *val = 0; /* read r0 always returns 0 */
292 *nat = 0;
293 return 0;
294 }
295 UNW_DPRINT(0, "unwind.%s: trying to access non-existent r%u\n",
296 __FUNCTION__, regnum);
297 return -1;
298 }
299
300 if (regnum < 32) {
301 if (regnum >= 4 && regnum <= 7) {
302 /* access a preserved register */
303 ireg = &info->r4 + (regnum - 4);
304 addr = ireg->loc;
305 if (addr) {
306 nat_addr = addr + ireg->nat.off;
307 switch (ireg->nat.type) {
308 case UNW_NAT_VAL:
309 /* simulate getf.sig/setf.sig */
310 if (write) {
311 if (*nat) {
312 /* write NaTVal and be done with it */
313 addr[0] = 0;
314 addr[1] = 0x1fffe;
315 return 0;
316 }
317 addr[1] = 0x1003e;
318 } else {
319 if (addr[0] == 0 && addr[1] == 0x1ffe) {
320 /* return NaT and be done with it */
321 *val = 0;
322 *nat = 1;
323 return 0;
324 }
325 }
326 /* fall through */
327 case UNW_NAT_NONE:
328 dummy_nat = 0;
329 nat_addr = &dummy_nat;
330 break;
331
332 case UNW_NAT_MEMSTK:
333 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
334 break;
335
336 case UNW_NAT_REGSTK:
337 nat_addr = ia64_rse_rnat_addr(addr);
338 if ((unsigned long) addr < info->regstk.limit
339 || (unsigned long) addr >= info->regstk.top)
340 {
341 UNW_DPRINT(0, "unwind.%s: %p outside of regstk "
342 "[0x%lx-0x%lx)\n",
343 __FUNCTION__, (void *) addr,
344 info->regstk.limit,
345 info->regstk.top);
346 return -1;
347 }
348 if ((unsigned long) nat_addr >= info->regstk.top)
349 nat_addr = &info->sw->ar_rnat;
350 nat_mask = (1UL << ia64_rse_slot_num(addr));
351 break;
352 }
353 } else {
354 addr = &info->sw->r4 + (regnum - 4);
355 nat_addr = &info->sw->ar_unat;
356 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
357 }
358 } else {
359 /* access a scratch register */
360 pt = get_scratch_regs(info);
361 addr = (unsigned long *) ((unsigned long)pt + pt_regs_off(regnum));
362 if (info->pri_unat_loc)
363 nat_addr = info->pri_unat_loc;
364 else
365 nat_addr = &info->sw->ar_unat;
366 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
367 }
368 } else {
369 /* access a stacked register */
370 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
371 nat_addr = ia64_rse_rnat_addr(addr);
372 if ((unsigned long) addr < info->regstk.limit
373 || (unsigned long) addr >= info->regstk.top)
374 {
375 UNW_DPRINT(0, "unwind.%s: ignoring attempt to access register outside "
376 "of rbs\n", __FUNCTION__);
377 return -1;
378 }
379 if ((unsigned long) nat_addr >= info->regstk.top)
380 nat_addr = &info->sw->ar_rnat;
381 nat_mask = (1UL << ia64_rse_slot_num(addr));
382 }
383
384 if (write) {
385 if (read_only(addr)) {
386 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
387 __FUNCTION__);
388 } else {
389 *addr = *val;
390 if (*nat)
391 *nat_addr |= nat_mask;
392 else
393 *nat_addr &= ~nat_mask;
394 }
395 } else {
396 if ((*nat_addr & nat_mask) == 0) {
397 *val = *addr;
398 *nat = 0;
399 } else {
400 *val = 0; /* if register is a NaT, *addr may contain kernel data! */
401 *nat = 1;
402 }
403 }
404 return 0;
405 }
406 EXPORT_SYMBOL(unw_access_gr);
407
408 int
409 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
410 {
411 unsigned long *addr;
412 struct pt_regs *pt;
413
414 switch (regnum) {
415 /* scratch: */
416 case 0: pt = get_scratch_regs(info); addr = &pt->b0; break;
417 case 6: pt = get_scratch_regs(info); addr = &pt->b6; break;
418 case 7: pt = get_scratch_regs(info); addr = &pt->b7; break;
419
420 /* preserved: */
421 case 1: case 2: case 3: case 4: case 5:
422 addr = *(&info->b1_loc + (regnum - 1));
423 if (!addr)
424 addr = &info->sw->b1 + (regnum - 1);
425 break;
426
427 default:
428 UNW_DPRINT(0, "unwind.%s: trying to access non-existent b%u\n",
429 __FUNCTION__, regnum);
430 return -1;
431 }
432 if (write)
433 if (read_only(addr)) {
434 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
435 __FUNCTION__);
436 } else
437 *addr = *val;
438 else
439 *val = *addr;
440 return 0;
441 }
442 EXPORT_SYMBOL(unw_access_br);
443
444 int
445 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
446 {
447 struct ia64_fpreg *addr = NULL;
448 struct pt_regs *pt;
449
450 if ((unsigned) (regnum - 2) >= 126) {
451 UNW_DPRINT(0, "unwind.%s: trying to access non-existent f%u\n",
452 __FUNCTION__, regnum);
453 return -1;
454 }
455
456 if (regnum <= 5) {
457 addr = *(&info->f2_loc + (regnum - 2));
458 if (!addr)
459 addr = &info->sw->f2 + (regnum - 2);
460 } else if (regnum <= 15) {
461 if (regnum <= 11) {
462 pt = get_scratch_regs(info);
463 addr = &pt->f6 + (regnum - 6);
464 }
465 else
466 addr = &info->sw->f12 + (regnum - 12);
467 } else if (regnum <= 31) {
468 addr = info->fr_loc[regnum - 16];
469 if (!addr)
470 addr = &info->sw->f16 + (regnum - 16);
471 } else {
472 struct task_struct *t = info->task;
473
474 if (write)
475 ia64_sync_fph(t);
476 else
477 ia64_flush_fph(t);
478 addr = t->thread.fph + (regnum - 32);
479 }
480
481 if (write)
482 if (read_only(addr)) {
483 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
484 __FUNCTION__);
485 } else
486 *addr = *val;
487 else
488 *val = *addr;
489 return 0;
490 }
491 EXPORT_SYMBOL(unw_access_fr);
492
493 int
494 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
495 {
496 unsigned long *addr;
497 struct pt_regs *pt;
498
499 switch (regnum) {
500 case UNW_AR_BSP:
501 addr = info->bsp_loc;
502 if (!addr)
503 addr = &info->sw->ar_bspstore;
504 break;
505
506 case UNW_AR_BSPSTORE:
507 addr = info->bspstore_loc;
508 if (!addr)
509 addr = &info->sw->ar_bspstore;
510 break;
511
512 case UNW_AR_PFS:
513 addr = info->pfs_loc;
514 if (!addr)
515 addr = &info->sw->ar_pfs;
516 break;
517
518 case UNW_AR_RNAT:
519 addr = info->rnat_loc;
520 if (!addr)
521 addr = &info->sw->ar_rnat;
522 break;
523
524 case UNW_AR_UNAT:
525 addr = info->unat_loc;
526 if (!addr)
527 addr = &info->sw->ar_unat;
528 break;
529
530 case UNW_AR_LC:
531 addr = info->lc_loc;
532 if (!addr)
533 addr = &info->sw->ar_lc;
534 break;
535
536 case UNW_AR_EC:
537 if (!info->cfm_loc)
538 return -1;
539 if (write)
540 *info->cfm_loc =
541 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
542 else
543 *val = (*info->cfm_loc >> 52) & 0x3f;
544 return 0;
545
546 case UNW_AR_FPSR:
547 addr = info->fpsr_loc;
548 if (!addr)
549 addr = &info->sw->ar_fpsr;
550 break;
551
552 case UNW_AR_RSC:
553 pt = get_scratch_regs(info);
554 addr = &pt->ar_rsc;
555 break;
556
557 case UNW_AR_CCV:
558 pt = get_scratch_regs(info);
559 addr = &pt->ar_ccv;
560 break;
561
562 case UNW_AR_CSD:
563 pt = get_scratch_regs(info);
564 addr = &pt->ar_csd;
565 break;
566
567 case UNW_AR_SSD:
568 pt = get_scratch_regs(info);
569 addr = &pt->ar_ssd;
570 break;
571
572 default:
573 UNW_DPRINT(0, "unwind.%s: trying to access non-existent ar%u\n",
574 __FUNCTION__, regnum);
575 return -1;
576 }
577
578 if (write) {
579 if (read_only(addr)) {
580 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
581 __FUNCTION__);
582 } else
583 *addr = *val;
584 } else
585 *val = *addr;
586 return 0;
587 }
588 EXPORT_SYMBOL(unw_access_ar);
589
590 int
591 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
592 {
593 unsigned long *addr;
594
595 addr = info->pr_loc;
596 if (!addr)
597 addr = &info->sw->pr;
598
599 if (write) {
600 if (read_only(addr)) {
601 UNW_DPRINT(0, "unwind.%s: ignoring attempt to write read-only location\n",
602 __FUNCTION__);
603 } else
604 *addr = *val;
605 } else
606 *val = *addr;
607 return 0;
608 }
609 EXPORT_SYMBOL(unw_access_pr);
610
611 \f
612 /* Routines to manipulate the state stack. */
613
614 static inline void
615 push (struct unw_state_record *sr)
616 {
617 struct unw_reg_state *rs;
618
619 rs = alloc_reg_state();
620 if (!rs) {
621 printk(KERN_ERR "unwind: cannot stack reg state!\n");
622 return;
623 }
624 memcpy(rs, &sr->curr, sizeof(*rs));
625 sr->curr.next = rs;
626 }
627
628 static void
629 pop (struct unw_state_record *sr)
630 {
631 struct unw_reg_state *rs = sr->curr.next;
632
633 if (!rs) {
634 printk(KERN_ERR "unwind: stack underflow!\n");
635 return;
636 }
637 memcpy(&sr->curr, rs, sizeof(*rs));
638 free_reg_state(rs);
639 }
640
641 /* Make a copy of the state stack. Non-recursive to avoid stack overflows. */
642 static struct unw_reg_state *
643 dup_state_stack (struct unw_reg_state *rs)
644 {
645 struct unw_reg_state *copy, *prev = NULL, *first = NULL;
646
647 while (rs) {
648 copy = alloc_reg_state();
649 if (!copy) {
650 printk(KERN_ERR "unwind.dup_state_stack: out of memory\n");
651 return NULL;
652 }
653 memcpy(copy, rs, sizeof(*copy));
654 if (first)
655 prev->next = copy;
656 else
657 first = copy;
658 rs = rs->next;
659 prev = copy;
660 }
661 return first;
662 }
663
664 /* Free all stacked register states (but not RS itself). */
665 static void
666 free_state_stack (struct unw_reg_state *rs)
667 {
668 struct unw_reg_state *p, *next;
669
670 for (p = rs->next; p != NULL; p = next) {
671 next = p->next;
672 free_reg_state(p);
673 }
674 rs->next = NULL;
675 }
676 \f
677 /* Unwind decoder routines */
678
679 static enum unw_register_index __attribute_const__
680 decode_abreg (unsigned char abreg, int memory)
681 {
682 switch (abreg) {
683 case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
684 case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
685 case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
686 case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
687 case 0x60: return UNW_REG_PR;
688 case 0x61: return UNW_REG_PSP;
689 case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
690 case 0x63: return UNW_REG_RP;
691 case 0x64: return UNW_REG_BSP;
692 case 0x65: return UNW_REG_BSPSTORE;
693 case 0x66: return UNW_REG_RNAT;
694 case 0x67: return UNW_REG_UNAT;
695 case 0x68: return UNW_REG_FPSR;
696 case 0x69: return UNW_REG_PFS;
697 case 0x6a: return UNW_REG_LC;
698 default:
699 break;
700 }
701 UNW_DPRINT(0, "unwind.%s: bad abreg=0x%x\n", __FUNCTION__, abreg);
702 return UNW_REG_LC;
703 }
704
705 static void
706 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
707 {
708 reg->val = val;
709 reg->where = where;
710 if (reg->when == UNW_WHEN_NEVER)
711 reg->when = when;
712 }
713
714 static void
715 alloc_spill_area (unsigned long *offp, unsigned long regsize,
716 struct unw_reg_info *lo, struct unw_reg_info *hi)
717 {
718 struct unw_reg_info *reg;
719
720 for (reg = hi; reg >= lo; --reg) {
721 if (reg->where == UNW_WHERE_SPILL_HOME) {
722 reg->where = UNW_WHERE_PSPREL;
723 *offp -= regsize;
724 reg->val = *offp;
725 }
726 }
727 }
728
729 static inline void
730 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
731 {
732 struct unw_reg_info *reg;
733
734 for (reg = *regp; reg <= lim; ++reg) {
735 if (reg->where == UNW_WHERE_SPILL_HOME) {
736 reg->when = t;
737 *regp = reg + 1;
738 return;
739 }
740 }
741 UNW_DPRINT(0, "unwind.%s: excess spill!\n", __FUNCTION__);
742 }
743
744 static inline void
745 finish_prologue (struct unw_state_record *sr)
746 {
747 struct unw_reg_info *reg;
748 unsigned long off;
749 int i;
750
751 /*
752 * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
753 * for Using Unwind Descriptors", rule 3):
754 */
755 for (i = 0; i < (int) ARRAY_SIZE(unw.save_order); ++i) {
756 reg = sr->curr.reg + unw.save_order[i];
757 if (reg->where == UNW_WHERE_GR_SAVE) {
758 reg->where = UNW_WHERE_GR;
759 reg->val = sr->gr_save_loc++;
760 }
761 }
762
763 /*
764 * Next, compute when the fp, general, and branch registers get
765 * saved. This must come before alloc_spill_area() because
766 * we need to know which registers are spilled to their home
767 * locations.
768 */
769 if (sr->imask) {
770 unsigned char kind, mask = 0, *cp = sr->imask;
771 int t;
772 static const unsigned char limit[3] = {
773 UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
774 };
775 struct unw_reg_info *(regs[3]);
776
777 regs[0] = sr->curr.reg + UNW_REG_F2;
778 regs[1] = sr->curr.reg + UNW_REG_R4;
779 regs[2] = sr->curr.reg + UNW_REG_B1;
780
781 for (t = 0; t < sr->region_len; ++t) {
782 if ((t & 3) == 0)
783 mask = *cp++;
784 kind = (mask >> 2*(3-(t & 3))) & 3;
785 if (kind > 0)
786 spill_next_when(&regs[kind - 1], sr->curr.reg + limit[kind - 1],
787 sr->region_start + t);
788 }
789 }
790 /*
791 * Next, lay out the memory stack spill area:
792 */
793 if (sr->any_spills) {
794 off = sr->spill_offset;
795 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
796 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
797 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
798 }
799 }
800
801 /*
802 * Region header descriptors.
803 */
804
805 static void
806 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
807 struct unw_state_record *sr)
808 {
809 int i, region_start;
810
811 if (!(sr->in_body || sr->first_region))
812 finish_prologue(sr);
813 sr->first_region = 0;
814
815 /* check if we're done: */
816 if (sr->when_target < sr->region_start + sr->region_len) {
817 sr->done = 1;
818 return;
819 }
820
821 region_start = sr->region_start + sr->region_len;
822
823 for (i = 0; i < sr->epilogue_count; ++i)
824 pop(sr);
825 sr->epilogue_count = 0;
826 sr->epilogue_start = UNW_WHEN_NEVER;
827
828 sr->region_start = region_start;
829 sr->region_len = rlen;
830 sr->in_body = body;
831
832 if (!body) {
833 push(sr);
834
835 for (i = 0; i < 4; ++i) {
836 if (mask & 0x8)
837 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
838 sr->region_start + sr->region_len - 1, grsave++);
839 mask <<= 1;
840 }
841 sr->gr_save_loc = grsave;
842 sr->any_spills = 0;
843 sr->imask = NULL;
844 sr->spill_offset = 0x10; /* default to psp+16 */
845 }
846 }
847
848 /*
849 * Prologue descriptors.
850 */
851
852 static inline void
853 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
854 {
855 if (abi == 3 && context == 'i') {
856 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
857 UNW_DPRINT(3, "unwind.%s: interrupt frame\n", __FUNCTION__);
858 }
859 else
860 UNW_DPRINT(0, "unwind%s: ignoring unwabi(abi=0x%x,context=0x%x)\n",
861 __FUNCTION__, abi, context);
862 }
863
864 static inline void
865 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
866 {
867 int i;
868
869 for (i = 0; i < 5; ++i) {
870 if (brmask & 1)
871 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
872 sr->region_start + sr->region_len - 1, gr++);
873 brmask >>= 1;
874 }
875 }
876
877 static inline void
878 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
879 {
880 int i;
881
882 for (i = 0; i < 5; ++i) {
883 if (brmask & 1) {
884 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
885 sr->region_start + sr->region_len - 1, 0);
886 sr->any_spills = 1;
887 }
888 brmask >>= 1;
889 }
890 }
891
892 static inline void
893 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
894 {
895 int i;
896
897 for (i = 0; i < 4; ++i) {
898 if ((grmask & 1) != 0) {
899 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
900 sr->region_start + sr->region_len - 1, 0);
901 sr->any_spills = 1;
902 }
903 grmask >>= 1;
904 }
905 for (i = 0; i < 20; ++i) {
906 if ((frmask & 1) != 0) {
907 int base = (i < 4) ? UNW_REG_F2 : UNW_REG_F16 - 4;
908 set_reg(sr->curr.reg + base + i, UNW_WHERE_SPILL_HOME,
909 sr->region_start + sr->region_len - 1, 0);
910 sr->any_spills = 1;
911 }
912 frmask >>= 1;
913 }
914 }
915
916 static inline void
917 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
918 {
919 int i;
920
921 for (i = 0; i < 4; ++i) {
922 if ((frmask & 1) != 0) {
923 set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
924 sr->region_start + sr->region_len - 1, 0);
925 sr->any_spills = 1;
926 }
927 frmask >>= 1;
928 }
929 }
930
931 static inline void
932 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
933 {
934 int i;
935
936 for (i = 0; i < 4; ++i) {
937 if ((grmask & 1) != 0)
938 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
939 sr->region_start + sr->region_len - 1, gr++);
940 grmask >>= 1;
941 }
942 }
943
944 static inline void
945 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
946 {
947 int i;
948
949 for (i = 0; i < 4; ++i) {
950 if ((grmask & 1) != 0) {
951 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
952 sr->region_start + sr->region_len - 1, 0);
953 sr->any_spills = 1;
954 }
955 grmask >>= 1;
956 }
957 }
958
959 static inline void
960 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
961 {
962 set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
963 sr->region_start + min_t(int, t, sr->region_len - 1), 16*size);
964 }
965
966 static inline void
967 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
968 {
969 sr->curr.reg[UNW_REG_PSP].when = sr->region_start + min_t(int, t, sr->region_len - 1);
970 }
971
972 static inline void
973 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
974 {
975 set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
976 }
977
978 static inline void
979 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
980 {
981 set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
982 0x10 - 4*pspoff);
983 }
984
985 static inline void
986 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
987 {
988 set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
989 4*spoff);
990 }
991
992 static inline void
993 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
994 {
995 sr->return_link_reg = dst;
996 }
997
998 static inline void
999 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
1000 {
1001 struct unw_reg_info *reg = sr->curr.reg + regnum;
1002
1003 if (reg->where == UNW_WHERE_NONE)
1004 reg->where = UNW_WHERE_GR_SAVE;
1005 reg->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1006 }
1007
1008 static inline void
1009 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
1010 {
1011 sr->spill_offset = 0x10 - 4*pspoff;
1012 }
1013
1014 static inline unsigned char *
1015 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
1016 {
1017 sr->imask = imaskp;
1018 return imaskp + (2*sr->region_len + 7)/8;
1019 }
1020
1021 /*
1022 * Body descriptors.
1023 */
1024 static inline void
1025 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
1026 {
1027 sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
1028 sr->epilogue_count = ecount + 1;
1029 }
1030
1031 static inline void
1032 desc_copy_state (unw_word label, struct unw_state_record *sr)
1033 {
1034 struct unw_labeled_state *ls;
1035
1036 for (ls = sr->labeled_states; ls; ls = ls->next) {
1037 if (ls->label == label) {
1038 free_state_stack(&sr->curr);
1039 memcpy(&sr->curr, &ls->saved_state, sizeof(sr->curr));
1040 sr->curr.next = dup_state_stack(ls->saved_state.next);
1041 return;
1042 }
1043 }
1044 printk(KERN_ERR "unwind: failed to find state labeled 0x%lx\n", label);
1045 }
1046
1047 static inline void
1048 desc_label_state (unw_word label, struct unw_state_record *sr)
1049 {
1050 struct unw_labeled_state *ls;
1051
1052 ls = alloc_labeled_state();
1053 if (!ls) {
1054 printk(KERN_ERR "unwind.desc_label_state(): out of memory\n");
1055 return;
1056 }
1057 ls->label = label;
1058 memcpy(&ls->saved_state, &sr->curr, sizeof(ls->saved_state));
1059 ls->saved_state.next = dup_state_stack(sr->curr.next);
1060
1061 /* insert into list of labeled states: */
1062 ls->next = sr->labeled_states;
1063 sr->labeled_states = ls;
1064 }
1065
1066 /*
1067 * General descriptors.
1068 */
1069
1070 static inline int
1071 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
1072 {
1073 if (sr->when_target <= sr->region_start + min_t(int, t, sr->region_len - 1))
1074 return 0;
1075 if (qp > 0) {
1076 if ((sr->pr_val & (1UL << qp)) == 0)
1077 return 0;
1078 sr->pr_mask |= (1UL << qp);
1079 }
1080 return 1;
1081 }
1082
1083 static inline void
1084 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
1085 {
1086 struct unw_reg_info *r;
1087
1088 if (!desc_is_active(qp, t, sr))
1089 return;
1090
1091 r = sr->curr.reg + decode_abreg(abreg, 0);
1092 r->where = UNW_WHERE_NONE;
1093 r->when = UNW_WHEN_NEVER;
1094 r->val = 0;
1095 }
1096
1097 static inline void
1098 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
1099 unsigned char ytreg, struct unw_state_record *sr)
1100 {
1101 enum unw_where where = UNW_WHERE_GR;
1102 struct unw_reg_info *r;
1103
1104 if (!desc_is_active(qp, t, sr))
1105 return;
1106
1107 if (x)
1108 where = UNW_WHERE_BR;
1109 else if (ytreg & 0x80)
1110 where = UNW_WHERE_FR;
1111
1112 r = sr->curr.reg + decode_abreg(abreg, 0);
1113 r->where = where;
1114 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1115 r->val = (ytreg & 0x7f);
1116 }
1117
1118 static inline void
1119 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
1120 struct unw_state_record *sr)
1121 {
1122 struct unw_reg_info *r;
1123
1124 if (!desc_is_active(qp, t, sr))
1125 return;
1126
1127 r = sr->curr.reg + decode_abreg(abreg, 1);
1128 r->where = UNW_WHERE_PSPREL;
1129 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1130 r->val = 0x10 - 4*pspoff;
1131 }
1132
1133 static inline void
1134 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1135 struct unw_state_record *sr)
1136 {
1137 struct unw_reg_info *r;
1138
1139 if (!desc_is_active(qp, t, sr))
1140 return;
1141
1142 r = sr->curr.reg + decode_abreg(abreg, 1);
1143 r->where = UNW_WHERE_SPREL;
1144 r->when = sr->region_start + min_t(int, t, sr->region_len - 1);
1145 r->val = 4*spoff;
1146 }
1147
1148 #define UNW_DEC_BAD_CODE(code) printk(KERN_ERR "unwind: unknown code 0x%02x\n", \
1149 code);
1150
1151 /*
1152 * region headers:
1153 */
1154 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg) desc_prologue(0,r,m,gr,arg)
1155 #define UNW_DEC_PROLOGUE(fmt,b,r,arg) desc_prologue(b,r,0,32,arg)
1156 /*
1157 * prologue descriptors:
1158 */
1159 #define UNW_DEC_ABI(fmt,a,c,arg) desc_abi(a,c,arg)
1160 #define UNW_DEC_BR_GR(fmt,b,g,arg) desc_br_gr(b,g,arg)
1161 #define UNW_DEC_BR_MEM(fmt,b,arg) desc_br_mem(b,arg)
1162 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg) desc_frgr_mem(g,f,arg)
1163 #define UNW_DEC_FR_MEM(fmt,f,arg) desc_fr_mem(f,arg)
1164 #define UNW_DEC_GR_GR(fmt,m,g,arg) desc_gr_gr(m,g,arg)
1165 #define UNW_DEC_GR_MEM(fmt,m,arg) desc_gr_mem(m,arg)
1166 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg) desc_mem_stack_f(t,s,arg)
1167 #define UNW_DEC_MEM_STACK_V(fmt,t,arg) desc_mem_stack_v(t,arg)
1168 #define UNW_DEC_REG_GR(fmt,r,d,arg) desc_reg_gr(r,d,arg)
1169 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg) desc_reg_psprel(r,o,arg)
1170 #define UNW_DEC_REG_SPREL(fmt,r,o,arg) desc_reg_sprel(r,o,arg)
1171 #define UNW_DEC_REG_WHEN(fmt,r,t,arg) desc_reg_when(r,t,arg)
1172 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1173 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1174 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg) desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1175 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg) desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1176 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg) desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1177 #define UNW_DEC_RP_BR(fmt,d,arg) desc_rp_br(d,arg)
1178 #define UNW_DEC_SPILL_BASE(fmt,o,arg) desc_spill_base(o,arg)
1179 #define UNW_DEC_SPILL_MASK(fmt,m,arg) (m = desc_spill_mask(m,arg))
1180 /*
1181 * body descriptors:
1182 */
1183 #define UNW_DEC_EPILOGUE(fmt,t,c,arg) desc_epilogue(t,c,arg)
1184 #define UNW_DEC_COPY_STATE(fmt,l,arg) desc_copy_state(l,arg)
1185 #define UNW_DEC_LABEL_STATE(fmt,l,arg) desc_label_state(l,arg)
1186 /*
1187 * general unwind descriptors:
1188 */
1189 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg) desc_spill_reg_p(p,t,a,x,y,arg)
1190 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg) desc_spill_reg_p(0,t,a,x,y,arg)
1191 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg) desc_spill_psprel_p(p,t,a,o,arg)
1192 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg) desc_spill_psprel_p(0,t,a,o,arg)
1193 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg) desc_spill_sprel_p(p,t,a,o,arg)
1194 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg) desc_spill_sprel_p(0,t,a,o,arg)
1195 #define UNW_DEC_RESTORE_P(f,p,t,a,arg) desc_restore_p(p,t,a,arg)
1196 #define UNW_DEC_RESTORE(f,t,a,arg) desc_restore_p(0,t,a,arg)
1197
1198 #include "unwind_decoder.c"
1199
1200 \f
1201 /* Unwind scripts. */
1202
1203 static inline unw_hash_index_t
1204 hash (unsigned long ip)
1205 {
1206 # define hashmagic 0x9e3779b97f4a7c16UL /* based on (sqrt(5)/2-1)*2^64 */
1207
1208 return (ip >> 4)*hashmagic >> (64 - UNW_LOG_HASH_SIZE);
1209 #undef hashmagic
1210 }
1211
1212 static inline long
1213 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1214 {
1215 read_lock(&script->lock);
1216 if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1217 /* keep the read lock... */
1218 return 1;
1219 read_unlock(&script->lock);
1220 return 0;
1221 }
1222
1223 static inline struct unw_script *
1224 script_lookup (struct unw_frame_info *info)
1225 {
1226 struct unw_script *script = unw.cache + info->hint;
1227 unsigned short index;
1228 unsigned long ip, pr;
1229
1230 if (UNW_DEBUG_ON(0))
1231 return NULL; /* Always regenerate scripts in debug mode */
1232
1233 STAT(++unw.stat.cache.lookups);
1234
1235 ip = info->ip;
1236 pr = info->pr;
1237
1238 if (cache_match(script, ip, pr)) {
1239 STAT(++unw.stat.cache.hinted_hits);
1240 return script;
1241 }
1242
1243 index = unw.hash[hash(ip)];
1244 if (index >= UNW_CACHE_SIZE)
1245 return NULL;
1246
1247 script = unw.cache + index;
1248 while (1) {
1249 if (cache_match(script, ip, pr)) {
1250 /* update hint; no locking required as single-word writes are atomic */
1251 STAT(++unw.stat.cache.normal_hits);
1252 unw.cache[info->prev_script].hint = script - unw.cache;
1253 return script;
1254 }
1255 if (script->coll_chain >= UNW_HASH_SIZE)
1256 return NULL;
1257 script = unw.cache + script->coll_chain;
1258 STAT(++unw.stat.cache.collision_chain_traversals);
1259 }
1260 }
1261
1262 /*
1263 * On returning, a write lock for the SCRIPT is still being held.
1264 */
1265 static inline struct unw_script *
1266 script_new (unsigned long ip)
1267 {
1268 struct unw_script *script, *prev, *tmp;
1269 unw_hash_index_t index;
1270 unsigned short head;
1271
1272 STAT(++unw.stat.script.news);
1273
1274 /*
1275 * Can't (easily) use cmpxchg() here because of ABA problem
1276 * that is intrinsic in cmpxchg()...
1277 */
1278 head = unw.lru_head;
1279 script = unw.cache + head;
1280 unw.lru_head = script->lru_chain;
1281
1282 /*
1283 * We'd deadlock here if we interrupted a thread that is holding a read lock on
1284 * script->lock. Thus, if the write_trylock() fails, we simply bail out. The
1285 * alternative would be to disable interrupts whenever we hold a read-lock, but
1286 * that seems silly.
1287 */
1288 if (!write_trylock(&script->lock))
1289 return NULL;
1290
1291 /* re-insert script at the tail of the LRU chain: */
1292 unw.cache[unw.lru_tail].lru_chain = head;
1293 unw.lru_tail = head;
1294
1295 /* remove the old script from the hash table (if it's there): */
1296 if (script->ip) {
1297 index = hash(script->ip);
1298 tmp = unw.cache + unw.hash[index];
1299 prev = NULL;
1300 while (1) {
1301 if (tmp == script) {
1302 if (prev)
1303 prev->coll_chain = tmp->coll_chain;
1304 else
1305 unw.hash[index] = tmp->coll_chain;
1306 break;
1307 } else
1308 prev = tmp;
1309 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1310 /* old script wasn't in the hash-table */
1311 break;
1312 tmp = unw.cache + tmp->coll_chain;
1313 }
1314 }
1315
1316 /* enter new script in the hash table */
1317 index = hash(ip);
1318 script->coll_chain = unw.hash[index];
1319 unw.hash[index] = script - unw.cache;
1320
1321 script->ip = ip; /* set new IP while we're holding the locks */
1322
1323 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1324
1325 script->flags = 0;
1326 script->hint = 0;
1327 script->count = 0;
1328 return script;
1329 }
1330
1331 static void
1332 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1333 {
1334 script->pr_mask = sr->pr_mask;
1335 script->pr_val = sr->pr_val;
1336 /*
1337 * We could down-grade our write-lock on script->lock here but
1338 * the rwlock API doesn't offer atomic lock downgrading, so
1339 * we'll just keep the write-lock and release it later when
1340 * we're done using the script.
1341 */
1342 }
1343
1344 static inline void
1345 script_emit (struct unw_script *script, struct unw_insn insn)
1346 {
1347 if (script->count >= UNW_MAX_SCRIPT_LEN) {
1348 UNW_DPRINT(0, "unwind.%s: script exceeds maximum size of %u instructions!\n",
1349 __FUNCTION__, UNW_MAX_SCRIPT_LEN);
1350 return;
1351 }
1352 script->insn[script->count++] = insn;
1353 }
1354
1355 static inline void
1356 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1357 {
1358 struct unw_reg_info *r = sr->curr.reg + i;
1359 enum unw_insn_opcode opc;
1360 struct unw_insn insn;
1361 unsigned long val = 0;
1362
1363 switch (r->where) {
1364 case UNW_WHERE_GR:
1365 if (r->val >= 32) {
1366 /* register got spilled to a stacked register */
1367 opc = UNW_INSN_SETNAT_TYPE;
1368 val = UNW_NAT_REGSTK;
1369 } else
1370 /* register got spilled to a scratch register */
1371 opc = UNW_INSN_SETNAT_MEMSTK;
1372 break;
1373
1374 case UNW_WHERE_FR:
1375 opc = UNW_INSN_SETNAT_TYPE;
1376 val = UNW_NAT_VAL;
1377 break;
1378
1379 case UNW_WHERE_BR:
1380 opc = UNW_INSN_SETNAT_TYPE;
1381 val = UNW_NAT_NONE;
1382 break;
1383
1384 case UNW_WHERE_PSPREL:
1385 case UNW_WHERE_SPREL:
1386 opc = UNW_INSN_SETNAT_MEMSTK;
1387 break;
1388
1389 default:
1390 UNW_DPRINT(0, "unwind.%s: don't know how to emit nat info for where = %u\n",
1391 __FUNCTION__, r->where);
1392 return;
1393 }
1394 insn.opc = opc;
1395 insn.dst = unw.preg_index[i];
1396 insn.val = val;
1397 script_emit(script, insn);
1398 }
1399
1400 static void
1401 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1402 {
1403 struct unw_reg_info *r = sr->curr.reg + i;
1404 enum unw_insn_opcode opc;
1405 unsigned long val, rval;
1406 struct unw_insn insn;
1407 long need_nat_info;
1408
1409 if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1410 return;
1411
1412 opc = UNW_INSN_MOVE;
1413 val = rval = r->val;
1414 need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1415
1416 switch (r->where) {
1417 case UNW_WHERE_GR:
1418 if (rval >= 32) {
1419 opc = UNW_INSN_MOVE_STACKED;
1420 val = rval - 32;
1421 } else if (rval >= 4 && rval <= 7) {
1422 if (need_nat_info) {
1423 opc = UNW_INSN_MOVE2;
1424 need_nat_info = 0;
1425 }
1426 val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1427 } else if (rval == 0) {
1428 opc = UNW_INSN_MOVE_CONST;
1429 val = 0;
1430 } else {
1431 /* register got spilled to a scratch register */
1432 opc = UNW_INSN_MOVE_SCRATCH;
1433 val = pt_regs_off(rval);
1434 }
1435 break;
1436
1437 case UNW_WHERE_FR:
1438 if (rval <= 5)
1439 val = unw.preg_index[UNW_REG_F2 + (rval - 2)];
1440 else if (rval >= 16 && rval <= 31)
1441 val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1442 else {
1443 opc = UNW_INSN_MOVE_SCRATCH;
1444 if (rval <= 11)
1445 val = offsetof(struct pt_regs, f6) + 16*(rval - 6);
1446 else
1447 UNW_DPRINT(0, "unwind.%s: kernel may not touch f%lu\n",
1448 __FUNCTION__, rval);
1449 }
1450 break;
1451
1452 case UNW_WHERE_BR:
1453 if (rval >= 1 && rval <= 5)
1454 val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1455 else {
1456 opc = UNW_INSN_MOVE_SCRATCH;
1457 if (rval == 0)
1458 val = offsetof(struct pt_regs, b0);
1459 else if (rval == 6)
1460 val = offsetof(struct pt_regs, b6);
1461 else
1462 val = offsetof(struct pt_regs, b7);
1463 }
1464 break;
1465
1466 case UNW_WHERE_SPREL:
1467 opc = UNW_INSN_ADD_SP;
1468 break;
1469
1470 case UNW_WHERE_PSPREL:
1471 opc = UNW_INSN_ADD_PSP;
1472 break;
1473
1474 default:
1475 UNW_DPRINT(0, "unwind%s: register %u has unexpected `where' value of %u\n",
1476 __FUNCTION__, i, r->where);
1477 break;
1478 }
1479 insn.opc = opc;
1480 insn.dst = unw.preg_index[i];
1481 insn.val = val;
1482 script_emit(script, insn);
1483 if (need_nat_info)
1484 emit_nat_info(sr, i, script);
1485
1486 if (i == UNW_REG_PSP) {
1487 /*
1488 * info->psp must contain the _value_ of the previous
1489 * sp, not it's save location. We get this by
1490 * dereferencing the value we just stored in
1491 * info->psp:
1492 */
1493 insn.opc = UNW_INSN_LOAD;
1494 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1495 script_emit(script, insn);
1496 }
1497 }
1498
1499 static inline const struct unw_table_entry *
1500 lookup (struct unw_table *table, unsigned long rel_ip)
1501 {
1502 const struct unw_table_entry *e = NULL;
1503 unsigned long lo, hi, mid;
1504
1505 /* do a binary search for right entry: */
1506 for (lo = 0, hi = table->length; lo < hi; ) {
1507 mid = (lo + hi) / 2;
1508 e = &table->array[mid];
1509 if (rel_ip < e->start_offset)
1510 hi = mid;
1511 else if (rel_ip >= e->end_offset)
1512 lo = mid + 1;
1513 else
1514 break;
1515 }
1516 if (rel_ip < e->start_offset || rel_ip >= e->end_offset)
1517 return NULL;
1518 return e;
1519 }
1520
1521 /*
1522 * Build an unwind script that unwinds from state OLD_STATE to the
1523 * entrypoint of the function that called OLD_STATE.
1524 */
1525 static inline struct unw_script *
1526 build_script (struct unw_frame_info *info)
1527 {
1528 const struct unw_table_entry *e = NULL;
1529 struct unw_script *script = NULL;
1530 struct unw_labeled_state *ls, *next;
1531 unsigned long ip = info->ip;
1532 struct unw_state_record sr;
1533 struct unw_table *table;
1534 struct unw_reg_info *r;
1535 struct unw_insn insn;
1536 u8 *dp, *desc_end;
1537 u64 hdr;
1538 int i;
1539 STAT(unsigned long start, parse_start;)
1540
1541 STAT(++unw.stat.script.builds; start = ia64_get_itc());
1542
1543 /* build state record */
1544 memset(&sr, 0, sizeof(sr));
1545 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1546 r->when = UNW_WHEN_NEVER;
1547 sr.pr_val = info->pr;
1548
1549 UNW_DPRINT(3, "unwind.%s: ip 0x%lx\n", __FUNCTION__, ip);
1550 script = script_new(ip);
1551 if (!script) {
1552 UNW_DPRINT(0, "unwind.%s: failed to create unwind script\n", __FUNCTION__);
1553 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1554 return NULL;
1555 }
1556 unw.cache[info->prev_script].hint = script - unw.cache;
1557
1558 /* search the kernels and the modules' unwind tables for IP: */
1559
1560 STAT(parse_start = ia64_get_itc());
1561
1562 for (table = unw.tables; table; table = table->next) {
1563 if (ip >= table->start && ip < table->end) {
1564 e = lookup(table, ip - table->segment_base);
1565 break;
1566 }
1567 }
1568 if (!e) {
1569 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs) */
1570 UNW_DPRINT(1, "unwind.%s: no unwind info for ip=0x%lx (prev ip=0x%lx)\n",
1571 __FUNCTION__, ip, unw.cache[info->prev_script].ip);
1572 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1573 sr.curr.reg[UNW_REG_RP].when = -1;
1574 sr.curr.reg[UNW_REG_RP].val = 0;
1575 compile_reg(&sr, UNW_REG_RP, script);
1576 script_finalize(script, &sr);
1577 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1578 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1579 return script;
1580 }
1581
1582 sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1583 + (ip & 0xfUL));
1584 hdr = *(u64 *) (table->segment_base + e->info_offset);
1585 dp = (u8 *) (table->segment_base + e->info_offset + 8);
1586 desc_end = dp + 8*UNW_LENGTH(hdr);
1587
1588 while (!sr.done && dp < desc_end)
1589 dp = unw_decode(dp, sr.in_body, &sr);
1590
1591 if (sr.when_target > sr.epilogue_start) {
1592 /*
1593 * sp has been restored and all values on the memory stack below
1594 * psp also have been restored.
1595 */
1596 sr.curr.reg[UNW_REG_PSP].val = 0;
1597 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1598 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1599 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1600 if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1601 || r->where == UNW_WHERE_SPREL)
1602 {
1603 r->val = 0;
1604 r->where = UNW_WHERE_NONE;
1605 r->when = UNW_WHEN_NEVER;
1606 }
1607 }
1608
1609 script->flags = sr.flags;
1610
1611 /*
1612 * If RP did't get saved, generate entry for the return link
1613 * register.
1614 */
1615 if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1616 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1617 sr.curr.reg[UNW_REG_RP].when = -1;
1618 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1619 UNW_DPRINT(1, "unwind.%s: using default for rp at ip=0x%lx where=%d val=0x%lx\n",
1620 __FUNCTION__, ip, sr.curr.reg[UNW_REG_RP].where,
1621 sr.curr.reg[UNW_REG_RP].val);
1622 }
1623
1624 #ifdef UNW_DEBUG
1625 UNW_DPRINT(1, "unwind.%s: state record for func 0x%lx, t=%u:\n",
1626 __FUNCTION__, table->segment_base + e->start_offset, sr.when_target);
1627 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1628 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1629 UNW_DPRINT(1, " %s <- ", unw.preg_name[r - sr.curr.reg]);
1630 switch (r->where) {
1631 case UNW_WHERE_GR: UNW_DPRINT(1, "r%lu", r->val); break;
1632 case UNW_WHERE_FR: UNW_DPRINT(1, "f%lu", r->val); break;
1633 case UNW_WHERE_BR: UNW_DPRINT(1, "b%lu", r->val); break;
1634 case UNW_WHERE_SPREL: UNW_DPRINT(1, "[sp+0x%lx]", r->val); break;
1635 case UNW_WHERE_PSPREL: UNW_DPRINT(1, "[psp+0x%lx]", r->val); break;
1636 case UNW_WHERE_NONE:
1637 UNW_DPRINT(1, "%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1638 break;
1639
1640 default:
1641 UNW_DPRINT(1, "BADWHERE(%d)", r->where);
1642 break;
1643 }
1644 UNW_DPRINT(1, "\t\t%d\n", r->when);
1645 }
1646 }
1647 #endif
1648
1649 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1650
1651 /* translate state record into unwinder instructions: */
1652
1653 /*
1654 * First, set psp if we're dealing with a fixed-size frame;
1655 * subsequent instructions may depend on this value.
1656 */
1657 if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1658 && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1659 && sr.curr.reg[UNW_REG_PSP].val != 0) {
1660 /* new psp is sp plus frame size */
1661 insn.opc = UNW_INSN_ADD;
1662 insn.dst = offsetof(struct unw_frame_info, psp)/8;
1663 insn.val = sr.curr.reg[UNW_REG_PSP].val; /* frame size */
1664 script_emit(script, insn);
1665 }
1666
1667 /* determine where the primary UNaT is: */
1668 if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1669 i = UNW_REG_PRI_UNAT_MEM;
1670 else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1671 i = UNW_REG_PRI_UNAT_GR;
1672 else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1673 i = UNW_REG_PRI_UNAT_MEM;
1674 else
1675 i = UNW_REG_PRI_UNAT_GR;
1676
1677 compile_reg(&sr, i, script);
1678
1679 for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1680 compile_reg(&sr, i, script);
1681
1682 /* free labeled register states & stack: */
1683
1684 STAT(parse_start = ia64_get_itc());
1685 for (ls = sr.labeled_states; ls; ls = next) {
1686 next = ls->next;
1687 free_state_stack(&ls->saved_state);
1688 free_labeled_state(ls);
1689 }
1690 free_state_stack(&sr.curr);
1691 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1692
1693 script_finalize(script, &sr);
1694 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1695 return script;
1696 }
1697
1698 /*
1699 * Apply the unwinding actions represented by OPS and update SR to
1700 * reflect the state that existed upon entry to the function that this
1701 * unwinder represents.
1702 */
1703 static inline void
1704 run_script (struct unw_script *script, struct unw_frame_info *state)
1705 {
1706 struct unw_insn *ip, *limit, next_insn;
1707 unsigned long opc, dst, val, off;
1708 unsigned long *s = (unsigned long *) state;
1709 STAT(unsigned long start;)
1710
1711 STAT(++unw.stat.script.runs; start = ia64_get_itc());
1712 state->flags = script->flags;
1713 ip = script->insn;
1714 limit = script->insn + script->count;
1715 next_insn = *ip;
1716
1717 while (ip++ < limit) {
1718 opc = next_insn.opc;
1719 dst = next_insn.dst;
1720 val = next_insn.val;
1721 next_insn = *ip;
1722
1723 redo:
1724 switch (opc) {
1725 case UNW_INSN_ADD:
1726 s[dst] += val;
1727 break;
1728
1729 case UNW_INSN_MOVE2:
1730 if (!s[val])
1731 goto lazy_init;
1732 s[dst+1] = s[val+1];
1733 s[dst] = s[val];
1734 break;
1735
1736 case UNW_INSN_MOVE:
1737 if (!s[val])
1738 goto lazy_init;
1739 s[dst] = s[val];
1740 break;
1741
1742 case UNW_INSN_MOVE_SCRATCH:
1743 if (state->pt) {
1744 s[dst] = (unsigned long) get_scratch_regs(state) + val;
1745 } else {
1746 s[dst] = 0;
1747 UNW_DPRINT(0, "unwind.%s: no state->pt, dst=%ld, val=%ld\n",
1748 __FUNCTION__, dst, val);
1749 }
1750 break;
1751
1752 case UNW_INSN_MOVE_CONST:
1753 if (val == 0)
1754 s[dst] = (unsigned long) &unw.r0;
1755 else {
1756 s[dst] = 0;
1757 UNW_DPRINT(0, "unwind.%s: UNW_INSN_MOVE_CONST bad val=%ld\n",
1758 __FUNCTION__, val);
1759 }
1760 break;
1761
1762
1763 case UNW_INSN_MOVE_STACKED:
1764 s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1765 val);
1766 break;
1767
1768 case UNW_INSN_ADD_PSP:
1769 s[dst] = state->psp + val;
1770 break;
1771
1772 case UNW_INSN_ADD_SP:
1773 s[dst] = state->sp + val;
1774 break;
1775
1776 case UNW_INSN_SETNAT_MEMSTK:
1777 if (!state->pri_unat_loc)
1778 state->pri_unat_loc = &state->sw->ar_unat;
1779 /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1780 s[dst+1] = ((unsigned long) state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1781 break;
1782
1783 case UNW_INSN_SETNAT_TYPE:
1784 s[dst+1] = val;
1785 break;
1786
1787 case UNW_INSN_LOAD:
1788 #ifdef UNW_DEBUG
1789 if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1790 || s[val] < TASK_SIZE)
1791 {
1792 UNW_DPRINT(0, "unwind.%s: rejecting bad psp=0x%lx\n",
1793 __FUNCTION__, s[val]);
1794 break;
1795 }
1796 #endif
1797 s[dst] = *(unsigned long *) s[val];
1798 break;
1799 }
1800 }
1801 STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1802 return;
1803
1804 lazy_init:
1805 off = unw.sw_off[val];
1806 s[val] = (unsigned long) state->sw + off;
1807 if (off >= offsetof(struct switch_stack, r4) && off <= offsetof(struct switch_stack, r7))
1808 /*
1809 * We're initializing a general register: init NaT info, too. Note that
1810 * the offset is a multiple of 8 which gives us the 3 bits needed for
1811 * the type field.
1812 */
1813 s[val+1] = (offsetof(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1814 goto redo;
1815 }
1816
1817 static int
1818 find_save_locs (struct unw_frame_info *info)
1819 {
1820 int have_write_lock = 0;
1821 struct unw_script *scr;
1822 unsigned long flags = 0;
1823
1824 if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1825 /* don't let obviously bad addresses pollute the cache */
1826 /* FIXME: should really be level 0 but it occurs too often. KAO */
1827 UNW_DPRINT(1, "unwind.%s: rejecting bad ip=0x%lx\n", __FUNCTION__, info->ip);
1828 info->rp_loc = NULL;
1829 return -1;
1830 }
1831
1832 scr = script_lookup(info);
1833 if (!scr) {
1834 spin_lock_irqsave(&unw.lock, flags);
1835 scr = build_script(info);
1836 if (!scr) {
1837 spin_unlock_irqrestore(&unw.lock, flags);
1838 UNW_DPRINT(0,
1839 "unwind.%s: failed to locate/build unwind script for ip %lx\n",
1840 __FUNCTION__, info->ip);
1841 return -1;
1842 }
1843 have_write_lock = 1;
1844 }
1845 info->hint = scr->hint;
1846 info->prev_script = scr - unw.cache;
1847
1848 run_script(scr, info);
1849
1850 if (have_write_lock) {
1851 write_unlock(&scr->lock);
1852 spin_unlock_irqrestore(&unw.lock, flags);
1853 } else
1854 read_unlock(&scr->lock);
1855 return 0;
1856 }
1857
1858 int
1859 unw_unwind (struct unw_frame_info *info)
1860 {
1861 unsigned long prev_ip, prev_sp, prev_bsp;
1862 unsigned long ip, pr, num_regs;
1863 STAT(unsigned long start, flags;)
1864 int retval;
1865
1866 STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1867
1868 prev_ip = info->ip;
1869 prev_sp = info->sp;
1870 prev_bsp = info->bsp;
1871
1872 /* restore the ip */
1873 if (!info->rp_loc) {
1874 /* FIXME: should really be level 0 but it occurs too often. KAO */
1875 UNW_DPRINT(1, "unwind.%s: failed to locate return link (ip=0x%lx)!\n",
1876 __FUNCTION__, info->ip);
1877 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1878 return -1;
1879 }
1880 ip = info->ip = *info->rp_loc;
1881 if (ip < GATE_ADDR) {
1882 UNW_DPRINT(2, "unwind.%s: reached user-space (ip=0x%lx)\n", __FUNCTION__, ip);
1883 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1884 return -1;
1885 }
1886
1887 /* restore the cfm: */
1888 if (!info->pfs_loc) {
1889 UNW_DPRINT(0, "unwind.%s: failed to locate ar.pfs!\n", __FUNCTION__);
1890 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1891 return -1;
1892 }
1893 info->cfm_loc = info->pfs_loc;
1894
1895 /* restore the bsp: */
1896 pr = info->pr;
1897 num_regs = 0;
1898 if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1899 info->pt = info->sp + 16;
1900 if ((pr & (1UL << PRED_NON_SYSCALL)) != 0)
1901 num_regs = *info->cfm_loc & 0x7f; /* size of frame */
1902 info->pfs_loc =
1903 (unsigned long *) (info->pt + offsetof(struct pt_regs, ar_pfs));
1904 UNW_DPRINT(3, "unwind.%s: interrupt_frame pt 0x%lx\n", __FUNCTION__, info->pt);
1905 } else
1906 num_regs = (*info->cfm_loc >> 7) & 0x7f; /* size of locals */
1907 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1908 if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1909 UNW_DPRINT(0, "unwind.%s: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1910 __FUNCTION__, info->bsp, info->regstk.limit, info->regstk.top);
1911 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1912 return -1;
1913 }
1914
1915 /* restore the sp: */
1916 info->sp = info->psp;
1917 if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1918 UNW_DPRINT(0, "unwind.%s: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1919 __FUNCTION__, info->sp, info->memstk.top, info->memstk.limit);
1920 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1921 return -1;
1922 }
1923
1924 if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1925 UNW_DPRINT(0, "unwind.%s: ip, sp, bsp unchanged; stopping here (ip=0x%lx)\n",
1926 __FUNCTION__, ip);
1927 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1928 return -1;
1929 }
1930
1931 /* as we unwind, the saved ar.unat becomes the primary unat: */
1932 info->pri_unat_loc = info->unat_loc;
1933
1934 /* finally, restore the predicates: */
1935 unw_get_pr(info, &info->pr);
1936
1937 retval = find_save_locs(info);
1938 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1939 return retval;
1940 }
1941 EXPORT_SYMBOL(unw_unwind);
1942
1943 int
1944 unw_unwind_to_user (struct unw_frame_info *info)
1945 {
1946 unsigned long ip, sp, pr = 0;
1947
1948 while (unw_unwind(info) >= 0) {
1949 unw_get_sp(info, &sp);
1950 if ((long)((unsigned long)info->task + IA64_STK_OFFSET - sp)
1951 < IA64_PT_REGS_SIZE) {
1952 UNW_DPRINT(0, "unwind.%s: ran off the top of the kernel stack\n",
1953 __FUNCTION__);
1954 break;
1955 }
1956 if (unw_is_intr_frame(info) &&
1957 (pr & (1UL << PRED_USER_STACK)))
1958 return 0;
1959 if (unw_get_pr (info, &pr) < 0) {
1960 unw_get_rp(info, &ip);
1961 UNW_DPRINT(0, "unwind.%s: failed to read "
1962 "predicate register (ip=0x%lx)\n",
1963 __FUNCTION__, ip);
1964 return -1;
1965 }
1966 }
1967 unw_get_ip(info, &ip);
1968 UNW_DPRINT(0, "unwind.%s: failed to unwind to user-level (ip=0x%lx)\n",
1969 __FUNCTION__, ip);
1970 return -1;
1971 }
1972 EXPORT_SYMBOL(unw_unwind_to_user);
1973
1974 static void
1975 init_frame_info (struct unw_frame_info *info, struct task_struct *t,
1976 struct switch_stack *sw, unsigned long stktop)
1977 {
1978 unsigned long rbslimit, rbstop, stklimit;
1979 STAT(unsigned long start, flags;)
1980
1981 STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1982
1983 /*
1984 * Subtle stuff here: we _could_ unwind through the switch_stack frame but we
1985 * don't want to do that because it would be slow as each preserved register would
1986 * have to be processed. Instead, what we do here is zero out the frame info and
1987 * start the unwind process at the function that created the switch_stack frame.
1988 * When a preserved value in switch_stack needs to be accessed, run_script() will
1989 * initialize the appropriate pointer on demand.
1990 */
1991 memset(info, 0, sizeof(*info));
1992
1993 rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1994 rbstop = sw->ar_bspstore;
1995 if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1996 rbstop = rbslimit;
1997
1998 stklimit = (unsigned long) t + IA64_STK_OFFSET;
1999 if (stktop <= rbstop)
2000 stktop = rbstop;
2001
2002 info->regstk.limit = rbslimit;
2003 info->regstk.top = rbstop;
2004 info->memstk.limit = stklimit;
2005 info->memstk.top = stktop;
2006 info->task = t;
2007 info->sw = sw;
2008 info->sp = info->psp = stktop;
2009 info->pr = sw->pr;
2010 UNW_DPRINT(3, "unwind.%s:\n"
2011 " task 0x%lx\n"
2012 " rbs = [0x%lx-0x%lx)\n"
2013 " stk = [0x%lx-0x%lx)\n"
2014 " pr 0x%lx\n"
2015 " sw 0x%lx\n"
2016 " sp 0x%lx\n",
2017 __FUNCTION__, (unsigned long) t, rbslimit, rbstop, stktop, stklimit,
2018 info->pr, (unsigned long) info->sw, info->sp);
2019 STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
2020 }
2021
2022 void
2023 unw_init_from_interruption (struct unw_frame_info *info, struct task_struct *t,
2024 struct pt_regs *pt, struct switch_stack *sw)
2025 {
2026 unsigned long sof;
2027
2028 init_frame_info(info, t, sw, pt->r12);
2029 info->cfm_loc = &pt->cr_ifs;
2030 info->unat_loc = &pt->ar_unat;
2031 info->pfs_loc = &pt->ar_pfs;
2032 sof = *info->cfm_loc & 0x7f;
2033 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sof);
2034 info->ip = pt->cr_iip + ia64_psr(pt)->ri;
2035 info->pt = (unsigned long) pt;
2036 UNW_DPRINT(3, "unwind.%s:\n"
2037 " bsp 0x%lx\n"
2038 " sof 0x%lx\n"
2039 " ip 0x%lx\n",
2040 __FUNCTION__, info->bsp, sof, info->ip);
2041 find_save_locs(info);
2042 }
2043
2044 void
2045 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
2046 {
2047 unsigned long sol;
2048
2049 init_frame_info(info, t, sw, (unsigned long) (sw + 1) - 16);
2050 info->cfm_loc = &sw->ar_pfs;
2051 sol = (*info->cfm_loc >> 7) & 0x7f;
2052 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
2053 info->ip = sw->b0;
2054 UNW_DPRINT(3, "unwind.%s:\n"
2055 " bsp 0x%lx\n"
2056 " sol 0x%lx\n"
2057 " ip 0x%lx\n",
2058 __FUNCTION__, info->bsp, sol, info->ip);
2059 find_save_locs(info);
2060 }
2061
2062 EXPORT_SYMBOL(unw_init_frame_info);
2063
2064 void
2065 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
2066 {
2067 struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
2068
2069 UNW_DPRINT(1, "unwind.%s\n", __FUNCTION__);
2070 unw_init_frame_info(info, t, sw);
2071 }
2072 EXPORT_SYMBOL(unw_init_from_blocked_task);
2073
2074 static void
2075 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
2076 unsigned long gp, const void *table_start, const void *table_end)
2077 {
2078 const struct unw_table_entry *start = table_start, *end = table_end;
2079
2080 table->name = name;
2081 table->segment_base = segment_base;
2082 table->gp = gp;
2083 table->start = segment_base + start[0].start_offset;
2084 table->end = segment_base + end[-1].end_offset;
2085 table->array = start;
2086 table->length = end - start;
2087 }
2088
2089 void *
2090 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
2091 const void *table_start, const void *table_end)
2092 {
2093 const struct unw_table_entry *start = table_start, *end = table_end;
2094 struct unw_table *table;
2095 unsigned long flags;
2096
2097 if (end - start <= 0) {
2098 UNW_DPRINT(0, "unwind.%s: ignoring attempt to insert empty unwind table\n",
2099 __FUNCTION__);
2100 return NULL;
2101 }
2102
2103 table = kmalloc(sizeof(*table), GFP_USER);
2104 if (!table)
2105 return NULL;
2106
2107 init_unwind_table(table, name, segment_base, gp, table_start, table_end);
2108
2109 spin_lock_irqsave(&unw.lock, flags);
2110 {
2111 /* keep kernel unwind table at the front (it's searched most commonly): */
2112 table->next = unw.tables->next;
2113 unw.tables->next = table;
2114 }
2115 spin_unlock_irqrestore(&unw.lock, flags);
2116
2117 return table;
2118 }
2119
2120 void
2121 unw_remove_unwind_table (void *handle)
2122 {
2123 struct unw_table *table, *prev;
2124 struct unw_script *tmp;
2125 unsigned long flags;
2126 long index;
2127
2128 if (!handle) {
2129 UNW_DPRINT(0, "unwind.%s: ignoring attempt to remove non-existent unwind table\n",
2130 __FUNCTION__);
2131 return;
2132 }
2133
2134 table = handle;
2135 if (table == &unw.kernel_table) {
2136 UNW_DPRINT(0, "unwind.%s: sorry, freeing the kernel's unwind table is a "
2137 "no-can-do!\n", __FUNCTION__);
2138 return;
2139 }
2140
2141 spin_lock_irqsave(&unw.lock, flags);
2142 {
2143 /* first, delete the table: */
2144
2145 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
2146 if (prev->next == table)
2147 break;
2148 if (!prev) {
2149 UNW_DPRINT(0, "unwind.%s: failed to find unwind table %p\n",
2150 __FUNCTION__, (void *) table);
2151 spin_unlock_irqrestore(&unw.lock, flags);
2152 return;
2153 }
2154 prev->next = table->next;
2155 }
2156 spin_unlock_irqrestore(&unw.lock, flags);
2157
2158 /* next, remove hash table entries for this table */
2159
2160 for (index = 0; index <= UNW_HASH_SIZE; ++index) {
2161 tmp = unw.cache + unw.hash[index];
2162 if (unw.hash[index] >= UNW_CACHE_SIZE
2163 || tmp->ip < table->start || tmp->ip >= table->end)
2164 continue;
2165
2166 write_lock(&tmp->lock);
2167 {
2168 if (tmp->ip >= table->start && tmp->ip < table->end) {
2169 unw.hash[index] = tmp->coll_chain;
2170 tmp->ip = 0;
2171 }
2172 }
2173 write_unlock(&tmp->lock);
2174 }
2175
2176 kfree(table);
2177 }
2178
2179 static int __init
2180 create_gate_table (void)
2181 {
2182 const struct unw_table_entry *entry, *start, *end;
2183 unsigned long *lp, segbase = GATE_ADDR;
2184 size_t info_size, size;
2185 char *info;
2186 Elf64_Phdr *punw = NULL, *phdr = (Elf64_Phdr *) (GATE_ADDR + GATE_EHDR->e_phoff);
2187 int i;
2188
2189 for (i = 0; i < GATE_EHDR->e_phnum; ++i, ++phdr)
2190 if (phdr->p_type == PT_IA_64_UNWIND) {
2191 punw = phdr;
2192 break;
2193 }
2194
2195 if (!punw) {
2196 printk("%s: failed to find gate DSO's unwind table!\n", __FUNCTION__);
2197 return 0;
2198 }
2199
2200 start = (const struct unw_table_entry *) punw->p_vaddr;
2201 end = (struct unw_table_entry *) ((char *) start + punw->p_memsz);
2202 size = 0;
2203
2204 unw_add_unwind_table("linux-gate.so", segbase, 0, start, end);
2205
2206 for (entry = start; entry < end; ++entry)
2207 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2208 size += 8; /* reserve space for "end of table" marker */
2209
2210 unw.gate_table = kmalloc(size, GFP_KERNEL);
2211 if (!unw.gate_table) {
2212 unw.gate_table_size = 0;
2213 printk(KERN_ERR "%s: unable to create unwind data for gate page!\n", __FUNCTION__);
2214 return 0;
2215 }
2216 unw.gate_table_size = size;
2217
2218 lp = unw.gate_table;
2219 info = (char *) unw.gate_table + size;
2220
2221 for (entry = start; entry < end; ++entry, lp += 3) {
2222 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
2223 info -= info_size;
2224 memcpy(info, (char *) segbase + entry->info_offset, info_size);
2225
2226 lp[0] = segbase + entry->start_offset; /* start */
2227 lp[1] = segbase + entry->end_offset; /* end */
2228 lp[2] = info - (char *) unw.gate_table; /* info */
2229 }
2230 *lp = 0; /* end-of-table marker */
2231 return 0;
2232 }
2233
2234 __initcall(create_gate_table);
2235
2236 void __init
2237 unw_init (void)
2238 {
2239 extern char __gp[];
2240 extern void unw_hash_index_t_is_too_narrow (void);
2241 long i, off;
2242
2243 if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
2244 unw_hash_index_t_is_too_narrow();
2245
2246 unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2247 unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2248 unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2249 unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2250 unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2251 unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2252 unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2253 unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2254 for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2255 unw.sw_off[unw.preg_index[i]] = off;
2256 for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2257 unw.sw_off[unw.preg_index[i]] = off;
2258 for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2259 unw.sw_off[unw.preg_index[i]] = off;
2260 for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2261 unw.sw_off[unw.preg_index[i]] = off;
2262
2263 for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2264 if (i > 0)
2265 unw.cache[i].lru_chain = (i - 1);
2266 unw.cache[i].coll_chain = -1;
2267 rwlock_init(&unw.cache[i].lock);
2268 }
2269 unw.lru_head = UNW_CACHE_SIZE - 1;
2270 unw.lru_tail = 0;
2271
2272 init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) __gp,
2273 __start_unwind, __end_unwind);
2274 }
2275
2276 /*
2277 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2278 *
2279 * This system call has been deprecated. The new and improved way to get
2280 * at the kernel's unwind info is via the gate DSO. The address of the
2281 * ELF header for this DSO is passed to user-level via AT_SYSINFO_EHDR.
2282 *
2283 * DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED DEPRECATED
2284 *
2285 * This system call copies the unwind data into the buffer pointed to by BUF and returns
2286 * the size of the unwind data. If BUF_SIZE is smaller than the size of the unwind data
2287 * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2288 * unwind data.
2289 *
2290 * The first portion of the unwind data contains an unwind table and rest contains the
2291 * associated unwind info (in no particular order). The unwind table consists of a table
2292 * of entries of the form:
2293 *
2294 * u64 start; (64-bit address of start of function)
2295 * u64 end; (64-bit address of start of function)
2296 * u64 info; (BUF-relative offset to unwind info)
2297 *
2298 * The end of the unwind table is indicated by an entry with a START address of zero.
2299 *
2300 * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2301 * on the format of the unwind info.
2302 *
2303 * ERRORS
2304 * EFAULT BUF points outside your accessible address space.
2305 */
2306 asmlinkage long
2307 sys_getunwind (void __user *buf, size_t buf_size)
2308 {
2309 if (buf && buf_size >= unw.gate_table_size)
2310 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2311 return -EFAULT;
2312 return unw.gate_table_size;
2313 }
Verdex Pro support