?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xen: cleancache shim to Xen Transcendent Memory
[linux-2.6/next.git] / arch / ia64 / kernel / module.c
blob1481b0a28ca03d46c71de2ca775393554d81bbb0
1 /*
2 * IA-64-specific support for kernel module loader.
3 *
4 * Copyright (C) 2003 Hewlett-Packard Co
5 * David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * Loosely based on patch by Rusty Russell.
8 */
9
10 /* relocs tested so far:
11
12 DIR64LSB
13 FPTR64LSB
14 GPREL22
15 LDXMOV
16 LDXMOV
17 LTOFF22
18 LTOFF22X
19 LTOFF22X
20 LTOFF_FPTR22
21 PCREL21B (for br.call only; br.cond is not supported out of modules!)
22 PCREL60B (for brl.cond only; brl.call is not supported for modules!)
23 PCREL64LSB
24 SECREL32LSB
25 SEGREL64LSB
26 */
27
28
29 #include <linux/kernel.h>
30 #include <linux/sched.h>
31 #include <linux/elf.h>
32 #include <linux/moduleloader.h>
33 #include <linux/string.h>
34 #include <linux/vmalloc.h>
35
36 #include <asm/patch.h>
37 #include <asm/unaligned.h>
38
39 #define ARCH_MODULE_DEBUG 0
40
41 #if ARCH_MODULE_DEBUG
42 # define DEBUGP printk
43 # define inline
44 #else
45 # define DEBUGP(fmt , a...)
46 #endif
47
48 #ifdef CONFIG_ITANIUM
49 # define USE_BRL 0
50 #else
51 # define USE_BRL 1
52 #endif
53
54 #define MAX_LTOFF ((uint64_t) (1 << 22)) /* max. allowable linkage-table offset */
55
56 /* Define some relocation helper macros/types: */
57
58 #define FORMAT_SHIFT 0
59 #define FORMAT_BITS 3
60 #define FORMAT_MASK ((1 << FORMAT_BITS) - 1)
61 #define VALUE_SHIFT 3
62 #define VALUE_BITS 5
63 #define VALUE_MASK ((1 << VALUE_BITS) - 1)
64
65 enum reloc_target_format {
66 /* direct encoded formats: */
67 RF_NONE = 0,
68 RF_INSN14 = 1,
69 RF_INSN22 = 2,
70 RF_INSN64 = 3,
71 RF_32MSB = 4,
72 RF_32LSB = 5,
73 RF_64MSB = 6,
74 RF_64LSB = 7,
75
76 /* formats that cannot be directly decoded: */
77 RF_INSN60,
78 RF_INSN21B, /* imm21 form 1 */
79 RF_INSN21M, /* imm21 form 2 */
80 RF_INSN21F /* imm21 form 3 */
81 };
82
83 enum reloc_value_formula {
84 RV_DIRECT = 4, /* S + A */
85 RV_GPREL = 5, /* @gprel(S + A) */
86 RV_LTREL = 6, /* @ltoff(S + A) */
87 RV_PLTREL = 7, /* @pltoff(S + A) */
88 RV_FPTR = 8, /* @fptr(S + A) */
89 RV_PCREL = 9, /* S + A - P */
90 RV_LTREL_FPTR = 10, /* @ltoff(@fptr(S + A)) */
91 RV_SEGREL = 11, /* @segrel(S + A) */
92 RV_SECREL = 12, /* @secrel(S + A) */
93 RV_BDREL = 13, /* BD + A */
94 RV_LTV = 14, /* S + A (like RV_DIRECT, except frozen at static link-time) */
95 RV_PCREL2 = 15, /* S + A - P */
96 RV_SPECIAL = 16, /* various (see below) */
97 RV_RSVD17 = 17,
98 RV_TPREL = 18, /* @tprel(S + A) */
99 RV_LTREL_TPREL = 19, /* @ltoff(@tprel(S + A)) */
100 RV_DTPMOD = 20, /* @dtpmod(S + A) */
101 RV_LTREL_DTPMOD = 21, /* @ltoff(@dtpmod(S + A)) */
102 RV_DTPREL = 22, /* @dtprel(S + A) */
103 RV_LTREL_DTPREL = 23, /* @ltoff(@dtprel(S + A)) */
104 RV_RSVD24 = 24,
105 RV_RSVD25 = 25,
106 RV_RSVD26 = 26,
107 RV_RSVD27 = 27
108 /* 28-31 reserved for implementation-specific purposes. */
109 };
110
111 #define N(reloc) [R_IA64_##reloc] = #reloc
112
113 static const char *reloc_name[256] = {
114 N(NONE), N(IMM14), N(IMM22), N(IMM64),
115 N(DIR32MSB), N(DIR32LSB), N(DIR64MSB), N(DIR64LSB),
116 N(GPREL22), N(GPREL64I), N(GPREL32MSB), N(GPREL32LSB),
117 N(GPREL64MSB), N(GPREL64LSB), N(LTOFF22), N(LTOFF64I),
118 N(PLTOFF22), N(PLTOFF64I), N(PLTOFF64MSB), N(PLTOFF64LSB),
119 N(FPTR64I), N(FPTR32MSB), N(FPTR32LSB), N(FPTR64MSB),
120 N(FPTR64LSB), N(PCREL60B), N(PCREL21B), N(PCREL21M),
121 N(PCREL21F), N(PCREL32MSB), N(PCREL32LSB), N(PCREL64MSB),
122 N(PCREL64LSB), N(LTOFF_FPTR22), N(LTOFF_FPTR64I), N(LTOFF_FPTR32MSB),
123 N(LTOFF_FPTR32LSB), N(LTOFF_FPTR64MSB), N(LTOFF_FPTR64LSB), N(SEGREL32MSB),
124 N(SEGREL32LSB), N(SEGREL64MSB), N(SEGREL64LSB), N(SECREL32MSB),
125 N(SECREL32LSB), N(SECREL64MSB), N(SECREL64LSB), N(REL32MSB),
126 N(REL32LSB), N(REL64MSB), N(REL64LSB), N(LTV32MSB),
127 N(LTV32LSB), N(LTV64MSB), N(LTV64LSB), N(PCREL21BI),
128 N(PCREL22), N(PCREL64I), N(IPLTMSB), N(IPLTLSB),
129 N(COPY), N(LTOFF22X), N(LDXMOV), N(TPREL14),
130 N(TPREL22), N(TPREL64I), N(TPREL64MSB), N(TPREL64LSB),
131 N(LTOFF_TPREL22), N(DTPMOD64MSB), N(DTPMOD64LSB), N(LTOFF_DTPMOD22),
132 N(DTPREL14), N(DTPREL22), N(DTPREL64I), N(DTPREL32MSB),
133 N(DTPREL32LSB), N(DTPREL64MSB), N(DTPREL64LSB), N(LTOFF_DTPREL22)
134 };
135
136 #undef N
137
138 /* Opaque struct for insns, to protect against derefs. */
139 struct insn;
140
141 static inline uint64_t
142 bundle (const struct insn *insn)
143 {
144 return (uint64_t) insn & ~0xfUL;
145 }
146
147 static inline int
148 slot (const struct insn *insn)
149 {
150 return (uint64_t) insn & 0x3;
151 }
152
153 static int
154 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val)
155 {
156 if (slot(insn) != 2) {
157 printk(KERN_ERR "%s: invalid slot number %d for IMM64\n",
158 mod->name, slot(insn));
159 return 0;
160 }
161 ia64_patch_imm64((u64) insn, val);
162 return 1;
163 }
164
165 static int
166 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val)
167 {
168 if (slot(insn) != 2) {
169 printk(KERN_ERR "%s: invalid slot number %d for IMM60\n",
170 mod->name, slot(insn));
171 return 0;
172 }
173 if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) {
174 printk(KERN_ERR "%s: value %ld out of IMM60 range\n",
175 mod->name, (long) val);
176 return 0;
177 }
178 ia64_patch_imm60((u64) insn, val);
179 return 1;
180 }
181
182 static int
183 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val)
184 {
185 if (val + (1 << 21) >= (1 << 22)) {
186 printk(KERN_ERR "%s: value %li out of IMM22 range\n",
187 mod->name, (long)val);
188 return 0;
189 }
190 ia64_patch((u64) insn, 0x01fffcfe000UL, ( ((val & 0x200000UL) << 15) /* bit 21 -> 36 */
191 | ((val & 0x1f0000UL) << 6) /* bit 16 -> 22 */
192 | ((val & 0x00ff80UL) << 20) /* bit 7 -> 27 */
193 | ((val & 0x00007fUL) << 13) /* bit 0 -> 13 */));
194 return 1;
195 }
196
197 static int
198 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val)
199 {
200 if (val + (1 << 20) >= (1 << 21)) {
201 printk(KERN_ERR "%s: value %li out of IMM21b range\n",
202 mod->name, (long)val);
203 return 0;
204 }
205 ia64_patch((u64) insn, 0x11ffffe000UL, ( ((val & 0x100000UL) << 16) /* bit 20 -> 36 */
206 | ((val & 0x0fffffUL) << 13) /* bit 0 -> 13 */));
207 return 1;
208 }
209
210 #if USE_BRL
211
212 struct plt_entry {
213 /* Three instruction bundles in PLT. */
214 unsigned char bundle[2][16];
215 };
216
217 static const struct plt_entry ia64_plt_template = {
218 {
219 {
220 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
221 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */
222 0x00, 0x00, 0x00, 0x60
223 },
224 {
225 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
226 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* brl.many gp=TARGET_GP */
227 0x08, 0x00, 0x00, 0xc0
228 }
229 }
230 };
231
232 static int
233 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
234 {
235 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp)
236 && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2),
237 (target_ip - (int64_t) plt->bundle[1]) / 16))
238 return 1;
239 return 0;
240 }
241
242 unsigned long
243 plt_target (struct plt_entry *plt)
244 {
245 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1];
246 long off;
247
248 b0 = b[0]; b1 = b[1];
249 off = ( ((b1 & 0x00fffff000000000UL) >> 36) /* imm20b -> bit 0 */
250 | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36) /* imm39 -> bit 20 */
251 | ((b1 & 0x0800000000000000UL) << 0)); /* i -> bit 59 */
252 return (long) plt->bundle[1] + 16*off;
253 }
254
255 #else /* !USE_BRL */
256
257 struct plt_entry {
258 /* Three instruction bundles in PLT. */
259 unsigned char bundle[3][16];
260 };
261
262 static const struct plt_entry ia64_plt_template = {
263 {
264 {
265 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
266 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* movl r16=TARGET_IP */
267 0x02, 0x00, 0x00, 0x60
268 },
269 {
270 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
271 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */
272 0x00, 0x00, 0x00, 0x60
273 },
274 {
275 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */
276 0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /* mov b6=r16 */
277 0x60, 0x00, 0x80, 0x00 /* br.few b6 */
278 }
279 }
280 };
281
282 static int
283 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
284 {
285 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip)
286 && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp))
287 return 1;
288 return 0;
289 }
290
291 unsigned long
292 plt_target (struct plt_entry *plt)
293 {
294 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0];
295
296 b0 = b[0]; b1 = b[1];
297 return ( ((b1 & 0x000007f000000000) >> 36) /* imm7b -> bit 0 */
298 | ((b1 & 0x07fc000000000000) >> 43) /* imm9d -> bit 7 */
299 | ((b1 & 0x0003e00000000000) >> 29) /* imm5c -> bit 16 */
300 | ((b1 & 0x0000100000000000) >> 23) /* ic -> bit 21 */
301 | ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40) /* imm41 -> bit 22 */
302 | ((b1 & 0x0800000000000000) << 4)); /* i -> bit 63 */
303 }
304
305 #endif /* !USE_BRL */
306
307 void *
308 module_alloc (unsigned long size)
309 {
310 if (!size)
311 return NULL;
312 return vmalloc(size);
313 }
314
315 void
316 module_free (struct module *mod, void *module_region)
317 {
318 if (mod && mod->arch.init_unw_table &&
319 module_region == mod->module_init) {
320 unw_remove_unwind_table(mod->arch.init_unw_table);
321 mod->arch.init_unw_table = NULL;
322 }
323 vfree(module_region);
324 }
325
326 /* Have we already seen one of these relocations? */
327 /* FIXME: we could look in other sections, too --RR */
328 static int
329 duplicate_reloc (const Elf64_Rela *rela, unsigned int num)
330 {
331 unsigned int i;
332
333 for (i = 0; i < num; i++) {
334 if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend)
335 return 1;
336 }
337 return 0;
338 }
339
340 /* Count how many GOT entries we may need */
341 static unsigned int
342 count_gots (const Elf64_Rela *rela, unsigned int num)
343 {
344 unsigned int i, ret = 0;
345
346 /* Sure, this is order(n^2), but it's usually short, and not
347 time critical */
348 for (i = 0; i < num; i++) {
349 switch (ELF64_R_TYPE(rela[i].r_info)) {
350 case R_IA64_LTOFF22:
351 case R_IA64_LTOFF22X:
352 case R_IA64_LTOFF64I:
353 case R_IA64_LTOFF_FPTR22:
354 case R_IA64_LTOFF_FPTR64I:
355 case R_IA64_LTOFF_FPTR32MSB:
356 case R_IA64_LTOFF_FPTR32LSB:
357 case R_IA64_LTOFF_FPTR64MSB:
358 case R_IA64_LTOFF_FPTR64LSB:
359 if (!duplicate_reloc(rela, i))
360 ret++;
361 break;
362 }
363 }
364 return ret;
365 }
366
367 /* Count how many PLT entries we may need */
368 static unsigned int
369 count_plts (const Elf64_Rela *rela, unsigned int num)
370 {
371 unsigned int i, ret = 0;
372
373 /* Sure, this is order(n^2), but it's usually short, and not
374 time critical */
375 for (i = 0; i < num; i++) {
376 switch (ELF64_R_TYPE(rela[i].r_info)) {
377 case R_IA64_PCREL21B:
378 case R_IA64_PLTOFF22:
379 case R_IA64_PLTOFF64I:
380 case R_IA64_PLTOFF64MSB:
381 case R_IA64_PLTOFF64LSB:
382 case R_IA64_IPLTMSB:
383 case R_IA64_IPLTLSB:
384 if (!duplicate_reloc(rela, i))
385 ret++;
386 break;
387 }
388 }
389 return ret;
390 }
391
392 /* We need to create an function-descriptors for any internal function
393 which is referenced. */
394 static unsigned int
395 count_fdescs (const Elf64_Rela *rela, unsigned int num)
396 {
397 unsigned int i, ret = 0;
398
399 /* Sure, this is order(n^2), but it's usually short, and not time critical. */
400 for (i = 0; i < num; i++) {
401 switch (ELF64_R_TYPE(rela[i].r_info)) {
402 case R_IA64_FPTR64I:
403 case R_IA64_FPTR32LSB:
404 case R_IA64_FPTR32MSB:
405 case R_IA64_FPTR64LSB:
406 case R_IA64_FPTR64MSB:
407 case R_IA64_LTOFF_FPTR22:
408 case R_IA64_LTOFF_FPTR32LSB:
409 case R_IA64_LTOFF_FPTR32MSB:
410 case R_IA64_LTOFF_FPTR64I:
411 case R_IA64_LTOFF_FPTR64LSB:
412 case R_IA64_LTOFF_FPTR64MSB:
413 case R_IA64_IPLTMSB:
414 case R_IA64_IPLTLSB:
415 /*
416 * Jumps to static functions sometimes go straight to their
417 * offset. Of course, that may not be possible if the jump is
418 * from init -> core or vice. versa, so we need to generate an
419 * FDESC (and PLT etc) for that.
420 */
421 case R_IA64_PCREL21B:
422 if (!duplicate_reloc(rela, i))
423 ret++;
424 break;
425 }
426 }
427 return ret;
428 }
429
430 int
431 module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings,
432 struct module *mod)
433 {
434 unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0;
435 Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum;
436
437 /*
438 * To store the PLTs and function-descriptors, we expand the .text section for
439 * core module-code and the .init.text section for initialization code.
440 */
441 for (s = sechdrs; s < sechdrs_end; ++s)
442 if (strcmp(".core.plt", secstrings + s->sh_name) == 0)
443 mod->arch.core_plt = s;
444 else if (strcmp(".init.plt", secstrings + s->sh_name) == 0)
445 mod->arch.init_plt = s;
446 else if (strcmp(".got", secstrings + s->sh_name) == 0)
447 mod->arch.got = s;
448 else if (strcmp(".opd", secstrings + s->sh_name) == 0)
449 mod->arch.opd = s;
450 else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0)
451 mod->arch.unwind = s;
452 #ifdef CONFIG_PARAVIRT
453 else if (strcmp(".paravirt_bundles",
454 secstrings + s->sh_name) == 0)
455 mod->arch.paravirt_bundles = s;
456 else if (strcmp(".paravirt_insts",
457 secstrings + s->sh_name) == 0)
458 mod->arch.paravirt_insts = s;
459 #endif
460
461 if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) {
462 printk(KERN_ERR "%s: sections missing\n", mod->name);
463 return -ENOEXEC;
464 }
465
466 /* GOT and PLTs can occur in any relocated section... */
467 for (s = sechdrs + 1; s < sechdrs_end; ++s) {
468 const Elf64_Rela *rels = (void *)ehdr + s->sh_offset;
469 unsigned long numrels = s->sh_size/sizeof(Elf64_Rela);
470
471 if (s->sh_type != SHT_RELA)
472 continue;
473
474 gots += count_gots(rels, numrels);
475 fdescs += count_fdescs(rels, numrels);
476 if (strstr(secstrings + s->sh_name, ".init"))
477 init_plts += count_plts(rels, numrels);
478 else
479 core_plts += count_plts(rels, numrels);
480 }
481
482 mod->arch.core_plt->sh_type = SHT_NOBITS;
483 mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
484 mod->arch.core_plt->sh_addralign = 16;
485 mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry);
486 mod->arch.init_plt->sh_type = SHT_NOBITS;
487 mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
488 mod->arch.init_plt->sh_addralign = 16;
489 mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry);
490 mod->arch.got->sh_type = SHT_NOBITS;
491 mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC;
492 mod->arch.got->sh_addralign = 8;
493 mod->arch.got->sh_size = gots * sizeof(struct got_entry);
494 mod->arch.opd->sh_type = SHT_NOBITS;
495 mod->arch.opd->sh_flags = SHF_ALLOC;
496 mod->arch.opd->sh_addralign = 8;
497 mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc);
498 DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n",
499 __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size,
500 mod->arch.got->sh_size, mod->arch.opd->sh_size);
501 return 0;
502 }
503
504 static inline int
505 in_init (const struct module *mod, uint64_t addr)
506 {
507 return addr - (uint64_t) mod->module_init < mod->init_size;
508 }
509
510 static inline int
511 in_core (const struct module *mod, uint64_t addr)
512 {
513 return addr - (uint64_t) mod->module_core < mod->core_size;
514 }
515
516 static inline int
517 is_internal (const struct module *mod, uint64_t value)
518 {
519 return in_init(mod, value) || in_core(mod, value);
520 }
521
522 /*
523 * Get gp-relative offset for the linkage-table entry of VALUE.
524 */
525 static uint64_t
526 get_ltoff (struct module *mod, uint64_t value, int *okp)
527 {
528 struct got_entry *got, *e;
529
530 if (!*okp)
531 return 0;
532
533 got = (void *) mod->arch.got->sh_addr;
534 for (e = got; e < got + mod->arch.next_got_entry; ++e)
535 if (e->val == value)
536 goto found;
537
538 /* Not enough GOT entries? */
539 BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size));
540
541 e->val = value;
542 ++mod->arch.next_got_entry;
543 found:
544 return (uint64_t) e - mod->arch.gp;
545 }
546
547 static inline int
548 gp_addressable (struct module *mod, uint64_t value)
549 {
550 return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF;
551 }
552
553 /* Get PC-relative PLT entry for this value. Returns 0 on failure. */
554 static uint64_t
555 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp)
556 {
557 struct plt_entry *plt, *plt_end;
558 uint64_t target_ip, target_gp;
559
560 if (!*okp)
561 return 0;
562
563 if (in_init(mod, (uint64_t) insn)) {
564 plt = (void *) mod->arch.init_plt->sh_addr;
565 plt_end = (void *) plt + mod->arch.init_plt->sh_size;
566 } else {
567 plt = (void *) mod->arch.core_plt->sh_addr;
568 plt_end = (void *) plt + mod->arch.core_plt->sh_size;
569 }
570
571 /* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */
572 target_ip = ((uint64_t *) value)[0];
573 target_gp = ((uint64_t *) value)[1];
574
575 /* Look for existing PLT entry. */
576 while (plt->bundle[0][0]) {
577 if (plt_target(plt) == target_ip)
578 goto found;
579 if (++plt >= plt_end)
580 BUG();
581 }
582 *plt = ia64_plt_template;
583 if (!patch_plt(mod, plt, target_ip, target_gp)) {
584 *okp = 0;
585 return 0;
586 }
587 #if ARCH_MODULE_DEBUG
588 if (plt_target(plt) != target_ip) {
589 printk("%s: mistargeted PLT: wanted %lx, got %lx\n",
590 __func__, target_ip, plt_target(plt));
591 *okp = 0;
592 return 0;
593 }
594 #endif
595 found:
596 return (uint64_t) plt;
597 }
598
599 /* Get function descriptor for VALUE. */
600 static uint64_t
601 get_fdesc (struct module *mod, uint64_t value, int *okp)
602 {
603 struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr;
604
605 if (!*okp)
606 return 0;
607
608 if (!value) {
609 printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name);
610 return 0;
611 }
612
613 if (!is_internal(mod, value))
614 /*
615 * If it's not a module-local entry-point, "value" already points to a
616 * function-descriptor.
617 */
618 return value;
619
620 /* Look for existing function descriptor. */
621 while (fdesc->ip) {
622 if (fdesc->ip == value)
623 return (uint64_t)fdesc;
624 if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size)
625 BUG();
626 }
627
628 /* Create new one */
629 fdesc->ip = value;
630 fdesc->gp = mod->arch.gp;
631 return (uint64_t) fdesc;
632 }
633
634 static inline int
635 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend,
636 Elf64_Shdr *sec, void *location)
637 {
638 enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK;
639 enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK;
640 uint64_t val;
641 int ok = 1;
642
643 val = sym->st_value + addend;
644
645 switch (formula) {
646 case RV_SEGREL: /* segment base is arbitrarily chosen to be 0 for kernel modules */
647 case RV_DIRECT:
648 break;
649
650 case RV_GPREL: val -= mod->arch.gp; break;
651 case RV_LTREL: val = get_ltoff(mod, val, &ok); break;
652 case RV_PLTREL: val = get_plt(mod, location, val, &ok); break;
653 case RV_FPTR: val = get_fdesc(mod, val, &ok); break;
654 case RV_SECREL: val -= sec->sh_addr; break;
655 case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break;
656
657 case RV_PCREL:
658 switch (r_type) {
659 case R_IA64_PCREL21B:
660 if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) ||
661 (in_core(mod, val) && in_init(mod, (uint64_t)location))) {
662 /*
663 * Init section may have been allocated far away from core,
664 * if the branch won't reach, then allocate a plt for it.
665 */
666 uint64_t delta = ((int64_t)val - (int64_t)location) / 16;
667 if (delta + (1 << 20) >= (1 << 21)) {
668 val = get_fdesc(mod, val, &ok);
669 val = get_plt(mod, location, val, &ok);
670 }
671 } else if (!is_internal(mod, val))
672 val = get_plt(mod, location, val, &ok);
673 /* FALL THROUGH */
674 default:
675 val -= bundle(location);
676 break;
677
678 case R_IA64_PCREL32MSB:
679 case R_IA64_PCREL32LSB:
680 case R_IA64_PCREL64MSB:
681 case R_IA64_PCREL64LSB:
682 val -= (uint64_t) location;
683 break;
684
685 }
686 switch (r_type) {
687 case R_IA64_PCREL60B: format = RF_INSN60; break;
688 case R_IA64_PCREL21B: format = RF_INSN21B; break;
689 case R_IA64_PCREL21M: format = RF_INSN21M; break;
690 case R_IA64_PCREL21F: format = RF_INSN21F; break;
691 default: break;
692 }
693 break;
694
695 case RV_BDREL:
696 val -= (uint64_t) (in_init(mod, val) ? mod->module_init : mod->module_core);
697 break;
698
699 case RV_LTV:
700 /* can link-time value relocs happen here? */
701 BUG();
702 break;
703
704 case RV_PCREL2:
705 if (r_type == R_IA64_PCREL21BI) {
706 if (!is_internal(mod, val)) {
707 printk(KERN_ERR "%s: %s reloc against "
708 "non-local symbol (%lx)\n", __func__,
709 reloc_name[r_type], (unsigned long)val);
710 return -ENOEXEC;
711 }
712 format = RF_INSN21B;
713 }
714 val -= bundle(location);
715 break;
716
717 case RV_SPECIAL:
718 switch (r_type) {
719 case R_IA64_IPLTMSB:
720 case R_IA64_IPLTLSB:
721 val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok);
722 format = RF_64LSB;
723 if (r_type == R_IA64_IPLTMSB)
724 format = RF_64MSB;
725 break;
726
727 case R_IA64_SUB:
728 val = addend - sym->st_value;
729 format = RF_INSN64;
730 break;
731
732 case R_IA64_LTOFF22X:
733 if (gp_addressable(mod, val))
734 val -= mod->arch.gp;
735 else
736 val = get_ltoff(mod, val, &ok);
737 format = RF_INSN22;
738 break;
739
740 case R_IA64_LDXMOV:
741 if (gp_addressable(mod, val)) {
742 /* turn "ld8" into "mov": */
743 DEBUGP("%s: patching ld8 at %p to mov\n", __func__, location);
744 ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL);
745 }
746 return 0;
747
748 default:
749 if (reloc_name[r_type])
750 printk(KERN_ERR "%s: special reloc %s not supported",
751 mod->name, reloc_name[r_type]);
752 else
753 printk(KERN_ERR "%s: unknown special reloc %x\n",
754 mod->name, r_type);
755 return -ENOEXEC;
756 }
757 break;
758
759 case RV_TPREL:
760 case RV_LTREL_TPREL:
761 case RV_DTPMOD:
762 case RV_LTREL_DTPMOD:
763 case RV_DTPREL:
764 case RV_LTREL_DTPREL:
765 printk(KERN_ERR "%s: %s reloc not supported\n",
766 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?");
767 return -ENOEXEC;
768
769 default:
770 printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type);
771 return -ENOEXEC;
772 }
773
774 if (!ok)
775 return -ENOEXEC;
776
777 DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __func__, location, val,
778 reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend);
779
780 switch (format) {
781 case RF_INSN21B: ok = apply_imm21b(mod, location, (int64_t) val / 16); break;
782 case RF_INSN22: ok = apply_imm22(mod, location, val); break;
783 case RF_INSN64: ok = apply_imm64(mod, location, val); break;
784 case RF_INSN60: ok = apply_imm60(mod, location, (int64_t) val / 16); break;
785 case RF_32LSB: put_unaligned(val, (uint32_t *) location); break;
786 case RF_64LSB: put_unaligned(val, (uint64_t *) location); break;
787 case RF_32MSB: /* ia64 Linux is little-endian... */
788 case RF_64MSB: /* ia64 Linux is little-endian... */
789 case RF_INSN14: /* must be within-module, i.e., resolved by "ld -r" */
790 case RF_INSN21M: /* must be within-module, i.e., resolved by "ld -r" */
791 case RF_INSN21F: /* must be within-module, i.e., resolved by "ld -r" */
792 printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n",
793 mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?");
794 return -ENOEXEC;
795
796 default:
797 printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n",
798 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format);
799 return -ENOEXEC;
800 }
801 return ok ? 0 : -ENOEXEC;
802 }
803
804 int
805 apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex,
806 unsigned int relsec, struct module *mod)
807 {
808 unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela);
809 Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr;
810 Elf64_Shdr *target_sec;
811 int ret;
812
813 DEBUGP("%s: applying section %u (%u relocs) to %u\n", __func__,
814 relsec, n, sechdrs[relsec].sh_info);
815
816 target_sec = sechdrs + sechdrs[relsec].sh_info;
817
818 if (target_sec->sh_entsize == ~0UL)
819 /*
820 * If target section wasn't allocated, we don't need to relocate it.
821 * Happens, e.g., for debug sections.
822 */
823 return 0;
824
825 if (!mod->arch.gp) {
826 /*
827 * XXX Should have an arch-hook for running this after final section
828 * addresses have been selected...
829 */
830 uint64_t gp;
831 if (mod->core_size > MAX_LTOFF)
832 /*
833 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated
834 * at the end of the module.
835 */
836 gp = mod->core_size - MAX_LTOFF / 2;
837 else
838 gp = mod->core_size / 2;
839 gp = (uint64_t) mod->module_core + ((gp + 7) & -8);
840 mod->arch.gp = gp;
841 DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp);
842 }
843
844 for (i = 0; i < n; i++) {
845 ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info),
846 ((Elf64_Sym *) sechdrs[symindex].sh_addr
847 + ELF64_R_SYM(rela[i].r_info)),
848 rela[i].r_addend, target_sec,
849 (void *) target_sec->sh_addr + rela[i].r_offset);
850 if (ret < 0)
851 return ret;
852 }
853 return 0;
854 }
855
856 int
857 apply_relocate (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex,
858 unsigned int relsec, struct module *mod)
859 {
860 printk(KERN_ERR "module %s: REL relocs in section %u unsupported\n", mod->name, relsec);
861 return -ENOEXEC;
862 }
863
864 /*
865 * Modules contain a single unwind table which covers both the core and the init text
866 * sections but since the two are not contiguous, we need to split this table up such that
867 * we can register (and unregister) each "segment" separately. Fortunately, this sounds
868 * more complicated than it really is.
869 */
870 static void
871 register_unwind_table (struct module *mod)
872 {
873 struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr;
874 struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start);
875 struct unw_table_entry tmp, *e1, *e2, *core, *init;
876 unsigned long num_init = 0, num_core = 0;
877
878 /* First, count how many init and core unwind-table entries there are. */
879 for (e1 = start; e1 < end; ++e1)
880 if (in_init(mod, e1->start_offset))
881 ++num_init;
882 else
883 ++num_core;
884 /*
885 * Second, sort the table such that all unwind-table entries for the init and core
886 * text sections are nicely separated. We do this with a stupid bubble sort
887 * (unwind tables don't get ridiculously huge).
888 */
889 for (e1 = start; e1 < end; ++e1) {
890 for (e2 = e1 + 1; e2 < end; ++e2) {
891 if (e2->start_offset < e1->start_offset) {
892 tmp = *e1;
893 *e1 = *e2;
894 *e2 = tmp;
895 }
896 }
897 }
898 /*
899 * Third, locate the init and core segments in the unwind table:
900 */
901 if (in_init(mod, start->start_offset)) {
902 init = start;
903 core = start + num_init;
904 } else {
905 core = start;
906 init = start + num_core;
907 }
908
909 DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __func__,
910 mod->name, mod->arch.gp, num_init, num_core);
911
912 /*
913 * Fourth, register both tables (if not empty).
914 */
915 if (num_core > 0) {
916 mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
917 core, core + num_core);
918 DEBUGP("%s: core: handle=%p [%p-%p)\n", __func__,
919 mod->arch.core_unw_table, core, core + num_core);
920 }
921 if (num_init > 0) {
922 mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
923 init, init + num_init);
924 DEBUGP("%s: init: handle=%p [%p-%p)\n", __func__,
925 mod->arch.init_unw_table, init, init + num_init);
926 }
927 }
928
929 int
930 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod)
931 {
932 DEBUGP("%s: init: entry=%p\n", __func__, mod->init);
933 if (mod->arch.unwind)
934 register_unwind_table(mod);
935 #ifdef CONFIG_PARAVIRT
936 if (mod->arch.paravirt_bundles) {
937 struct paravirt_patch_site_bundle *start =
938 (struct paravirt_patch_site_bundle *)
939 mod->arch.paravirt_bundles->sh_addr;
940 struct paravirt_patch_site_bundle *end =
941 (struct paravirt_patch_site_bundle *)
942 (mod->arch.paravirt_bundles->sh_addr +
943 mod->arch.paravirt_bundles->sh_size);
944
945 paravirt_patch_apply_bundle(start, end);
946 }
947 if (mod->arch.paravirt_insts) {
948 struct paravirt_patch_site_inst *start =
949 (struct paravirt_patch_site_inst *)
950 mod->arch.paravirt_insts->sh_addr;
951 struct paravirt_patch_site_inst *end =
952 (struct paravirt_patch_site_inst *)
953 (mod->arch.paravirt_insts->sh_addr +
954 mod->arch.paravirt_insts->sh_size);
955
956 paravirt_patch_apply_inst(start, end);
957 }
958 #endif
959 return 0;
960 }
961
962 void
963 module_arch_cleanup (struct module *mod)
964 {
965 if (mod->arch.init_unw_table)
966 unw_remove_unwind_table(mod->arch.init_unw_table);
967 if (mod->arch.core_unw_table)
968 unw_remove_unwind_table(mod->arch.core_unw_table);
969 }
linux-next