?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mm: fix exec activate_mm vs TLB shootdown and lazy tlb switching race
[linux/fpc-iii.git] / arch / ia64 / kernel / module.c
blob95e8d130e1235a02eef095d35fbbf4ef9ddebfd1
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * IA-64-specific support for kernel module loader.
4 *
5 * Copyright (C) 2003 Hewlett-Packard Co
6 * David Mosberger-Tang <davidm@hpl.hp.com>
7 *
8 * Loosely based on patch by Rusty Russell.
9 */
10
11 /* relocs tested so far:
12
13 DIR64LSB
14 FPTR64LSB
15 GPREL22
16 LDXMOV
17 LDXMOV
18 LTOFF22
19 LTOFF22X
20 LTOFF22X
21 LTOFF_FPTR22
22 PCREL21B (for br.call only; br.cond is not supported out of modules!)
23 PCREL60B (for brl.cond only; brl.call is not supported for modules!)
24 PCREL64LSB
25 SECREL32LSB
26 SEGREL64LSB
27 */
28
29
30 #include <linux/kernel.h>
31 #include <linux/sched.h>
32 #include <linux/elf.h>
33 #include <linux/moduleloader.h>
34 #include <linux/string.h>
35 #include <linux/vmalloc.h>
36
37 #include <asm/patch.h>
38 #include <asm/unaligned.h>
39
40 #define ARCH_MODULE_DEBUG 0
41
42 #if ARCH_MODULE_DEBUG
43 # define DEBUGP printk
44 # define inline
45 #else
46 # define DEBUGP(fmt , a...)
47 #endif
48
49 #ifdef CONFIG_ITANIUM
50 # define USE_BRL 0
51 #else
52 # define USE_BRL 1
53 #endif
54
55 #define MAX_LTOFF ((uint64_t) (1 << 22)) /* max. allowable linkage-table offset */
56
57 /* Define some relocation helper macros/types: */
58
59 #define FORMAT_SHIFT 0
60 #define FORMAT_BITS 3
61 #define FORMAT_MASK ((1 << FORMAT_BITS) - 1)
62 #define VALUE_SHIFT 3
63 #define VALUE_BITS 5
64 #define VALUE_MASK ((1 << VALUE_BITS) - 1)
65
66 enum reloc_target_format {
67 /* direct encoded formats: */
68 RF_NONE = 0,
69 RF_INSN14 = 1,
70 RF_INSN22 = 2,
71 RF_INSN64 = 3,
72 RF_32MSB = 4,
73 RF_32LSB = 5,
74 RF_64MSB = 6,
75 RF_64LSB = 7,
76
77 /* formats that cannot be directly decoded: */
78 RF_INSN60,
79 RF_INSN21B, /* imm21 form 1 */
80 RF_INSN21M, /* imm21 form 2 */
81 RF_INSN21F /* imm21 form 3 */
82 };
83
84 enum reloc_value_formula {
85 RV_DIRECT = 4, /* S + A */
86 RV_GPREL = 5, /* @gprel(S + A) */
87 RV_LTREL = 6, /* @ltoff(S + A) */
88 RV_PLTREL = 7, /* @pltoff(S + A) */
89 RV_FPTR = 8, /* @fptr(S + A) */
90 RV_PCREL = 9, /* S + A - P */
91 RV_LTREL_FPTR = 10, /* @ltoff(@fptr(S + A)) */
92 RV_SEGREL = 11, /* @segrel(S + A) */
93 RV_SECREL = 12, /* @secrel(S + A) */
94 RV_BDREL = 13, /* BD + A */
95 RV_LTV = 14, /* S + A (like RV_DIRECT, except frozen at static link-time) */
96 RV_PCREL2 = 15, /* S + A - P */
97 RV_SPECIAL = 16, /* various (see below) */
98 RV_RSVD17 = 17,
99 RV_TPREL = 18, /* @tprel(S + A) */
100 RV_LTREL_TPREL = 19, /* @ltoff(@tprel(S + A)) */
101 RV_DTPMOD = 20, /* @dtpmod(S + A) */
102 RV_LTREL_DTPMOD = 21, /* @ltoff(@dtpmod(S + A)) */
103 RV_DTPREL = 22, /* @dtprel(S + A) */
104 RV_LTREL_DTPREL = 23, /* @ltoff(@dtprel(S + A)) */
105 RV_RSVD24 = 24,
106 RV_RSVD25 = 25,
107 RV_RSVD26 = 26,
108 RV_RSVD27 = 27
109 /* 28-31 reserved for implementation-specific purposes. */
110 };
111
112 #define N(reloc) [R_IA64_##reloc] = #reloc
113
114 static const char *reloc_name[256] = {
115 N(NONE), N(IMM14), N(IMM22), N(IMM64),
116 N(DIR32MSB), N(DIR32LSB), N(DIR64MSB), N(DIR64LSB),
117 N(GPREL22), N(GPREL64I), N(GPREL32MSB), N(GPREL32LSB),
118 N(GPREL64MSB), N(GPREL64LSB), N(LTOFF22), N(LTOFF64I),
119 N(PLTOFF22), N(PLTOFF64I), N(PLTOFF64MSB), N(PLTOFF64LSB),
120 N(FPTR64I), N(FPTR32MSB), N(FPTR32LSB), N(FPTR64MSB),
121 N(FPTR64LSB), N(PCREL60B), N(PCREL21B), N(PCREL21M),
122 N(PCREL21F), N(PCREL32MSB), N(PCREL32LSB), N(PCREL64MSB),
123 N(PCREL64LSB), N(LTOFF_FPTR22), N(LTOFF_FPTR64I), N(LTOFF_FPTR32MSB),
124 N(LTOFF_FPTR32LSB), N(LTOFF_FPTR64MSB), N(LTOFF_FPTR64LSB), N(SEGREL32MSB),
125 N(SEGREL32LSB), N(SEGREL64MSB), N(SEGREL64LSB), N(SECREL32MSB),
126 N(SECREL32LSB), N(SECREL64MSB), N(SECREL64LSB), N(REL32MSB),
127 N(REL32LSB), N(REL64MSB), N(REL64LSB), N(LTV32MSB),
128 N(LTV32LSB), N(LTV64MSB), N(LTV64LSB), N(PCREL21BI),
129 N(PCREL22), N(PCREL64I), N(IPLTMSB), N(IPLTLSB),
130 N(COPY), N(LTOFF22X), N(LDXMOV), N(TPREL14),
131 N(TPREL22), N(TPREL64I), N(TPREL64MSB), N(TPREL64LSB),
132 N(LTOFF_TPREL22), N(DTPMOD64MSB), N(DTPMOD64LSB), N(LTOFF_DTPMOD22),
133 N(DTPREL14), N(DTPREL22), N(DTPREL64I), N(DTPREL32MSB),
134 N(DTPREL32LSB), N(DTPREL64MSB), N(DTPREL64LSB), N(LTOFF_DTPREL22)
135 };
136
137 #undef N
138
139 /* Opaque struct for insns, to protect against derefs. */
140 struct insn;
141
142 static inline uint64_t
143 bundle (const struct insn *insn)
144 {
145 return (uint64_t) insn & ~0xfUL;
146 }
147
148 static inline int
149 slot (const struct insn *insn)
150 {
151 return (uint64_t) insn & 0x3;
152 }
153
154 static int
155 apply_imm64 (struct module *mod, struct insn *insn, uint64_t val)
156 {
157 if (slot(insn) != 1 && slot(insn) != 2) {
158 printk(KERN_ERR "%s: invalid slot number %d for IMM64\n",
159 mod->name, slot(insn));
160 return 0;
161 }
162 ia64_patch_imm64((u64) insn, val);
163 return 1;
164 }
165
166 static int
167 apply_imm60 (struct module *mod, struct insn *insn, uint64_t val)
168 {
169 if (slot(insn) != 1 && slot(insn) != 2) {
170 printk(KERN_ERR "%s: invalid slot number %d for IMM60\n",
171 mod->name, slot(insn));
172 return 0;
173 }
174 if (val + ((uint64_t) 1 << 59) >= (1UL << 60)) {
175 printk(KERN_ERR "%s: value %ld out of IMM60 range\n",
176 mod->name, (long) val);
177 return 0;
178 }
179 ia64_patch_imm60((u64) insn, val);
180 return 1;
181 }
182
183 static int
184 apply_imm22 (struct module *mod, struct insn *insn, uint64_t val)
185 {
186 if (val + (1 << 21) >= (1 << 22)) {
187 printk(KERN_ERR "%s: value %li out of IMM22 range\n",
188 mod->name, (long)val);
189 return 0;
190 }
191 ia64_patch((u64) insn, 0x01fffcfe000UL, ( ((val & 0x200000UL) << 15) /* bit 21 -> 36 */
192 | ((val & 0x1f0000UL) << 6) /* bit 16 -> 22 */
193 | ((val & 0x00ff80UL) << 20) /* bit 7 -> 27 */
194 | ((val & 0x00007fUL) << 13) /* bit 0 -> 13 */));
195 return 1;
196 }
197
198 static int
199 apply_imm21b (struct module *mod, struct insn *insn, uint64_t val)
200 {
201 if (val + (1 << 20) >= (1 << 21)) {
202 printk(KERN_ERR "%s: value %li out of IMM21b range\n",
203 mod->name, (long)val);
204 return 0;
205 }
206 ia64_patch((u64) insn, 0x11ffffe000UL, ( ((val & 0x100000UL) << 16) /* bit 20 -> 36 */
207 | ((val & 0x0fffffUL) << 13) /* bit 0 -> 13 */));
208 return 1;
209 }
210
211 #if USE_BRL
212
213 struct plt_entry {
214 /* Three instruction bundles in PLT. */
215 unsigned char bundle[2][16];
216 };
217
218 static const struct plt_entry ia64_plt_template = {
219 {
220 {
221 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
222 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */
223 0x00, 0x00, 0x00, 0x60
224 },
225 {
226 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
227 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* brl.many gp=TARGET_GP */
228 0x08, 0x00, 0x00, 0xc0
229 }
230 }
231 };
232
233 static int
234 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
235 {
236 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_gp)
237 && apply_imm60(mod, (struct insn *) (plt->bundle[1] + 2),
238 (target_ip - (int64_t) plt->bundle[1]) / 16))
239 return 1;
240 return 0;
241 }
242
243 unsigned long
244 plt_target (struct plt_entry *plt)
245 {
246 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[1];
247 long off;
248
249 b0 = b[0]; b1 = b[1];
250 off = ( ((b1 & 0x00fffff000000000UL) >> 36) /* imm20b -> bit 0 */
251 | ((b0 >> 48) << 20) | ((b1 & 0x7fffffUL) << 36) /* imm39 -> bit 20 */
252 | ((b1 & 0x0800000000000000UL) << 0)); /* i -> bit 59 */
253 return (long) plt->bundle[1] + 16*off;
254 }
255
256 #else /* !USE_BRL */
257
258 struct plt_entry {
259 /* Three instruction bundles in PLT. */
260 unsigned char bundle[3][16];
261 };
262
263 static const struct plt_entry ia64_plt_template = {
264 {
265 {
266 0x05, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
267 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* movl r16=TARGET_IP */
268 0x02, 0x00, 0x00, 0x60
269 },
270 {
271 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MLX] nop.m 0 */
272 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, /* movl gp=TARGET_GP */
273 0x00, 0x00, 0x00, 0x60
274 },
275 {
276 0x11, 0x00, 0x00, 0x00, 0x01, 0x00, /* [MIB] nop.m 0 */
277 0x60, 0x80, 0x04, 0x80, 0x03, 0x00, /* mov b6=r16 */
278 0x60, 0x00, 0x80, 0x00 /* br.few b6 */
279 }
280 }
281 };
282
283 static int
284 patch_plt (struct module *mod, struct plt_entry *plt, long target_ip, unsigned long target_gp)
285 {
286 if (apply_imm64(mod, (struct insn *) (plt->bundle[0] + 2), target_ip)
287 && apply_imm64(mod, (struct insn *) (plt->bundle[1] + 2), target_gp))
288 return 1;
289 return 0;
290 }
291
292 unsigned long
293 plt_target (struct plt_entry *plt)
294 {
295 uint64_t b0, b1, *b = (uint64_t *) plt->bundle[0];
296
297 b0 = b[0]; b1 = b[1];
298 return ( ((b1 & 0x000007f000000000) >> 36) /* imm7b -> bit 0 */
299 | ((b1 & 0x07fc000000000000) >> 43) /* imm9d -> bit 7 */
300 | ((b1 & 0x0003e00000000000) >> 29) /* imm5c -> bit 16 */
301 | ((b1 & 0x0000100000000000) >> 23) /* ic -> bit 21 */
302 | ((b0 >> 46) << 22) | ((b1 & 0x7fffff) << 40) /* imm41 -> bit 22 */
303 | ((b1 & 0x0800000000000000) << 4)); /* i -> bit 63 */
304 }
305
306 #endif /* !USE_BRL */
307
308 void
309 module_arch_freeing_init (struct module *mod)
310 {
311 if (mod->arch.init_unw_table) {
312 unw_remove_unwind_table(mod->arch.init_unw_table);
313 mod->arch.init_unw_table = NULL;
314 }
315 }
316
317 /* Have we already seen one of these relocations? */
318 /* FIXME: we could look in other sections, too --RR */
319 static int
320 duplicate_reloc (const Elf64_Rela *rela, unsigned int num)
321 {
322 unsigned int i;
323
324 for (i = 0; i < num; i++) {
325 if (rela[i].r_info == rela[num].r_info && rela[i].r_addend == rela[num].r_addend)
326 return 1;
327 }
328 return 0;
329 }
330
331 /* Count how many GOT entries we may need */
332 static unsigned int
333 count_gots (const Elf64_Rela *rela, unsigned int num)
334 {
335 unsigned int i, ret = 0;
336
337 /* Sure, this is order(n^2), but it's usually short, and not
338 time critical */
339 for (i = 0; i < num; i++) {
340 switch (ELF64_R_TYPE(rela[i].r_info)) {
341 case R_IA64_LTOFF22:
342 case R_IA64_LTOFF22X:
343 case R_IA64_LTOFF64I:
344 case R_IA64_LTOFF_FPTR22:
345 case R_IA64_LTOFF_FPTR64I:
346 case R_IA64_LTOFF_FPTR32MSB:
347 case R_IA64_LTOFF_FPTR32LSB:
348 case R_IA64_LTOFF_FPTR64MSB:
349 case R_IA64_LTOFF_FPTR64LSB:
350 if (!duplicate_reloc(rela, i))
351 ret++;
352 break;
353 }
354 }
355 return ret;
356 }
357
358 /* Count how many PLT entries we may need */
359 static unsigned int
360 count_plts (const Elf64_Rela *rela, unsigned int num)
361 {
362 unsigned int i, ret = 0;
363
364 /* Sure, this is order(n^2), but it's usually short, and not
365 time critical */
366 for (i = 0; i < num; i++) {
367 switch (ELF64_R_TYPE(rela[i].r_info)) {
368 case R_IA64_PCREL21B:
369 case R_IA64_PLTOFF22:
370 case R_IA64_PLTOFF64I:
371 case R_IA64_PLTOFF64MSB:
372 case R_IA64_PLTOFF64LSB:
373 case R_IA64_IPLTMSB:
374 case R_IA64_IPLTLSB:
375 if (!duplicate_reloc(rela, i))
376 ret++;
377 break;
378 }
379 }
380 return ret;
381 }
382
383 /* We need to create an function-descriptors for any internal function
384 which is referenced. */
385 static unsigned int
386 count_fdescs (const Elf64_Rela *rela, unsigned int num)
387 {
388 unsigned int i, ret = 0;
389
390 /* Sure, this is order(n^2), but it's usually short, and not time critical. */
391 for (i = 0; i < num; i++) {
392 switch (ELF64_R_TYPE(rela[i].r_info)) {
393 case R_IA64_FPTR64I:
394 case R_IA64_FPTR32LSB:
395 case R_IA64_FPTR32MSB:
396 case R_IA64_FPTR64LSB:
397 case R_IA64_FPTR64MSB:
398 case R_IA64_LTOFF_FPTR22:
399 case R_IA64_LTOFF_FPTR32LSB:
400 case R_IA64_LTOFF_FPTR32MSB:
401 case R_IA64_LTOFF_FPTR64I:
402 case R_IA64_LTOFF_FPTR64LSB:
403 case R_IA64_LTOFF_FPTR64MSB:
404 case R_IA64_IPLTMSB:
405 case R_IA64_IPLTLSB:
406 /*
407 * Jumps to static functions sometimes go straight to their
408 * offset. Of course, that may not be possible if the jump is
409 * from init -> core or vice. versa, so we need to generate an
410 * FDESC (and PLT etc) for that.
411 */
412 case R_IA64_PCREL21B:
413 if (!duplicate_reloc(rela, i))
414 ret++;
415 break;
416 }
417 }
418 return ret;
419 }
420
421 int
422 module_frob_arch_sections (Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings,
423 struct module *mod)
424 {
425 unsigned long core_plts = 0, init_plts = 0, gots = 0, fdescs = 0;
426 Elf64_Shdr *s, *sechdrs_end = sechdrs + ehdr->e_shnum;
427
428 /*
429 * To store the PLTs and function-descriptors, we expand the .text section for
430 * core module-code and the .init.text section for initialization code.
431 */
432 for (s = sechdrs; s < sechdrs_end; ++s)
433 if (strcmp(".core.plt", secstrings + s->sh_name) == 0)
434 mod->arch.core_plt = s;
435 else if (strcmp(".init.plt", secstrings + s->sh_name) == 0)
436 mod->arch.init_plt = s;
437 else if (strcmp(".got", secstrings + s->sh_name) == 0)
438 mod->arch.got = s;
439 else if (strcmp(".opd", secstrings + s->sh_name) == 0)
440 mod->arch.opd = s;
441 else if (strcmp(".IA_64.unwind", secstrings + s->sh_name) == 0)
442 mod->arch.unwind = s;
443
444 if (!mod->arch.core_plt || !mod->arch.init_plt || !mod->arch.got || !mod->arch.opd) {
445 printk(KERN_ERR "%s: sections missing\n", mod->name);
446 return -ENOEXEC;
447 }
448
449 /* GOT and PLTs can occur in any relocated section... */
450 for (s = sechdrs + 1; s < sechdrs_end; ++s) {
451 const Elf64_Rela *rels = (void *)ehdr + s->sh_offset;
452 unsigned long numrels = s->sh_size/sizeof(Elf64_Rela);
453
454 if (s->sh_type != SHT_RELA)
455 continue;
456
457 gots += count_gots(rels, numrels);
458 fdescs += count_fdescs(rels, numrels);
459 if (strstr(secstrings + s->sh_name, ".init"))
460 init_plts += count_plts(rels, numrels);
461 else
462 core_plts += count_plts(rels, numrels);
463 }
464
465 mod->arch.core_plt->sh_type = SHT_NOBITS;
466 mod->arch.core_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
467 mod->arch.core_plt->sh_addralign = 16;
468 mod->arch.core_plt->sh_size = core_plts * sizeof(struct plt_entry);
469 mod->arch.init_plt->sh_type = SHT_NOBITS;
470 mod->arch.init_plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
471 mod->arch.init_plt->sh_addralign = 16;
472 mod->arch.init_plt->sh_size = init_plts * sizeof(struct plt_entry);
473 mod->arch.got->sh_type = SHT_NOBITS;
474 mod->arch.got->sh_flags = ARCH_SHF_SMALL | SHF_ALLOC;
475 mod->arch.got->sh_addralign = 8;
476 mod->arch.got->sh_size = gots * sizeof(struct got_entry);
477 mod->arch.opd->sh_type = SHT_NOBITS;
478 mod->arch.opd->sh_flags = SHF_ALLOC;
479 mod->arch.opd->sh_addralign = 8;
480 mod->arch.opd->sh_size = fdescs * sizeof(struct fdesc);
481 DEBUGP("%s: core.plt=%lx, init.plt=%lx, got=%lx, fdesc=%lx\n",
482 __func__, mod->arch.core_plt->sh_size, mod->arch.init_plt->sh_size,
483 mod->arch.got->sh_size, mod->arch.opd->sh_size);
484 return 0;
485 }
486
487 static inline int
488 in_init (const struct module *mod, uint64_t addr)
489 {
490 return addr - (uint64_t) mod->init_layout.base < mod->init_layout.size;
491 }
492
493 static inline int
494 in_core (const struct module *mod, uint64_t addr)
495 {
496 return addr - (uint64_t) mod->core_layout.base < mod->core_layout.size;
497 }
498
499 static inline int
500 is_internal (const struct module *mod, uint64_t value)
501 {
502 return in_init(mod, value) || in_core(mod, value);
503 }
504
505 /*
506 * Get gp-relative offset for the linkage-table entry of VALUE.
507 */
508 static uint64_t
509 get_ltoff (struct module *mod, uint64_t value, int *okp)
510 {
511 struct got_entry *got, *e;
512
513 if (!*okp)
514 return 0;
515
516 got = (void *) mod->arch.got->sh_addr;
517 for (e = got; e < got + mod->arch.next_got_entry; ++e)
518 if (e->val == value)
519 goto found;
520
521 /* Not enough GOT entries? */
522 BUG_ON(e >= (struct got_entry *) (mod->arch.got->sh_addr + mod->arch.got->sh_size));
523
524 e->val = value;
525 ++mod->arch.next_got_entry;
526 found:
527 return (uint64_t) e - mod->arch.gp;
528 }
529
530 static inline int
531 gp_addressable (struct module *mod, uint64_t value)
532 {
533 return value - mod->arch.gp + MAX_LTOFF/2 < MAX_LTOFF;
534 }
535
536 /* Get PC-relative PLT entry for this value. Returns 0 on failure. */
537 static uint64_t
538 get_plt (struct module *mod, const struct insn *insn, uint64_t value, int *okp)
539 {
540 struct plt_entry *plt, *plt_end;
541 uint64_t target_ip, target_gp;
542
543 if (!*okp)
544 return 0;
545
546 if (in_init(mod, (uint64_t) insn)) {
547 plt = (void *) mod->arch.init_plt->sh_addr;
548 plt_end = (void *) plt + mod->arch.init_plt->sh_size;
549 } else {
550 plt = (void *) mod->arch.core_plt->sh_addr;
551 plt_end = (void *) plt + mod->arch.core_plt->sh_size;
552 }
553
554 /* "value" is a pointer to a function-descriptor; fetch the target ip/gp from it: */
555 target_ip = ((uint64_t *) value)[0];
556 target_gp = ((uint64_t *) value)[1];
557
558 /* Look for existing PLT entry. */
559 while (plt->bundle[0][0]) {
560 if (plt_target(plt) == target_ip)
561 goto found;
562 if (++plt >= plt_end)
563 BUG();
564 }
565 *plt = ia64_plt_template;
566 if (!patch_plt(mod, plt, target_ip, target_gp)) {
567 *okp = 0;
568 return 0;
569 }
570 #if ARCH_MODULE_DEBUG
571 if (plt_target(plt) != target_ip) {
572 printk("%s: mistargeted PLT: wanted %lx, got %lx\n",
573 __func__, target_ip, plt_target(plt));
574 *okp = 0;
575 return 0;
576 }
577 #endif
578 found:
579 return (uint64_t) plt;
580 }
581
582 /* Get function descriptor for VALUE. */
583 static uint64_t
584 get_fdesc (struct module *mod, uint64_t value, int *okp)
585 {
586 struct fdesc *fdesc = (void *) mod->arch.opd->sh_addr;
587
588 if (!*okp)
589 return 0;
590
591 if (!value) {
592 printk(KERN_ERR "%s: fdesc for zero requested!\n", mod->name);
593 return 0;
594 }
595
596 if (!is_internal(mod, value))
597 /*
598 * If it's not a module-local entry-point, "value" already points to a
599 * function-descriptor.
600 */
601 return value;
602
603 /* Look for existing function descriptor. */
604 while (fdesc->ip) {
605 if (fdesc->ip == value)
606 return (uint64_t)fdesc;
607 if ((uint64_t) ++fdesc >= mod->arch.opd->sh_addr + mod->arch.opd->sh_size)
608 BUG();
609 }
610
611 /* Create new one */
612 fdesc->ip = value;
613 fdesc->gp = mod->arch.gp;
614 return (uint64_t) fdesc;
615 }
616
617 static inline int
618 do_reloc (struct module *mod, uint8_t r_type, Elf64_Sym *sym, uint64_t addend,
619 Elf64_Shdr *sec, void *location)
620 {
621 enum reloc_target_format format = (r_type >> FORMAT_SHIFT) & FORMAT_MASK;
622 enum reloc_value_formula formula = (r_type >> VALUE_SHIFT) & VALUE_MASK;
623 uint64_t val;
624 int ok = 1;
625
626 val = sym->st_value + addend;
627
628 switch (formula) {
629 case RV_SEGREL: /* segment base is arbitrarily chosen to be 0 for kernel modules */
630 case RV_DIRECT:
631 break;
632
633 case RV_GPREL: val -= mod->arch.gp; break;
634 case RV_LTREL: val = get_ltoff(mod, val, &ok); break;
635 case RV_PLTREL: val = get_plt(mod, location, val, &ok); break;
636 case RV_FPTR: val = get_fdesc(mod, val, &ok); break;
637 case RV_SECREL: val -= sec->sh_addr; break;
638 case RV_LTREL_FPTR: val = get_ltoff(mod, get_fdesc(mod, val, &ok), &ok); break;
639
640 case RV_PCREL:
641 switch (r_type) {
642 case R_IA64_PCREL21B:
643 if ((in_init(mod, val) && in_core(mod, (uint64_t)location)) ||
644 (in_core(mod, val) && in_init(mod, (uint64_t)location))) {
645 /*
646 * Init section may have been allocated far away from core,
647 * if the branch won't reach, then allocate a plt for it.
648 */
649 uint64_t delta = ((int64_t)val - (int64_t)location) / 16;
650 if (delta + (1 << 20) >= (1 << 21)) {
651 val = get_fdesc(mod, val, &ok);
652 val = get_plt(mod, location, val, &ok);
653 }
654 } else if (!is_internal(mod, val))
655 val = get_plt(mod, location, val, &ok);
656 /* FALL THROUGH */
657 default:
658 val -= bundle(location);
659 break;
660
661 case R_IA64_PCREL32MSB:
662 case R_IA64_PCREL32LSB:
663 case R_IA64_PCREL64MSB:
664 case R_IA64_PCREL64LSB:
665 val -= (uint64_t) location;
666 break;
667
668 }
669 switch (r_type) {
670 case R_IA64_PCREL60B: format = RF_INSN60; break;
671 case R_IA64_PCREL21B: format = RF_INSN21B; break;
672 case R_IA64_PCREL21M: format = RF_INSN21M; break;
673 case R_IA64_PCREL21F: format = RF_INSN21F; break;
674 default: break;
675 }
676 break;
677
678 case RV_BDREL:
679 val -= (uint64_t) (in_init(mod, val) ? mod->init_layout.base : mod->core_layout.base);
680 break;
681
682 case RV_LTV:
683 /* can link-time value relocs happen here? */
684 BUG();
685 break;
686
687 case RV_PCREL2:
688 if (r_type == R_IA64_PCREL21BI) {
689 if (!is_internal(mod, val)) {
690 printk(KERN_ERR "%s: %s reloc against "
691 "non-local symbol (%lx)\n", __func__,
692 reloc_name[r_type], (unsigned long)val);
693 return -ENOEXEC;
694 }
695 format = RF_INSN21B;
696 }
697 val -= bundle(location);
698 break;
699
700 case RV_SPECIAL:
701 switch (r_type) {
702 case R_IA64_IPLTMSB:
703 case R_IA64_IPLTLSB:
704 val = get_fdesc(mod, get_plt(mod, location, val, &ok), &ok);
705 format = RF_64LSB;
706 if (r_type == R_IA64_IPLTMSB)
707 format = RF_64MSB;
708 break;
709
710 case R_IA64_SUB:
711 val = addend - sym->st_value;
712 format = RF_INSN64;
713 break;
714
715 case R_IA64_LTOFF22X:
716 if (gp_addressable(mod, val))
717 val -= mod->arch.gp;
718 else
719 val = get_ltoff(mod, val, &ok);
720 format = RF_INSN22;
721 break;
722
723 case R_IA64_LDXMOV:
724 if (gp_addressable(mod, val)) {
725 /* turn "ld8" into "mov": */
726 DEBUGP("%s: patching ld8 at %p to mov\n", __func__, location);
727 ia64_patch((u64) location, 0x1fff80fe000UL, 0x10000000000UL);
728 }
729 return 0;
730
731 default:
732 if (reloc_name[r_type])
733 printk(KERN_ERR "%s: special reloc %s not supported",
734 mod->name, reloc_name[r_type]);
735 else
736 printk(KERN_ERR "%s: unknown special reloc %x\n",
737 mod->name, r_type);
738 return -ENOEXEC;
739 }
740 break;
741
742 case RV_TPREL:
743 case RV_LTREL_TPREL:
744 case RV_DTPMOD:
745 case RV_LTREL_DTPMOD:
746 case RV_DTPREL:
747 case RV_LTREL_DTPREL:
748 printk(KERN_ERR "%s: %s reloc not supported\n",
749 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?");
750 return -ENOEXEC;
751
752 default:
753 printk(KERN_ERR "%s: unknown reloc %x\n", mod->name, r_type);
754 return -ENOEXEC;
755 }
756
757 if (!ok)
758 return -ENOEXEC;
759
760 DEBUGP("%s: [%p]<-%016lx = %s(%lx)\n", __func__, location, val,
761 reloc_name[r_type] ? reloc_name[r_type] : "?", sym->st_value + addend);
762
763 switch (format) {
764 case RF_INSN21B: ok = apply_imm21b(mod, location, (int64_t) val / 16); break;
765 case RF_INSN22: ok = apply_imm22(mod, location, val); break;
766 case RF_INSN64: ok = apply_imm64(mod, location, val); break;
767 case RF_INSN60: ok = apply_imm60(mod, location, (int64_t) val / 16); break;
768 case RF_32LSB: put_unaligned(val, (uint32_t *) location); break;
769 case RF_64LSB: put_unaligned(val, (uint64_t *) location); break;
770 case RF_32MSB: /* ia64 Linux is little-endian... */
771 case RF_64MSB: /* ia64 Linux is little-endian... */
772 case RF_INSN14: /* must be within-module, i.e., resolved by "ld -r" */
773 case RF_INSN21M: /* must be within-module, i.e., resolved by "ld -r" */
774 case RF_INSN21F: /* must be within-module, i.e., resolved by "ld -r" */
775 printk(KERN_ERR "%s: format %u needed by %s reloc is not supported\n",
776 mod->name, format, reloc_name[r_type] ? reloc_name[r_type] : "?");
777 return -ENOEXEC;
778
779 default:
780 printk(KERN_ERR "%s: relocation %s resulted in unknown format %u\n",
781 mod->name, reloc_name[r_type] ? reloc_name[r_type] : "?", format);
782 return -ENOEXEC;
783 }
784 return ok ? 0 : -ENOEXEC;
785 }
786
787 int
788 apply_relocate_add (Elf64_Shdr *sechdrs, const char *strtab, unsigned int symindex,
789 unsigned int relsec, struct module *mod)
790 {
791 unsigned int i, n = sechdrs[relsec].sh_size / sizeof(Elf64_Rela);
792 Elf64_Rela *rela = (void *) sechdrs[relsec].sh_addr;
793 Elf64_Shdr *target_sec;
794 int ret;
795
796 DEBUGP("%s: applying section %u (%u relocs) to %u\n", __func__,
797 relsec, n, sechdrs[relsec].sh_info);
798
799 target_sec = sechdrs + sechdrs[relsec].sh_info;
800
801 if (target_sec->sh_entsize == ~0UL)
802 /*
803 * If target section wasn't allocated, we don't need to relocate it.
804 * Happens, e.g., for debug sections.
805 */
806 return 0;
807
808 if (!mod->arch.gp) {
809 /*
810 * XXX Should have an arch-hook for running this after final section
811 * addresses have been selected...
812 */
813 uint64_t gp;
814 if (mod->core_layout.size > MAX_LTOFF)
815 /*
816 * This takes advantage of fact that SHF_ARCH_SMALL gets allocated
817 * at the end of the module.
818 */
819 gp = mod->core_layout.size - MAX_LTOFF / 2;
820 else
821 gp = mod->core_layout.size / 2;
822 gp = (uint64_t) mod->core_layout.base + ((gp + 7) & -8);
823 mod->arch.gp = gp;
824 DEBUGP("%s: placing gp at 0x%lx\n", __func__, gp);
825 }
826
827 for (i = 0; i < n; i++) {
828 ret = do_reloc(mod, ELF64_R_TYPE(rela[i].r_info),
829 ((Elf64_Sym *) sechdrs[symindex].sh_addr
830 + ELF64_R_SYM(rela[i].r_info)),
831 rela[i].r_addend, target_sec,
832 (void *) target_sec->sh_addr + rela[i].r_offset);
833 if (ret < 0)
834 return ret;
835 }
836 return 0;
837 }
838
839 /*
840 * Modules contain a single unwind table which covers both the core and the init text
841 * sections but since the two are not contiguous, we need to split this table up such that
842 * we can register (and unregister) each "segment" separately. Fortunately, this sounds
843 * more complicated than it really is.
844 */
845 static void
846 register_unwind_table (struct module *mod)
847 {
848 struct unw_table_entry *start = (void *) mod->arch.unwind->sh_addr;
849 struct unw_table_entry *end = start + mod->arch.unwind->sh_size / sizeof (*start);
850 struct unw_table_entry tmp, *e1, *e2, *core, *init;
851 unsigned long num_init = 0, num_core = 0;
852
853 /* First, count how many init and core unwind-table entries there are. */
854 for (e1 = start; e1 < end; ++e1)
855 if (in_init(mod, e1->start_offset))
856 ++num_init;
857 else
858 ++num_core;
859 /*
860 * Second, sort the table such that all unwind-table entries for the init and core
861 * text sections are nicely separated. We do this with a stupid bubble sort
862 * (unwind tables don't get ridiculously huge).
863 */
864 for (e1 = start; e1 < end; ++e1) {
865 for (e2 = e1 + 1; e2 < end; ++e2) {
866 if (e2->start_offset < e1->start_offset) {
867 tmp = *e1;
868 *e1 = *e2;
869 *e2 = tmp;
870 }
871 }
872 }
873 /*
874 * Third, locate the init and core segments in the unwind table:
875 */
876 if (in_init(mod, start->start_offset)) {
877 init = start;
878 core = start + num_init;
879 } else {
880 core = start;
881 init = start + num_core;
882 }
883
884 DEBUGP("%s: name=%s, gp=%lx, num_init=%lu, num_core=%lu\n", __func__,
885 mod->name, mod->arch.gp, num_init, num_core);
886
887 /*
888 * Fourth, register both tables (if not empty).
889 */
890 if (num_core > 0) {
891 mod->arch.core_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
892 core, core + num_core);
893 DEBUGP("%s: core: handle=%p [%p-%p)\n", __func__,
894 mod->arch.core_unw_table, core, core + num_core);
895 }
896 if (num_init > 0) {
897 mod->arch.init_unw_table = unw_add_unwind_table(mod->name, 0, mod->arch.gp,
898 init, init + num_init);
899 DEBUGP("%s: init: handle=%p [%p-%p)\n", __func__,
900 mod->arch.init_unw_table, init, init + num_init);
901 }
902 }
903
904 int
905 module_finalize (const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *mod)
906 {
907 DEBUGP("%s: init: entry=%p\n", __func__, mod->init);
908 if (mod->arch.unwind)
909 register_unwind_table(mod);
910 return 0;
911 }
912
913 void
914 module_arch_cleanup (struct module *mod)
915 {
916 if (mod->arch.init_unw_table) {
917 unw_remove_unwind_table(mod->arch.init_unw_table);
918 mod->arch.init_unw_table = NULL;
919 }
920 if (mod->arch.core_unw_table) {
921 unw_remove_unwind_table(mod->arch.core_unw_table);
922 mod->arch.core_unw_table = NULL;
923 }
924 }
Linux with added FPC-III support