arch/ia64/kernel/paravirt_patch.c

   1 /******************************************************************************
   2  * linux/arch/ia64/xen/paravirt_patch.c
   3  *
   4  * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
   5  *                    VA Linux Systems Japan K.K.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  *
  21  */
  22
  23 #include <linux/init.h>
  24 #include <asm/intrinsics.h>
  25 #include <asm/kprobes.h>
  26 #include <asm/paravirt.h>
  27 #include <asm/paravirt_patch.h>
  28
  29 typedef union ia64_inst {
  30         struct {
  31                 unsigned long long qp : 6;
  32                 unsigned long long : 31;
  33                 unsigned long long opcode : 4;
  34                 unsigned long long reserved : 23;
  35         } generic;
  36         unsigned long long l;
  37 } ia64_inst_t;
  38
  39 /*
  40  * flush_icache_range() can't be used here.
  41  * we are here before cpu_init() which initializes
  42  * ia64_i_cache_stride_shift. flush_icache_range() uses it.
  43  */
  44 void __init_or_module
  45 paravirt_flush_i_cache_range(const void *instr, unsigned long size)
  46 {
  47         extern void paravirt_fc_i(const void *addr);
  48         unsigned long i;
  49
  50         for (i = 0; i < size; i += sizeof(bundle_t))
  51                 paravirt_fc_i(instr + i);
  52 }
  53
  54 bundle_t* __init_or_module
  55 paravirt_get_bundle(unsigned long tag)
  56 {
  57         return (bundle_t *)(tag & ~3UL);
  58 }
  59
  60 unsigned long __init_or_module
  61 paravirt_get_slot(unsigned long tag)
  62 {
  63         return tag & 3UL;
  64 }
  65
  66 unsigned long __init_or_module
  67 paravirt_get_num_inst(unsigned long stag, unsigned long etag)
  68 {
  69         bundle_t *sbundle = paravirt_get_bundle(stag);
  70         unsigned long sslot = paravirt_get_slot(stag);
  71         bundle_t *ebundle = paravirt_get_bundle(etag);
  72         unsigned long eslot = paravirt_get_slot(etag);
  73
  74         return (ebundle - sbundle) * 3 + eslot - sslot + 1;
  75 }
  76
  77 unsigned long __init_or_module
  78 paravirt_get_next_tag(unsigned long tag)
  79 {
  80         unsigned long slot = paravirt_get_slot(tag);
  81
  82         switch (slot) {
  83         case 0:
  84         case 1:
  85                 return tag + 1;
  86         case 2: {
  87                 bundle_t *bundle = paravirt_get_bundle(tag);
  88                 return (unsigned long)(bundle + 1);
  89         }
  90         default:
  91                 BUG();
  92         }
  93         /* NOTREACHED */
  94 }
  95
  96 ia64_inst_t __init_or_module
  97 paravirt_read_slot0(const bundle_t *bundle)
  98 {
  99         ia64_inst_t inst;
 100         inst.l = bundle->quad0.slot0;
 101         return inst;
 102 }
 103
 104 ia64_inst_t __init_or_module
 105 paravirt_read_slot1(const bundle_t *bundle)
 106 {
 107         ia64_inst_t inst;
 108         inst.l = bundle->quad0.slot1_p0 |
 109                 ((unsigned long long)bundle->quad1.slot1_p1 << 18UL);
 110         return inst;
 111 }
 112
 113 ia64_inst_t __init_or_module
 114 paravirt_read_slot2(const bundle_t *bundle)
 115 {
 116         ia64_inst_t inst;
 117         inst.l = bundle->quad1.slot2;
 118         return inst;
 119 }
 120
 121 ia64_inst_t __init_or_module
 122 paravirt_read_inst(unsigned long tag)
 123 {
 124         bundle_t *bundle = paravirt_get_bundle(tag);
 125         unsigned long slot = paravirt_get_slot(tag);
 126
 127         switch (slot) {
 128         case 0:
 129                 return paravirt_read_slot0(bundle);
 130         case 1:
 131                 return paravirt_read_slot1(bundle);
 132         case 2:
 133                 return paravirt_read_slot2(bundle);
 134         default:
 135                 BUG();
 136         }
 137         /* NOTREACHED */
 138 }
 139
 140 void __init_or_module
 141 paravirt_write_slot0(bundle_t *bundle, ia64_inst_t inst)
 142 {
 143         bundle->quad0.slot0 = inst.l;
 144 }
 145
 146 void __init_or_module
 147 paravirt_write_slot1(bundle_t *bundle, ia64_inst_t inst)
 148 {
 149         bundle->quad0.slot1_p0 = inst.l;
 150         bundle->quad1.slot1_p1 = inst.l >> 18UL;
 151 }
 152
 153 void __init_or_module
 154 paravirt_write_slot2(bundle_t *bundle, ia64_inst_t inst)
 155 {
 156         bundle->quad1.slot2 = inst.l;
 157 }
 158
 159 void __init_or_module
 160 paravirt_write_inst(unsigned long tag, ia64_inst_t inst)
 161 {
 162         bundle_t *bundle = paravirt_get_bundle(tag);
 163         unsigned long slot = paravirt_get_slot(tag);
 164
 165         switch (slot) {
 166         case 0:
 167                 paravirt_write_slot0(bundle, inst);
 168                 break;
 169         case 1:
 170                 paravirt_write_slot1(bundle, inst);
 171                 break;
 172         case 2:
 173                 paravirt_write_slot2(bundle, inst);
 174                 break;
 175         default:
 176                 BUG();
 177                 break;
 178         }
 179         paravirt_flush_i_cache_range(bundle, sizeof(*bundle));
 180 }
 181
 182 /* for debug */
 183 void
 184 paravirt_print_bundle(const bundle_t *bundle)
 185 {
 186         const unsigned long *quad = (const unsigned long *)bundle;
 187         ia64_inst_t slot0 = paravirt_read_slot0(bundle);
 188         ia64_inst_t slot1 = paravirt_read_slot1(bundle);
 189         ia64_inst_t slot2 = paravirt_read_slot2(bundle);
 190
 191         printk(KERN_DEBUG
 192                "bundle 0x%p 0x%016lx 0x%016lx\n", bundle, quad[0], quad[1]);
 193         printk(KERN_DEBUG
 194                "bundle template 0x%x\n",
 195                bundle->quad0.template);
 196         printk(KERN_DEBUG
 197                "slot0 0x%lx slot1_p0 0x%lx slot1_p1 0x%lx slot2 0x%lx\n",
 198                (unsigned long)bundle->quad0.slot0,
 199                (unsigned long)bundle->quad0.slot1_p0,
 200                (unsigned long)bundle->quad1.slot1_p1,
 201                (unsigned long)bundle->quad1.slot2);
 202         printk(KERN_DEBUG
 203                "slot0 0x%016llx slot1 0x%016llx slot2 0x%016llx\n",
 204                slot0.l, slot1.l, slot2.l);
 205 }
 206
 207 static int noreplace_paravirt __init_or_module = 0;
 208
 209 static int __init setup_noreplace_paravirt(char *str)
 210 {
 211         noreplace_paravirt = 1;
 212         return 1;
 213 }
 214 __setup("noreplace-paravirt", setup_noreplace_paravirt);
 215
 216 #ifdef ASM_SUPPORTED
 217 static void __init_or_module
 218 fill_nop_bundle(void *sbundle, void *ebundle)
 219 {
 220         extern const char paravirt_nop_bundle[];
 221         extern const unsigned long paravirt_nop_bundle_size;
 222
 223         void *bundle = sbundle;
 224
 225         BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0);
 226         BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0);
 227
 228         while (bundle < ebundle) {
 229                 memcpy(bundle, paravirt_nop_bundle, paravirt_nop_bundle_size);
 230
 231                 bundle += paravirt_nop_bundle_size;
 232         }
 233 }
 234
 235 /* helper function */
 236 unsigned long __init_or_module
 237 __paravirt_patch_apply_bundle(void *sbundle, void *ebundle, unsigned long type,
 238                               const struct paravirt_patch_bundle_elem *elems,
 239                               unsigned long nelems,
 240                               const struct paravirt_patch_bundle_elem **found)
 241 {
 242         unsigned long used = 0;
 243         unsigned long i;
 244
 245         BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0);
 246         BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0);
 247
 248         found = NULL;
 249         for (i = 0; i < nelems; i++) {
 250                 const struct paravirt_patch_bundle_elem *p = &elems[i];
 251                 if (p->type == type) {
 252                         unsigned long need = p->ebundle - p->sbundle;
 253                         unsigned long room = ebundle - sbundle;
 254
 255                         if (found != NULL)
 256                                 *found = p;
 257
 258                         if (room < need) {
 259                                 /* no room to replace. skip it */
 260                                 printk(KERN_DEBUG
 261                                        "the space is too small to put "
 262                                        "bundles. type %ld need %ld room %ld\n",
 263                                        type, need, room);
 264                                 break;
 265                         }
 266
 267                         used = need;
 268                         memcpy(sbundle, p->sbundle, used);
 269                         break;
 270                 }
 271         }
 272
 273         return used;
 274 }
 275
 276 void __init_or_module
 277 paravirt_patch_apply_bundle(const struct paravirt_patch_site_bundle *start,
 278                             const struct paravirt_patch_site_bundle *end)
 279 {
 280         const struct paravirt_patch_site_bundle *p;
 281
 282         if (noreplace_paravirt)
 283                 return;
 284         if (pv_init_ops.patch_bundle == NULL)
 285                 return;
 286
 287         for (p = start; p < end; p++) {
 288                 unsigned long used;
 289
 290                 used = (*pv_init_ops.patch_bundle)(p->sbundle, p->ebundle,
 291                                                    p->type);
 292                 if (used == 0)
 293                         continue;
 294
 295                 fill_nop_bundle(p->sbundle + used, p->ebundle);
 296                 paravirt_flush_i_cache_range(p->sbundle,
 297                                              p->ebundle - p->sbundle);
 298         }
 299         ia64_sync_i();
 300         ia64_srlz_i();
 301 }
 302
 303 /*
 304  * nop.i, nop.m, nop.f instruction are same format.
 305  * but nop.b has differennt format.
 306  * This doesn't support nop.b for now.
 307  */
 308 static void __init_or_module
 309 fill_nop_inst(unsigned long stag, unsigned long etag)
 310 {
 311         extern const bundle_t paravirt_nop_mfi_inst_bundle[];
 312         unsigned long tag;
 313         const ia64_inst_t nop_inst =
 314                 paravirt_read_slot0(paravirt_nop_mfi_inst_bundle);
 315
 316         for (tag = stag; tag < etag; tag = paravirt_get_next_tag(tag))
 317                 paravirt_write_inst(tag, nop_inst);
 318 }
 319
 320 void __init_or_module
 321 paravirt_patch_apply_inst(const struct paravirt_patch_site_inst *start,
 322                           const struct paravirt_patch_site_inst *end)
 323 {
 324         const struct paravirt_patch_site_inst *p;
 325
 326         if (noreplace_paravirt)
 327                 return;
 328         if (pv_init_ops.patch_inst == NULL)
 329                 return;
 330
 331         for (p = start; p < end; p++) {
 332                 unsigned long tag;
 333                 bundle_t *sbundle;
 334                 bundle_t *ebundle;
 335
 336                 tag = (*pv_init_ops.patch_inst)(p->stag, p->etag, p->type);
 337                 if (tag == p->stag)
 338                         continue;
 339
 340                 fill_nop_inst(tag, p->etag);
 341                 sbundle = paravirt_get_bundle(p->stag);
 342                 ebundle = paravirt_get_bundle(p->etag) + 1;
 343                 paravirt_flush_i_cache_range(sbundle, (ebundle - sbundle) *
 344                                              sizeof(bundle_t));
 345         }
 346         ia64_sync_i();
 347         ia64_srlz_i();
 348 }
 349 #endif /* ASM_SUPPOTED */
 350
 351 /* brl.cond.sptk.many <target64> X3 */
 352 typedef union inst_x3_op {
 353         ia64_inst_t inst;
 354         struct {
 355                 unsigned long qp: 6;
 356                 unsigned long btyp: 3;
 357                 unsigned long unused: 3;
 358                 unsigned long p: 1;
 359                 unsigned long imm20b: 20;
 360                 unsigned long wh: 2;
 361                 unsigned long d: 1;
 362                 unsigned long i: 1;
 363                 unsigned long opcode: 4;
 364         };
 365         unsigned long l;
 366 } inst_x3_op_t;
 367
 368 typedef union inst_x3_imm {
 369         ia64_inst_t inst;
 370         struct {
 371                 unsigned long unused: 2;
 372                 unsigned long imm39: 39;
 373         };
 374         unsigned long l;
 375 } inst_x3_imm_t;
 376
 377 void __init_or_module
 378 paravirt_patch_reloc_brl(unsigned long tag, const void *target)
 379 {
 380         unsigned long tag_op = paravirt_get_next_tag(tag);
 381         unsigned long tag_imm = tag;
 382         bundle_t *bundle = paravirt_get_bundle(tag);
 383
 384         ia64_inst_t inst_op = paravirt_read_inst(tag_op);
 385         ia64_inst_t inst_imm = paravirt_read_inst(tag_imm);
 386
 387         inst_x3_op_t inst_x3_op = { .l = inst_op.l };
 388         inst_x3_imm_t inst_x3_imm = { .l = inst_imm.l };
 389
 390         unsigned long imm60 =
 391                 ((unsigned long)target - (unsigned long)bundle) >> 4;
 392
 393         BUG_ON(paravirt_get_slot(tag) != 1); /* MLX */
 394         BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0);
 395
 396         /* imm60[59] 1bit */
 397         inst_x3_op.i = (imm60 >> 59) & 1;
 398         /* imm60[19:0] 20bit */
 399         inst_x3_op.imm20b = imm60 & ((1UL << 20) - 1);
 400         /* imm60[58:20] 39bit */
 401         inst_x3_imm.imm39 = (imm60 >> 20) & ((1UL << 39) - 1);
 402
 403         inst_op.l = inst_x3_op.l;
 404         inst_imm.l = inst_x3_imm.l;
 405
 406         paravirt_write_inst(tag_op, inst_op);
 407         paravirt_write_inst(tag_imm, inst_imm);
 408 }
 409
 410 /* br.cond.sptk.many <target25> B1 */
 411 typedef union inst_b1 {
 412         ia64_inst_t inst;
 413         struct {
 414                 unsigned long qp: 6;
 415                 unsigned long btype: 3;
 416                 unsigned long unused: 3;
 417                 unsigned long p: 1;
 418                 unsigned long imm20b: 20;
 419                 unsigned long wh: 2;
 420                 unsigned long d: 1;
 421                 unsigned long s: 1;
 422                 unsigned long opcode: 4;
 423         };
 424         unsigned long l;
 425 } inst_b1_t;
 426
 427 void __init
 428 paravirt_patch_reloc_br(unsigned long tag, const void *target)
 429 {
 430         bundle_t *bundle = paravirt_get_bundle(tag);
 431         ia64_inst_t inst = paravirt_read_inst(tag);
 432         unsigned long target25 = (unsigned long)target - (unsigned long)bundle;
 433         inst_b1_t inst_b1;
 434
 435         BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0);
 436
 437         inst_b1.l = inst.l;
 438         if (target25 & (1UL << 63))
 439                 inst_b1.s = 1;
 440         else
 441                 inst_b1.s = 0;
 442
 443         inst_b1.imm20b = target25 >> 4;
 444         inst.l = inst_b1.l;
 445
 446         paravirt_write_inst(tag, inst);
 447 }
 448
 449 void __init
 450 __paravirt_patch_apply_branch(
 451         unsigned long tag, unsigned long type,
 452         const struct paravirt_patch_branch_target *entries,
 453         unsigned int nr_entries)
 454 {
 455         unsigned int i;
 456         for (i = 0; i < nr_entries; i++) {
 457                 if (entries[i].type == type) {
 458                         paravirt_patch_reloc_br(tag, entries[i].entry);
 459                         break;
 460                 }
 461         }
 462 }
 463
 464 static void __init
 465 paravirt_patch_apply_branch(const struct paravirt_patch_site_branch *start,
 466                             const struct paravirt_patch_site_branch *end)
 467 {
 468         const struct paravirt_patch_site_branch *p;
 469
 470         if (noreplace_paravirt)
 471                 return;
 472         if (pv_init_ops.patch_branch == NULL)
 473                 return;
 474
 475         for (p = start; p < end; p++)
 476                 (*pv_init_ops.patch_branch)(p->tag, p->type);
 477
 478         ia64_sync_i();
 479         ia64_srlz_i();
 480 }
 481
 482 void __init
 483 paravirt_patch_apply(void)
 484 {
 485         extern const char __start_paravirt_bundles[];
 486         extern const char __stop_paravirt_bundles[];
 487         extern const char __start_paravirt_insts[];
 488         extern const char __stop_paravirt_insts[];
 489         extern const char __start_paravirt_branches[];
 490         extern const char __stop_paravirt_branches[];
 491
 492         paravirt_patch_apply_bundle((const struct paravirt_patch_site_bundle *)
 493                                     __start_paravirt_bundles,
 494                                     (const struct paravirt_patch_site_bundle *)
 495                                     __stop_paravirt_bundles);
 496         paravirt_patch_apply_inst((const struct paravirt_patch_site_inst *)
 497                                   __start_paravirt_insts,
 498                                   (const struct paravirt_patch_site_inst *)
 499                                   __stop_paravirt_insts);
 500         paravirt_patch_apply_branch((const struct paravirt_patch_site_branch *)
 501                                     __start_paravirt_branches,
 502                                     (const struct paravirt_patch_site_branch *)
 503                                     __stop_paravirt_branches);
 504 }
 505
 506 /*
 507  * Local variables:
 508  * mode: C
 509  * c-set-style: "linux"
 510  * c-basic-offset: 8
 511  * tab-width: 8
 512  * indent-tabs-mode: t
 513  * End:
 514  */