arch/ia64/kernel/paravirt.c

   1 /******************************************************************************
   2  * arch/ia64/kernel/paravirt.c
   3  *
   4  * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
   5  *                    VA Linux Systems Japan K.K.
   6  *     Yaozu (Eddie) Dong <eddie.dong@intel.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21  *
  22  */
  23
  24 #include <linux/init.h>
  25
  26 #include <linux/compiler.h>
  27 #include <linux/io.h>
  28 #include <linux/irq.h>
  29 #include <linux/module.h>
  30 #include <linux/types.h>
  31
  32 #include <asm/iosapic.h>
  33 #include <asm/paravirt.h>
  34
  35 /***************************************************************************
  36  * general info
  37  */
  38 struct pv_info pv_info = {
  39         .kernel_rpl = 0,
  40         .paravirt_enabled = 0,
  41         .name = "bare hardware"
  42 };
  43
  44 /***************************************************************************
  45  * pv_init_ops
  46  * initialization hooks.
  47  */
  48
  49 static void __init
  50 ia64_native_patch_branch(unsigned long tag, unsigned long type);
  51
  52 struct pv_init_ops pv_init_ops =
  53 {
  54 #ifdef ASM_SUPPORTED
  55         .patch_bundle = ia64_native_patch_bundle,
  56 #endif
  57         .patch_branch = ia64_native_patch_branch,
  58 };
  59
  60 /***************************************************************************
  61  * pv_cpu_ops
  62  * intrinsics hooks.
  63  */
  64
  65 #ifndef ASM_SUPPORTED
  66 /* ia64_native_xxx are macros so that we have to make them real functions */
  67
  68 #define DEFINE_VOID_FUNC1(name)                                 \
  69         static void                                             \
  70         ia64_native_ ## name ## _func(unsigned long arg)        \
  71         {                                                       \
  72                 ia64_native_ ## name(arg);                      \
  73         }
  74
  75 #define DEFINE_VOID_FUNC1_VOID(name)                            \
  76         static void                                             \
  77         ia64_native_ ## name ## _func(void *arg)                \
  78         {                                                       \
  79                 ia64_native_ ## name(arg);                      \
  80         }
  81
  82 #define DEFINE_VOID_FUNC2(name)                                 \
  83         static void                                             \
  84         ia64_native_ ## name ## _func(unsigned long arg0,       \
  85                                       unsigned long arg1)       \
  86         {                                                       \
  87                 ia64_native_ ## name(arg0, arg1);               \
  88         }
  89
  90 #define DEFINE_FUNC0(name)                      \
  91         static unsigned long                    \
  92         ia64_native_ ## name ## _func(void)     \
  93         {                                       \
  94                 return ia64_native_ ## name();  \
  95         }
  96
  97 #define DEFINE_FUNC1(name, type)                        \
  98         static unsigned long                            \
  99         ia64_native_ ## name ## _func(type arg)         \
 100         {                                               \
 101                 return ia64_native_ ## name(arg);       \
 102         }                                               \
 103
 104 DEFINE_VOID_FUNC1_VOID(fc);
 105 DEFINE_VOID_FUNC1(intrin_local_irq_restore);
 106
 107 DEFINE_VOID_FUNC2(ptcga);
 108 DEFINE_VOID_FUNC2(set_rr);
 109
 110 DEFINE_FUNC0(get_psr_i);
 111
 112 DEFINE_FUNC1(thash, unsigned long);
 113 DEFINE_FUNC1(get_cpuid, int);
 114 DEFINE_FUNC1(get_pmd, int);
 115 DEFINE_FUNC1(get_rr, unsigned long);
 116
 117 static void
 118 ia64_native_ssm_i_func(void)
 119 {
 120         ia64_native_ssm(IA64_PSR_I);
 121 }
 122
 123 static void
 124 ia64_native_rsm_i_func(void)
 125 {
 126         ia64_native_rsm(IA64_PSR_I);
 127 }
 128
 129 static void
 130 ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
 131                                 unsigned long val2, unsigned long val3,
 132                                 unsigned long val4)
 133 {
 134         ia64_native_set_rr0_to_rr4(val0, val1, val2, val3, val4);
 135 }
 136
 137 #define CASE_GET_REG(id)                                \
 138         case _IA64_REG_ ## id:                          \
 139         res = ia64_native_getreg(_IA64_REG_ ## id);     \
 140         break;
 141 #define CASE_GET_AR(id) CASE_GET_REG(AR_ ## id)
 142 #define CASE_GET_CR(id) CASE_GET_REG(CR_ ## id)
 143
 144 unsigned long
 145 ia64_native_getreg_func(int regnum)
 146 {
 147         unsigned long res = -1;
 148         switch (regnum) {
 149         CASE_GET_REG(GP);
 150         /*CASE_GET_REG(IP);*/ /* returned ip value shouldn't be constant */
 151         CASE_GET_REG(PSR);
 152         CASE_GET_REG(TP);
 153         CASE_GET_REG(SP);
 154
 155         CASE_GET_AR(KR0);
 156         CASE_GET_AR(KR1);
 157         CASE_GET_AR(KR2);
 158         CASE_GET_AR(KR3);
 159         CASE_GET_AR(KR4);
 160         CASE_GET_AR(KR5);
 161         CASE_GET_AR(KR6);
 162         CASE_GET_AR(KR7);
 163         CASE_GET_AR(RSC);
 164         CASE_GET_AR(BSP);
 165         CASE_GET_AR(BSPSTORE);
 166         CASE_GET_AR(RNAT);
 167         CASE_GET_AR(FCR);
 168         CASE_GET_AR(EFLAG);
 169         CASE_GET_AR(CSD);
 170         CASE_GET_AR(SSD);
 171         CASE_GET_AR(CFLAG);
 172         CASE_GET_AR(FSR);
 173         CASE_GET_AR(FIR);
 174         CASE_GET_AR(FDR);
 175         CASE_GET_AR(CCV);
 176         CASE_GET_AR(UNAT);
 177         CASE_GET_AR(FPSR);
 178         CASE_GET_AR(ITC);
 179         CASE_GET_AR(PFS);
 180         CASE_GET_AR(LC);
 181         CASE_GET_AR(EC);
 182
 183         CASE_GET_CR(DCR);
 184         CASE_GET_CR(ITM);
 185         CASE_GET_CR(IVA);
 186         CASE_GET_CR(PTA);
 187         CASE_GET_CR(IPSR);
 188         CASE_GET_CR(ISR);
 189         CASE_GET_CR(IIP);
 190         CASE_GET_CR(IFA);
 191         CASE_GET_CR(ITIR);
 192         CASE_GET_CR(IIPA);
 193         CASE_GET_CR(IFS);
 194         CASE_GET_CR(IIM);
 195         CASE_GET_CR(IHA);
 196         CASE_GET_CR(LID);
 197         CASE_GET_CR(IVR);
 198         CASE_GET_CR(TPR);
 199         CASE_GET_CR(EOI);
 200         CASE_GET_CR(IRR0);
 201         CASE_GET_CR(IRR1);
 202         CASE_GET_CR(IRR2);
 203         CASE_GET_CR(IRR3);
 204         CASE_GET_CR(ITV);
 205         CASE_GET_CR(PMV);
 206         CASE_GET_CR(CMCV);
 207         CASE_GET_CR(LRR0);
 208         CASE_GET_CR(LRR1);
 209
 210         default:
 211                 printk(KERN_CRIT "wrong_getreg %d\n", regnum);
 212                 break;
 213         }
 214         return res;
 215 }
 216
 217 #define CASE_SET_REG(id)                                \
 218         case _IA64_REG_ ## id:                          \
 219         ia64_native_setreg(_IA64_REG_ ## id, val);      \
 220         break;
 221 #define CASE_SET_AR(id) CASE_SET_REG(AR_ ## id)
 222 #define CASE_SET_CR(id) CASE_SET_REG(CR_ ## id)
 223
 224 void
 225 ia64_native_setreg_func(int regnum, unsigned long val)
 226 {
 227         switch (regnum) {
 228         case _IA64_REG_PSR_L:
 229                 ia64_native_setreg(_IA64_REG_PSR_L, val);
 230                 ia64_dv_serialize_data();
 231                 break;
 232         CASE_SET_REG(SP);
 233         CASE_SET_REG(GP);
 234
 235         CASE_SET_AR(KR0);
 236         CASE_SET_AR(KR1);
 237         CASE_SET_AR(KR2);
 238         CASE_SET_AR(KR3);
 239         CASE_SET_AR(KR4);
 240         CASE_SET_AR(KR5);
 241         CASE_SET_AR(KR6);
 242         CASE_SET_AR(KR7);
 243         CASE_SET_AR(RSC);
 244         CASE_SET_AR(BSP);
 245         CASE_SET_AR(BSPSTORE);
 246         CASE_SET_AR(RNAT);
 247         CASE_SET_AR(FCR);
 248         CASE_SET_AR(EFLAG);
 249         CASE_SET_AR(CSD);
 250         CASE_SET_AR(SSD);
 251         CASE_SET_AR(CFLAG);
 252         CASE_SET_AR(FSR);
 253         CASE_SET_AR(FIR);
 254         CASE_SET_AR(FDR);
 255         CASE_SET_AR(CCV);
 256         CASE_SET_AR(UNAT);
 257         CASE_SET_AR(FPSR);
 258         CASE_SET_AR(ITC);
 259         CASE_SET_AR(PFS);
 260         CASE_SET_AR(LC);
 261         CASE_SET_AR(EC);
 262
 263         CASE_SET_CR(DCR);
 264         CASE_SET_CR(ITM);
 265         CASE_SET_CR(IVA);
 266         CASE_SET_CR(PTA);
 267         CASE_SET_CR(IPSR);
 268         CASE_SET_CR(ISR);
 269         CASE_SET_CR(IIP);
 270         CASE_SET_CR(IFA);
 271         CASE_SET_CR(ITIR);
 272         CASE_SET_CR(IIPA);
 273         CASE_SET_CR(IFS);
 274         CASE_SET_CR(IIM);
 275         CASE_SET_CR(IHA);
 276         CASE_SET_CR(LID);
 277         CASE_SET_CR(IVR);
 278         CASE_SET_CR(TPR);
 279         CASE_SET_CR(EOI);
 280         CASE_SET_CR(IRR0);
 281         CASE_SET_CR(IRR1);
 282         CASE_SET_CR(IRR2);
 283         CASE_SET_CR(IRR3);
 284         CASE_SET_CR(ITV);
 285         CASE_SET_CR(PMV);
 286         CASE_SET_CR(CMCV);
 287         CASE_SET_CR(LRR0);
 288         CASE_SET_CR(LRR1);
 289         default:
 290                 printk(KERN_CRIT "wrong setreg %d\n", regnum);
 291                 break;
 292         }
 293 }
 294 #else
 295
 296 #define __DEFINE_FUNC(name, code)                                       \
 297         extern const char ia64_native_ ## name ## _direct_start[];      \
 298         extern const char ia64_native_ ## name ## _direct_end[];        \
 299         asm (".align 32\n"                                              \
 300              ".proc ia64_native_" #name "_func\n"                       \
 301              "ia64_native_" #name "_func:\n"                            \
 302              "ia64_native_" #name "_direct_start:\n"                    \
 303              code                                                       \
 304              "ia64_native_" #name "_direct_end:\n"                      \
 305              "br.cond.sptk.many b6\n"                                   \
 306              ".endp ia64_native_" #name "_func\n")
 307
 308 #define DEFINE_VOID_FUNC0(name, code)                           \
 309         extern void                                             \
 310         ia64_native_ ## name ## _func(void);                    \
 311         __DEFINE_FUNC(name, code)
 312
 313 #define DEFINE_VOID_FUNC1(name, code)                           \
 314         extern void                                             \
 315         ia64_native_ ## name ## _func(unsigned long arg);       \
 316         __DEFINE_FUNC(name, code)
 317
 318 #define DEFINE_VOID_FUNC1_VOID(name, code)                      \
 319         extern void                                             \
 320         ia64_native_ ## name ## _func(void *arg);               \
 321         __DEFINE_FUNC(name, code)
 322
 323 #define DEFINE_VOID_FUNC2(name, code)                           \
 324         extern void                                             \
 325         ia64_native_ ## name ## _func(unsigned long arg0,       \
 326                                       unsigned long arg1);      \
 327         __DEFINE_FUNC(name, code)
 328
 329 #define DEFINE_FUNC0(name, code)                \
 330         extern unsigned long                    \
 331         ia64_native_ ## name ## _func(void);    \
 332         __DEFINE_FUNC(name, code)
 333
 334 #define DEFINE_FUNC1(name, type, code)                  \
 335         extern unsigned long                            \
 336         ia64_native_ ## name ## _func(type arg);        \
 337         __DEFINE_FUNC(name, code)
 338
 339 DEFINE_VOID_FUNC1_VOID(fc,
 340                        "fc r8\n");
 341 DEFINE_VOID_FUNC1(intrin_local_irq_restore,
 342                   ";;\n"
 343                   "     cmp.ne p6, p7 = r8, r0\n"
 344                   ";;\n"
 345                   "(p6) ssm psr.i\n"
 346                   "(p7) rsm psr.i\n"
 347                   ";;\n"
 348                   "(p6) srlz.d\n");
 349
 350 DEFINE_VOID_FUNC2(ptcga,
 351                   "ptc.ga r8, r9\n");
 352 DEFINE_VOID_FUNC2(set_rr,
 353                   "mov rr[r8] = r9\n");
 354
 355 /* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
 356 DEFINE_FUNC0(get_psr_i,
 357              "mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
 358              "mov r8 = psr\n"
 359              ";;\n"
 360              "and r8 = r2, r8\n");
 361
 362 DEFINE_FUNC1(thash, unsigned long,
 363              "thash r8 = r8\n");
 364 DEFINE_FUNC1(get_cpuid, int,
 365              "mov r8 = cpuid[r8]\n");
 366 DEFINE_FUNC1(get_pmd, int,
 367              "mov r8 = pmd[r8]\n");
 368 DEFINE_FUNC1(get_rr, unsigned long,
 369              "mov r8 = rr[r8]\n");
 370
 371 DEFINE_VOID_FUNC0(ssm_i,
 372                   "ssm psr.i\n");
 373 DEFINE_VOID_FUNC0(rsm_i,
 374                   "rsm psr.i\n");
 375
 376 extern void
 377 ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
 378                                 unsigned long val2, unsigned long val3,
 379                                 unsigned long val4);
 380 __DEFINE_FUNC(set_rr0_to_rr4,
 381               "mov rr[r0] = r8\n"
 382               "movl r2 = 0x2000000000000000\n"
 383               ";;\n"
 384               "mov rr[r2] = r9\n"
 385               "shl r3 = r2, 1\n"        /* movl r3 = 0x4000000000000000 */
 386               ";;\n"
 387               "add r2 = r2, r3\n"       /* movl r2 = 0x6000000000000000 */
 388               "mov rr[r3] = r10\n"
 389               ";;\n"
 390               "mov rr[r2] = r11\n"
 391               "shl r3 = r3, 1\n"        /* movl r3 = 0x8000000000000000 */
 392               ";;\n"
 393               "mov rr[r3] = r14\n");
 394
 395 extern unsigned long ia64_native_getreg_func(int regnum);
 396 asm(".global ia64_native_getreg_func\n");
 397 #define __DEFINE_GET_REG(id, reg)                       \
 398         "mov r2 = " __stringify(_IA64_REG_ ## id) "\n"  \
 399         ";;\n"                                          \
 400         "cmp.eq p6, p0 = r2, r8\n"                      \
 401         ";;\n"                                          \
 402         "(p6) mov r8 = " #reg "\n"                      \
 403         "(p6) br.cond.sptk.many b6\n"                   \
 404         ";;\n"
 405 #define __DEFINE_GET_AR(id, reg)        __DEFINE_GET_REG(AR_ ## id, ar.reg)
 406 #define __DEFINE_GET_CR(id, reg)        __DEFINE_GET_REG(CR_ ## id, cr.reg)
 407
 408 __DEFINE_FUNC(getreg,
 409               __DEFINE_GET_REG(GP, gp)
 410               /*__DEFINE_GET_REG(IP, ip)*/ /* returned ip value shouldn't be constant */
 411               __DEFINE_GET_REG(PSR, psr)
 412               __DEFINE_GET_REG(TP, tp)
 413               __DEFINE_GET_REG(SP, sp)
 414
 415               __DEFINE_GET_REG(AR_KR0, ar0)
 416               __DEFINE_GET_REG(AR_KR1, ar1)
 417               __DEFINE_GET_REG(AR_KR2, ar2)
 418               __DEFINE_GET_REG(AR_KR3, ar3)
 419               __DEFINE_GET_REG(AR_KR4, ar4)
 420               __DEFINE_GET_REG(AR_KR5, ar5)
 421               __DEFINE_GET_REG(AR_KR6, ar6)
 422               __DEFINE_GET_REG(AR_KR7, ar7)
 423               __DEFINE_GET_AR(RSC, rsc)
 424               __DEFINE_GET_AR(BSP, bsp)
 425               __DEFINE_GET_AR(BSPSTORE, bspstore)
 426               __DEFINE_GET_AR(RNAT, rnat)
 427               __DEFINE_GET_AR(FCR, fcr)
 428               __DEFINE_GET_AR(EFLAG, eflag)
 429               __DEFINE_GET_AR(CSD, csd)
 430               __DEFINE_GET_AR(SSD, ssd)
 431               __DEFINE_GET_REG(AR_CFLAG, ar27)
 432               __DEFINE_GET_AR(FSR, fsr)
 433               __DEFINE_GET_AR(FIR, fir)
 434               __DEFINE_GET_AR(FDR, fdr)
 435               __DEFINE_GET_AR(CCV, ccv)
 436               __DEFINE_GET_AR(UNAT, unat)
 437               __DEFINE_GET_AR(FPSR, fpsr)
 438               __DEFINE_GET_AR(ITC, itc)
 439               __DEFINE_GET_AR(PFS, pfs)
 440               __DEFINE_GET_AR(LC, lc)
 441               __DEFINE_GET_AR(EC, ec)
 442
 443               __DEFINE_GET_CR(DCR, dcr)
 444               __DEFINE_GET_CR(ITM, itm)
 445               __DEFINE_GET_CR(IVA, iva)
 446               __DEFINE_GET_CR(PTA, pta)
 447               __DEFINE_GET_CR(IPSR, ipsr)
 448               __DEFINE_GET_CR(ISR, isr)
 449               __DEFINE_GET_CR(IIP, iip)
 450               __DEFINE_GET_CR(IFA, ifa)
 451               __DEFINE_GET_CR(ITIR, itir)
 452               __DEFINE_GET_CR(IIPA, iipa)
 453               __DEFINE_GET_CR(IFS, ifs)
 454               __DEFINE_GET_CR(IIM, iim)
 455               __DEFINE_GET_CR(IHA, iha)
 456               __DEFINE_GET_CR(LID, lid)
 457               __DEFINE_GET_CR(IVR, ivr)
 458               __DEFINE_GET_CR(TPR, tpr)
 459               __DEFINE_GET_CR(EOI, eoi)
 460               __DEFINE_GET_CR(IRR0, irr0)
 461               __DEFINE_GET_CR(IRR1, irr1)
 462               __DEFINE_GET_CR(IRR2, irr2)
 463               __DEFINE_GET_CR(IRR3, irr3)
 464               __DEFINE_GET_CR(ITV, itv)
 465               __DEFINE_GET_CR(PMV, pmv)
 466               __DEFINE_GET_CR(CMCV, cmcv)
 467               __DEFINE_GET_CR(LRR0, lrr0)
 468               __DEFINE_GET_CR(LRR1, lrr1)
 469
 470               "mov r8 = -1\n"   /* unsupported case */
 471         );
 472
 473 extern void ia64_native_setreg_func(int regnum, unsigned long val);
 474 asm(".global ia64_native_setreg_func\n");
 475 #define __DEFINE_SET_REG(id, reg)                       \
 476         "mov r2 = " __stringify(_IA64_REG_ ## id) "\n"  \
 477         ";;\n"                                          \
 478         "cmp.eq p6, p0 = r2, r9\n"                      \
 479         ";;\n"                                          \
 480         "(p6) mov " #reg " = r8\n"                      \
 481         "(p6) br.cond.sptk.many b6\n"                   \
 482         ";;\n"
 483 #define __DEFINE_SET_AR(id, reg)        __DEFINE_SET_REG(AR_ ## id, ar.reg)
 484 #define __DEFINE_SET_CR(id, reg)        __DEFINE_SET_REG(CR_ ## id, cr.reg)
 485 __DEFINE_FUNC(setreg,
 486               "mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
 487               ";;\n"
 488               "cmp.eq p6, p0 = r2, r9\n"
 489               ";;\n"
 490               "(p6) mov psr.l = r8\n"
 491 #ifdef HAVE_SERIALIZE_DIRECTIVE
 492               ".serialize.data\n"
 493 #endif
 494               "(p6) br.cond.sptk.many b6\n"
 495               __DEFINE_SET_REG(GP, gp)
 496               __DEFINE_SET_REG(SP, sp)
 497
 498               __DEFINE_SET_REG(AR_KR0, ar0)
 499               __DEFINE_SET_REG(AR_KR1, ar1)
 500               __DEFINE_SET_REG(AR_KR2, ar2)
 501               __DEFINE_SET_REG(AR_KR3, ar3)
 502               __DEFINE_SET_REG(AR_KR4, ar4)
 503               __DEFINE_SET_REG(AR_KR5, ar5)
 504               __DEFINE_SET_REG(AR_KR6, ar6)
 505               __DEFINE_SET_REG(AR_KR7, ar7)
 506               __DEFINE_SET_AR(RSC, rsc)
 507               __DEFINE_SET_AR(BSP, bsp)
 508               __DEFINE_SET_AR(BSPSTORE, bspstore)
 509               __DEFINE_SET_AR(RNAT, rnat)
 510               __DEFINE_SET_AR(FCR, fcr)
 511               __DEFINE_SET_AR(EFLAG, eflag)
 512               __DEFINE_SET_AR(CSD, csd)
 513               __DEFINE_SET_AR(SSD, ssd)
 514               __DEFINE_SET_REG(AR_CFLAG, ar27)
 515               __DEFINE_SET_AR(FSR, fsr)
 516               __DEFINE_SET_AR(FIR, fir)
 517               __DEFINE_SET_AR(FDR, fdr)
 518               __DEFINE_SET_AR(CCV, ccv)
 519               __DEFINE_SET_AR(UNAT, unat)
 520               __DEFINE_SET_AR(FPSR, fpsr)
 521               __DEFINE_SET_AR(ITC, itc)
 522               __DEFINE_SET_AR(PFS, pfs)
 523               __DEFINE_SET_AR(LC, lc)
 524               __DEFINE_SET_AR(EC, ec)
 525
 526               __DEFINE_SET_CR(DCR, dcr)
 527               __DEFINE_SET_CR(ITM, itm)
 528               __DEFINE_SET_CR(IVA, iva)
 529               __DEFINE_SET_CR(PTA, pta)
 530               __DEFINE_SET_CR(IPSR, ipsr)
 531               __DEFINE_SET_CR(ISR, isr)
 532               __DEFINE_SET_CR(IIP, iip)
 533               __DEFINE_SET_CR(IFA, ifa)
 534               __DEFINE_SET_CR(ITIR, itir)
 535               __DEFINE_SET_CR(IIPA, iipa)
 536               __DEFINE_SET_CR(IFS, ifs)
 537               __DEFINE_SET_CR(IIM, iim)
 538               __DEFINE_SET_CR(IHA, iha)
 539               __DEFINE_SET_CR(LID, lid)
 540               __DEFINE_SET_CR(IVR, ivr)
 541               __DEFINE_SET_CR(TPR, tpr)
 542               __DEFINE_SET_CR(EOI, eoi)
 543               __DEFINE_SET_CR(IRR0, irr0)
 544               __DEFINE_SET_CR(IRR1, irr1)
 545               __DEFINE_SET_CR(IRR2, irr2)
 546               __DEFINE_SET_CR(IRR3, irr3)
 547               __DEFINE_SET_CR(ITV, itv)
 548               __DEFINE_SET_CR(PMV, pmv)
 549               __DEFINE_SET_CR(CMCV, cmcv)
 550               __DEFINE_SET_CR(LRR0, lrr0)
 551               __DEFINE_SET_CR(LRR1, lrr1)
 552         );
 553 #endif
 554
 555 struct pv_cpu_ops pv_cpu_ops = {
 556         .fc             = ia64_native_fc_func,
 557         .thash          = ia64_native_thash_func,
 558         .get_cpuid      = ia64_native_get_cpuid_func,
 559         .get_pmd        = ia64_native_get_pmd_func,
 560         .ptcga          = ia64_native_ptcga_func,
 561         .get_rr         = ia64_native_get_rr_func,
 562         .set_rr         = ia64_native_set_rr_func,
 563         .set_rr0_to_rr4 = ia64_native_set_rr0_to_rr4_func,
 564         .ssm_i          = ia64_native_ssm_i_func,
 565         .getreg         = ia64_native_getreg_func,
 566         .setreg         = ia64_native_setreg_func,
 567         .rsm_i          = ia64_native_rsm_i_func,
 568         .get_psr_i      = ia64_native_get_psr_i_func,
 569         .intrin_local_irq_restore
 570                         = ia64_native_intrin_local_irq_restore_func,
 571 };
 572 EXPORT_SYMBOL(pv_cpu_ops);
 573
 574 /******************************************************************************
 575  * replacement of hand written assembly codes.
 576  */
 577
 578 void
 579 paravirt_cpu_asm_init(const struct pv_cpu_asm_switch *cpu_asm_switch)
 580 {
 581         extern unsigned long paravirt_switch_to_targ;
 582         extern unsigned long paravirt_leave_syscall_targ;
 583         extern unsigned long paravirt_work_processed_syscall_targ;
 584         extern unsigned long paravirt_leave_kernel_targ;
 585
 586         paravirt_switch_to_targ = cpu_asm_switch->switch_to;
 587         paravirt_leave_syscall_targ = cpu_asm_switch->leave_syscall;
 588         paravirt_work_processed_syscall_targ =
 589                 cpu_asm_switch->work_processed_syscall;
 590         paravirt_leave_kernel_targ = cpu_asm_switch->leave_kernel;
 591 }
 592
 593 /***************************************************************************
 594  * pv_iosapic_ops
 595  * iosapic read/write hooks.
 596  */
 597
 598 static unsigned int
 599 ia64_native_iosapic_read(char __iomem *iosapic, unsigned int reg)
 600 {
 601         return __ia64_native_iosapic_read(iosapic, reg);
 602 }
 603
 604 static void
 605 ia64_native_iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
 606 {
 607         __ia64_native_iosapic_write(iosapic, reg, val);
 608 }
 609
 610 struct pv_iosapic_ops pv_iosapic_ops = {
 611         .pcat_compat_init = ia64_native_iosapic_pcat_compat_init,
 612         .__get_irq_chip = ia64_native_iosapic_get_irq_chip,
 613
 614         .__read = ia64_native_iosapic_read,
 615         .__write = ia64_native_iosapic_write,
 616 };
 617
 618 /***************************************************************************
 619  * pv_irq_ops
 620  * irq operations
 621  */
 622
 623 struct pv_irq_ops pv_irq_ops = {
 624         .register_ipi = ia64_native_register_ipi,
 625
 626         .assign_irq_vector = ia64_native_assign_irq_vector,
 627         .free_irq_vector = ia64_native_free_irq_vector,
 628         .register_percpu_irq = ia64_native_register_percpu_irq,
 629
 630         .resend_irq = ia64_native_resend_irq,
 631 };
 632
 633 /***************************************************************************
 634  * pv_time_ops
 635  * time operations
 636  */
 637 struct static_key paravirt_steal_enabled;
 638 struct static_key paravirt_steal_rq_enabled;
 639
 640 static int
 641 ia64_native_do_steal_accounting(unsigned long *new_itm)
 642 {
 643         return 0;
 644 }
 645
 646 struct pv_time_ops pv_time_ops = {
 647         .do_steal_accounting = ia64_native_do_steal_accounting,
 648         .sched_clock = ia64_native_sched_clock,
 649 };
 650
 651 /***************************************************************************
 652  * binary pacthing
 653  * pv_init_ops.patch_bundle
 654  */
 655
 656 #ifdef ASM_SUPPORTED
 657 #define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg)     \
 658         __DEFINE_FUNC(get_ ## name,                     \
 659                       ";;\n"                            \
 660                       "mov r8 = " #reg "\n"             \
 661                       ";;\n")
 662
 663 #define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)     \
 664         __DEFINE_FUNC(set_ ## name,                     \
 665                       ";;\n"                            \
 666                       "mov " #reg " = r8\n"             \
 667                       ";;\n")
 668
 669 #define IA64_NATIVE_PATCH_DEFINE_REG(name, reg)         \
 670         IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg);    \
 671         IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg)     \
 672
 673 #define IA64_NATIVE_PATCH_DEFINE_AR(name, reg)                  \
 674         IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
 675
 676 #define IA64_NATIVE_PATCH_DEFINE_CR(name, reg)                  \
 677         IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
 678
 679
 680 IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
 681 IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
 682
 683 /* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
 684 __DEFINE_FUNC(set_psr_l,
 685               ";;\n"
 686               "mov psr.l = r8\n"
 687 #ifdef HAVE_SERIALIZE_DIRECTIVE
 688               ".serialize.data\n"
 689 #endif
 690               ";;\n");
 691
 692 IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
 693 IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
 694
 695 IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
 696 IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
 697 IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
 698 IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
 699 IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
 700 IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
 701 IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
 702 IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
 703
 704 IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
 705 IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
 706 IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
 707 IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
 708 IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
 709 IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
 710 IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
 711 IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
 712 IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
 713 IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
 714 IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
 715 IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
 716 IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
 717 IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
 718 IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
 719 IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
 720 IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
 721 IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
 722 IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
 723
 724 IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
 725 IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
 726 IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
 727 IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
 728 IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
 729 IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
 730 IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
 731 IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
 732 IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
 733 IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
 734 IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
 735 IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
 736 IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
 737 IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
 738 IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
 739 IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
 740 IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
 741 IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
 742 IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
 743 IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
 744 IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
 745 IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
 746 IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
 747 IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
 748 IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
 749 IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
 750
 751 static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
 752 __initdata_or_module =
 753 {
 754 #define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type)               \
 755         {                                                       \
 756                 (void*)ia64_native_ ## name ## _direct_start,   \
 757                 (void*)ia64_native_ ## name ## _direct_end,     \
 758                 PARAVIRT_PATCH_TYPE_ ## type,                   \
 759         }
 760
 761         IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
 762         IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
 763         IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
 764         IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
 765         IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
 766         IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
 767         IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
 768         IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
 769         IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
 770         IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
 771         IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
 772         IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
 773                                       INTRIN_LOCAL_IRQ_RESTORE),
 774
 775 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg)                 \
 776         {                                                               \
 777                 (void*)ia64_native_get_ ## name ## _direct_start,       \
 778                 (void*)ia64_native_get_ ## name ## _direct_end,         \
 779                 PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg,         \
 780         }
 781
 782 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)                 \
 783         {                                                               \
 784                 (void*)ia64_native_set_ ## name ## _direct_start,       \
 785                 (void*)ia64_native_set_ ## name ## _direct_end,         \
 786                 PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg,         \
 787         }
 788
 789 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg)            \
 790         IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg),        \
 791         IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg)         \
 792
 793 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg)             \
 794         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
 795
 796 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg)             \
 797         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
 798
 799         IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
 800         IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
 801
 802         IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
 803
 804         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
 805         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
 806
 807         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
 808         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
 809         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
 810         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
 811         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
 812         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
 813         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
 814         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
 815
 816         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
 817         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
 818         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
 819         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
 820         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
 821         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
 822         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
 823         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
 824         IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
 825         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
 826         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
 827         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
 828         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
 829         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
 830         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
 831         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
 832         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
 833         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
 834         IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
 835
 836         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
 837         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
 838         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
 839         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
 840         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
 841         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
 842         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
 843         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
 844         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
 845         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
 846         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
 847         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
 848         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
 849         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
 850         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
 851         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
 852         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
 853         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
 854         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
 855         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
 856         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
 857         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
 858         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
 859         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
 860         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
 861         IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
 862 };
 863
 864 unsigned long __init_or_module
 865 ia64_native_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
 866 {
 867         const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
 868                 sizeof(ia64_native_patch_bundle_elems[0]);
 869
 870         return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
 871                                               ia64_native_patch_bundle_elems,
 872                                               nelems, NULL);
 873 }
 874 #endif /* ASM_SUPPOTED */
 875
 876 extern const char ia64_native_switch_to[];
 877 extern const char ia64_native_leave_syscall[];
 878 extern const char ia64_native_work_processed_syscall[];
 879 extern const char ia64_native_leave_kernel[];
 880
 881 const struct paravirt_patch_branch_target ia64_native_branch_target[]
 882 __initconst = {
 883 #define PARAVIRT_BR_TARGET(name, type)                  \
 884         {                                               \
 885                 ia64_native_ ## name,                   \
 886                 PARAVIRT_PATCH_TYPE_BR_ ## type,        \
 887         }
 888         PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
 889         PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
 890         PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
 891         PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
 892 };
 893
 894 static void __init
 895 ia64_native_patch_branch(unsigned long tag, unsigned long type)
 896 {
 897         const unsigned long nelem =
 898                 sizeof(ia64_native_branch_target) /
 899                 sizeof(ia64_native_branch_target[0]);
 900         __paravirt_patch_apply_branch(tag, type,
 901                                       ia64_native_branch_target, nelem);
 902 }