target/i386/tcg/sysemu/svm_helper.c

   1 /*
   2  *  x86 SVM helpers (sysemu only)
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "cpu.h"
  23 #include "exec/helper-proto.h"
  24 #include "exec/exec-all.h"
  25 #include "exec/cpu_ldst.h"
  26 #include "tcg/helper-tcg.h"
  27
  28 /* Secure Virtual Machine helpers */
  29
  30 static void svm_save_seg(CPUX86State *env, int mmu_idx, hwaddr addr,
  31                          const SegmentCache *sc)
  32 {
  33     cpu_stw_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, selector),
  34                       sc->selector, mmu_idx, 0);
  35     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, base),
  36                       sc->base, mmu_idx, 0);
  37     cpu_stl_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, limit),
  38                       sc->limit, mmu_idx, 0);
  39     cpu_stw_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, attrib),
  40                       ((sc->flags >> 8) & 0xff)
  41                       | ((sc->flags >> 12) & 0x0f00),
  42                       mmu_idx, 0);
  43 }
  44
  45 /*
  46  * VMRUN and VMLOAD canonicalizes (i.e., sign-extend to bit 63) all base
  47  * addresses in the segment registers that have been loaded.
  48  */
  49 static inline void svm_canonicalization(CPUX86State *env, target_ulong *seg_base)
  50 {
  51     uint16_t shift_amt = 64 - cpu_x86_virtual_addr_width(env);
  52     *seg_base = ((((long) *seg_base) << shift_amt) >> shift_amt);
  53 }
  54
  55 static void svm_load_seg(CPUX86State *env, int mmu_idx, hwaddr addr,
  56                          SegmentCache *sc)
  57 {
  58     unsigned int flags;
  59
  60     sc->selector =
  61         cpu_lduw_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, selector),
  62                            mmu_idx, 0);
  63     sc->base =
  64         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, base),
  65                           mmu_idx, 0);
  66     sc->limit =
  67         cpu_ldl_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, limit),
  68                           mmu_idx, 0);
  69     flags =
  70         cpu_lduw_mmuidx_ra(env, addr + offsetof(struct vmcb_seg, attrib),
  71                            mmu_idx, 0);
  72     sc->flags = ((flags & 0xff) << 8) | ((flags & 0x0f00) << 12);
  73
  74     svm_canonicalization(env, &sc->base);
  75 }
  76
  77 static void svm_load_seg_cache(CPUX86State *env, int mmu_idx,
  78                                hwaddr addr, int seg_reg)
  79 {
  80     SegmentCache sc;
  81
  82     svm_load_seg(env, mmu_idx, addr, &sc);
  83     cpu_x86_load_seg_cache(env, seg_reg, sc.selector,
  84                            sc.base, sc.limit, sc.flags);
  85 }
  86
  87 static inline bool is_efer_invalid_state (CPUX86State *env)
  88 {
  89     if (!(env->efer & MSR_EFER_SVME)) {
  90         return true;
  91     }
  92
  93     if (env->efer & MSR_EFER_RESERVED) {
  94         return true;
  95     }
  96
  97     if ((env->efer & (MSR_EFER_LMA | MSR_EFER_LME)) &&
  98             !(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM)) {
  99         return true;
 100     }
 101
 102     if ((env->efer & MSR_EFER_LME) && (env->cr[0] & CR0_PG_MASK)
 103                                 && !(env->cr[4] & CR4_PAE_MASK)) {
 104         return true;
 105     }
 106
 107     if ((env->efer & MSR_EFER_LME) && (env->cr[0] & CR0_PG_MASK)
 108                                 && !(env->cr[0] & CR0_PE_MASK)) {
 109         return true;
 110     }
 111
 112     if ((env->efer & MSR_EFER_LME) && (env->cr[0] & CR0_PG_MASK)
 113                                 && (env->cr[4] & CR4_PAE_MASK)
 114                                 && (env->segs[R_CS].flags & DESC_L_MASK)
 115                                 && (env->segs[R_CS].flags & DESC_B_MASK)) {
 116         return true;
 117     }
 118
 119     return false;
 120 }
 121
 122 static inline bool virtual_gif_enabled(CPUX86State *env)
 123 {
 124     if (likely(env->hflags & HF_GUEST_MASK)) {
 125         return (env->features[FEAT_SVM] & CPUID_SVM_VGIF)
 126                     && (env->int_ctl & V_GIF_ENABLED_MASK);
 127     }
 128     return false;
 129 }
 130
 131 static inline bool virtual_vm_load_save_enabled(CPUX86State *env, uint32_t exit_code, uintptr_t retaddr)
 132 {
 133     uint64_t lbr_ctl;
 134
 135     if (likely(env->hflags & HF_GUEST_MASK)) {
 136         if (likely(!(env->hflags2 & HF2_NPT_MASK)) || !(env->efer & MSR_EFER_LMA)) {
 137             cpu_vmexit(env, exit_code, 0, retaddr);
 138         }
 139
 140         lbr_ctl = x86_ldl_phys(env_cpu(env), env->vm_vmcb + offsetof(struct vmcb,
 141                                                   control.lbr_ctl));
 142         return (env->features[FEAT_SVM] & CPUID_SVM_V_VMSAVE_VMLOAD)
 143                 && (lbr_ctl & V_VMLOAD_VMSAVE_ENABLED_MASK);
 144
 145     }
 146
 147     return false;
 148 }
 149
 150 static inline bool virtual_gif_set(CPUX86State *env)
 151 {
 152     return !virtual_gif_enabled(env) || (env->int_ctl & V_GIF_MASK);
 153 }
 154
 155 void helper_vmrun(CPUX86State *env, int aflag, int next_eip_addend)
 156 {
 157     CPUState *cs = env_cpu(env);
 158     X86CPU *cpu = env_archcpu(env);
 159     target_ulong addr;
 160     uint64_t nested_ctl;
 161     uint32_t event_inj;
 162     uint32_t asid;
 163     uint64_t new_cr0;
 164     uint64_t new_cr3;
 165     uint64_t new_cr4;
 166     uint64_t new_dr6;
 167     uint64_t new_dr7;
 168
 169     if (aflag == 2) {
 170         addr = env->regs[R_EAX];
 171     } else {
 172         addr = (uint32_t)env->regs[R_EAX];
 173     }
 174
 175     /* Exceptions are checked before the intercept.  */
 176     if (addr & (0xfff | ((~0ULL) << env_archcpu(env)->phys_bits))) {
 177         raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
 178     }
 179
 180     cpu_svm_check_intercept_param(env, SVM_EXIT_VMRUN, 0, GETPC());
 181
 182     qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmrun! " TARGET_FMT_lx "\n", addr);
 183
 184     env->vm_vmcb = addr;
 185
 186     /* save the current CPU state in the hsave page */
 187     x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.gdtr.base),
 188              env->gdt.base);
 189     x86_stl_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.gdtr.limit),
 190              env->gdt.limit);
 191
 192     x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.idtr.base),
 193              env->idt.base);
 194     x86_stl_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.idtr.limit),
 195              env->idt.limit);
 196
 197     x86_stq_phys(cs,
 198              env->vm_hsave + offsetof(struct vmcb, save.cr0), env->cr[0]);
 199     x86_stq_phys(cs,
 200              env->vm_hsave + offsetof(struct vmcb, save.cr2), env->cr[2]);
 201     x86_stq_phys(cs,
 202              env->vm_hsave + offsetof(struct vmcb, save.cr3), env->cr[3]);
 203     x86_stq_phys(cs,
 204              env->vm_hsave + offsetof(struct vmcb, save.cr4), env->cr[4]);
 205     x86_stq_phys(cs,
 206              env->vm_hsave + offsetof(struct vmcb, save.dr6), env->dr[6]);
 207     x86_stq_phys(cs,
 208              env->vm_hsave + offsetof(struct vmcb, save.dr7), env->dr[7]);
 209
 210     x86_stq_phys(cs,
 211              env->vm_hsave + offsetof(struct vmcb, save.efer), env->efer);
 212     x86_stq_phys(cs,
 213              env->vm_hsave + offsetof(struct vmcb, save.rflags),
 214              cpu_compute_eflags(env));
 215
 216     svm_save_seg(env, MMU_PHYS_IDX,
 217                  env->vm_hsave + offsetof(struct vmcb, save.es),
 218                  &env->segs[R_ES]);
 219     svm_save_seg(env, MMU_PHYS_IDX,
 220                  env->vm_hsave + offsetof(struct vmcb, save.cs),
 221                  &env->segs[R_CS]);
 222     svm_save_seg(env, MMU_PHYS_IDX,
 223                  env->vm_hsave + offsetof(struct vmcb, save.ss),
 224                  &env->segs[R_SS]);
 225     svm_save_seg(env, MMU_PHYS_IDX,
 226                  env->vm_hsave + offsetof(struct vmcb, save.ds),
 227                  &env->segs[R_DS]);
 228
 229     x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.rip),
 230              env->eip + next_eip_addend);
 231     x86_stq_phys(cs,
 232              env->vm_hsave + offsetof(struct vmcb, save.rsp), env->regs[R_ESP]);
 233     x86_stq_phys(cs,
 234              env->vm_hsave + offsetof(struct vmcb, save.rax), env->regs[R_EAX]);
 235
 236     /* load the interception bitmaps so we do not need to access the
 237        vmcb in svm mode */
 238     env->intercept = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 239                                                       control.intercept));
 240     env->intercept_cr_read = x86_lduw_phys(cs, env->vm_vmcb +
 241                                        offsetof(struct vmcb,
 242                                                 control.intercept_cr_read));
 243     env->intercept_cr_write = x86_lduw_phys(cs, env->vm_vmcb +
 244                                         offsetof(struct vmcb,
 245                                                  control.intercept_cr_write));
 246     env->intercept_dr_read = x86_lduw_phys(cs, env->vm_vmcb +
 247                                        offsetof(struct vmcb,
 248                                                 control.intercept_dr_read));
 249     env->intercept_dr_write = x86_lduw_phys(cs, env->vm_vmcb +
 250                                         offsetof(struct vmcb,
 251                                                  control.intercept_dr_write));
 252     env->intercept_exceptions = x86_ldl_phys(cs, env->vm_vmcb +
 253                                          offsetof(struct vmcb,
 254                                                   control.intercept_exceptions
 255                                                   ));
 256
 257     env->hflags &= ~HF_INHIBIT_IRQ_MASK;
 258     if (x86_ldl_phys(cs, env->vm_vmcb +
 259                 offsetof(struct vmcb, control.int_state)) &
 260                  SVM_INTERRUPT_SHADOW_MASK) {
 261         env->hflags |= HF_INHIBIT_IRQ_MASK;
 262     }
 263
 264     nested_ctl = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 265                                                           control.nested_ctl));
 266     asid = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 267                                                           control.asid));
 268
 269     uint64_t msrpm_base_pa = x86_ldq_phys(cs, env->vm_vmcb +
 270                                     offsetof(struct vmcb,
 271                                             control.msrpm_base_pa));
 272     uint64_t iopm_base_pa = x86_ldq_phys(cs, env->vm_vmcb +
 273                                  offsetof(struct vmcb, control.iopm_base_pa));
 274
 275     if ((msrpm_base_pa & ~0xfff) >= (1ull << cpu->phys_bits) - SVM_MSRPM_SIZE) {
 276         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 277     }
 278
 279     if ((iopm_base_pa & ~0xfff) >= (1ull << cpu->phys_bits) - SVM_IOPM_SIZE) {
 280         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 281     }
 282
 283     env->nested_pg_mode = 0;
 284
 285     if (!cpu_svm_has_intercept(env, SVM_EXIT_VMRUN)) {
 286         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 287     }
 288     if (asid == 0) {
 289         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 290     }
 291
 292     if (nested_ctl & SVM_NPT_ENABLED) {
 293         env->nested_cr3 = x86_ldq_phys(cs,
 294                                 env->vm_vmcb + offsetof(struct vmcb,
 295                                                         control.nested_cr3));
 296         env->hflags2 |= HF2_NPT_MASK;
 297
 298         env->nested_pg_mode = get_pg_mode(env) & PG_MODE_SVM_MASK;
 299
 300         tlb_flush_by_mmuidx(cs, 1 << MMU_NESTED_IDX);
 301     }
 302
 303     /* enable intercepts */
 304     env->hflags |= HF_GUEST_MASK;
 305
 306     env->tsc_offset = x86_ldq_phys(cs, env->vm_vmcb +
 307                                offsetof(struct vmcb, control.tsc_offset));
 308
 309     new_cr0 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cr0));
 310     if (new_cr0 & SVM_CR0_RESERVED_MASK) {
 311         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 312     }
 313     if ((new_cr0 & CR0_NW_MASK) && !(new_cr0 & CR0_CD_MASK)) {
 314         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 315     }
 316     new_cr3 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cr3));
 317     if ((env->efer & MSR_EFER_LMA) &&
 318             (new_cr3 & ((~0ULL) << cpu->phys_bits))) {
 319         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 320     }
 321     new_cr4 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cr4));
 322     if (new_cr4 & cr4_reserved_bits(env)) {
 323         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 324     }
 325     /* clear exit_info_2 so we behave like the real hardware */
 326     x86_stq_phys(cs,
 327              env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2), 0);
 328
 329     cpu_x86_update_cr0(env, new_cr0);
 330     cpu_x86_update_cr4(env, new_cr4);
 331     cpu_x86_update_cr3(env, new_cr3);
 332     env->cr[2] = x86_ldq_phys(cs,
 333                           env->vm_vmcb + offsetof(struct vmcb, save.cr2));
 334     env->int_ctl = x86_ldl_phys(cs,
 335                        env->vm_vmcb + offsetof(struct vmcb, control.int_ctl));
 336     env->hflags2 &= ~(HF2_HIF_MASK | HF2_VINTR_MASK);
 337     if (env->int_ctl & V_INTR_MASKING_MASK) {
 338         env->hflags2 |= HF2_VINTR_MASK;
 339         if (env->eflags & IF_MASK) {
 340             env->hflags2 |= HF2_HIF_MASK;
 341         }
 342     }
 343
 344     cpu_load_efer(env,
 345                   x86_ldq_phys(cs,
 346                            env->vm_vmcb + offsetof(struct vmcb, save.efer)));
 347     env->eflags = 0;
 348     cpu_load_eflags(env, x86_ldq_phys(cs,
 349                                   env->vm_vmcb + offsetof(struct vmcb,
 350                                                           save.rflags)),
 351                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
 352
 353     svm_load_seg_cache(env, MMU_PHYS_IDX,
 354                        env->vm_vmcb + offsetof(struct vmcb, save.es), R_ES);
 355     svm_load_seg_cache(env, MMU_PHYS_IDX,
 356                        env->vm_vmcb + offsetof(struct vmcb, save.cs), R_CS);
 357     svm_load_seg_cache(env, MMU_PHYS_IDX,
 358                        env->vm_vmcb + offsetof(struct vmcb, save.ss), R_SS);
 359     svm_load_seg_cache(env, MMU_PHYS_IDX,
 360                        env->vm_vmcb + offsetof(struct vmcb, save.ds), R_DS);
 361     svm_load_seg(env, MMU_PHYS_IDX,
 362                  env->vm_vmcb + offsetof(struct vmcb, save.idtr), &env->idt);
 363     svm_load_seg(env, MMU_PHYS_IDX,
 364                  env->vm_vmcb + offsetof(struct vmcb, save.gdtr), &env->gdt);
 365
 366     env->eip = x86_ldq_phys(cs,
 367                         env->vm_vmcb + offsetof(struct vmcb, save.rip));
 368
 369     env->regs[R_ESP] = x86_ldq_phys(cs,
 370                                 env->vm_vmcb + offsetof(struct vmcb, save.rsp));
 371     env->regs[R_EAX] = x86_ldq_phys(cs,
 372                                 env->vm_vmcb + offsetof(struct vmcb, save.rax));
 373
 374     new_dr7 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.dr7));
 375     new_dr6 = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.dr6));
 376
 377 #ifdef TARGET_X86_64
 378     if (new_dr7 & DR_RESERVED_MASK) {
 379         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 380     }
 381     if (new_dr6 & DR_RESERVED_MASK) {
 382         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 383     }
 384 #endif
 385
 386     cpu_x86_update_dr7(env, new_dr7);
 387     env->dr[6] = new_dr6;
 388
 389     if (is_efer_invalid_state(env)) {
 390         cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 391     }
 392
 393     switch (x86_ldub_phys(cs,
 394                       env->vm_vmcb + offsetof(struct vmcb, control.tlb_ctl))) {
 395     case TLB_CONTROL_DO_NOTHING:
 396         break;
 397     case TLB_CONTROL_FLUSH_ALL_ASID:
 398         /* FIXME: this is not 100% correct but should work for now */
 399         tlb_flush(cs);
 400         break;
 401     }
 402
 403     env->hflags2 |= HF2_GIF_MASK;
 404
 405     if (ctl_has_irq(env)) {
 406         cs->interrupt_request |= CPU_INTERRUPT_VIRQ;
 407     }
 408
 409     if (virtual_gif_set(env)) {
 410         env->hflags2 |= HF2_VGIF_MASK;
 411     }
 412
 413     /* maybe we need to inject an event */
 414     event_inj = x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 415                                                  control.event_inj));
 416     if (event_inj & SVM_EVTINJ_VALID) {
 417         uint8_t vector = event_inj & SVM_EVTINJ_VEC_MASK;
 418         uint16_t valid_err = event_inj & SVM_EVTINJ_VALID_ERR;
 419         uint32_t event_inj_err = x86_ldl_phys(cs, env->vm_vmcb +
 420                                           offsetof(struct vmcb,
 421                                                    control.event_inj_err));
 422
 423         qemu_log_mask(CPU_LOG_TB_IN_ASM, "Injecting(%#hx): ", valid_err);
 424         /* FIXME: need to implement valid_err */
 425         switch (event_inj & SVM_EVTINJ_TYPE_MASK) {
 426         case SVM_EVTINJ_TYPE_INTR:
 427             cs->exception_index = vector;
 428             env->error_code = event_inj_err;
 429             env->exception_is_int = 0;
 430             env->exception_next_eip = -1;
 431             qemu_log_mask(CPU_LOG_TB_IN_ASM, "INTR");
 432             /* XXX: is it always correct? */
 433             do_interrupt_x86_hardirq(env, vector, 1);
 434             break;
 435         case SVM_EVTINJ_TYPE_NMI:
 436             cs->exception_index = EXCP02_NMI;
 437             env->error_code = event_inj_err;
 438             env->exception_is_int = 0;
 439             env->exception_next_eip = env->eip;
 440             qemu_log_mask(CPU_LOG_TB_IN_ASM, "NMI");
 441             cpu_loop_exit(cs);
 442             break;
 443         case SVM_EVTINJ_TYPE_EXEPT:
 444             if (vector == EXCP02_NMI || vector >= 31)  {
 445                 cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 446             }
 447             cs->exception_index = vector;
 448             env->error_code = event_inj_err;
 449             env->exception_is_int = 0;
 450             env->exception_next_eip = -1;
 451             qemu_log_mask(CPU_LOG_TB_IN_ASM, "EXEPT");
 452             cpu_loop_exit(cs);
 453             break;
 454         case SVM_EVTINJ_TYPE_SOFT:
 455             cs->exception_index = vector;
 456             env->error_code = event_inj_err;
 457             env->exception_is_int = 1;
 458             env->exception_next_eip = env->eip;
 459             qemu_log_mask(CPU_LOG_TB_IN_ASM, "SOFT");
 460             cpu_loop_exit(cs);
 461             break;
 462         default:
 463             cpu_vmexit(env, SVM_EXIT_ERR, 0, GETPC());
 464             break;
 465         }
 466         qemu_log_mask(CPU_LOG_TB_IN_ASM, " %#x %#x\n", cs->exception_index,
 467                       env->error_code);
 468     }
 469 }
 470
 471 void helper_vmmcall(CPUX86State *env)
 472 {
 473     cpu_svm_check_intercept_param(env, SVM_EXIT_VMMCALL, 0, GETPC());
 474     raise_exception(env, EXCP06_ILLOP);
 475 }
 476
 477 void helper_vmload(CPUX86State *env, int aflag)
 478 {
 479     int mmu_idx = MMU_PHYS_IDX;
 480     target_ulong addr;
 481
 482     if (aflag == 2) {
 483         addr = env->regs[R_EAX];
 484     } else {
 485         addr = (uint32_t)env->regs[R_EAX];
 486     }
 487
 488     /* Exceptions are checked before the intercept.  */
 489     if (addr & (0xfff | ((~0ULL) << env_archcpu(env)->phys_bits))) {
 490         raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
 491     }
 492
 493     cpu_svm_check_intercept_param(env, SVM_EXIT_VMLOAD, 0, GETPC());
 494
 495     if (virtual_vm_load_save_enabled(env, SVM_EXIT_VMLOAD, GETPC())) {
 496         mmu_idx = MMU_NESTED_IDX;
 497     }
 498
 499     svm_load_seg_cache(env, mmu_idx,
 500                        addr + offsetof(struct vmcb, save.fs), R_FS);
 501     svm_load_seg_cache(env, mmu_idx,
 502                        addr + offsetof(struct vmcb, save.gs), R_GS);
 503     svm_load_seg(env, mmu_idx,
 504                  addr + offsetof(struct vmcb, save.tr), &env->tr);
 505     svm_load_seg(env, mmu_idx,
 506                  addr + offsetof(struct vmcb, save.ldtr), &env->ldt);
 507
 508 #ifdef TARGET_X86_64
 509     env->kernelgsbase =
 510         cpu_ldq_mmuidx_ra(env,
 511                           addr + offsetof(struct vmcb, save.kernel_gs_base),
 512                           mmu_idx, 0);
 513     env->lstar =
 514         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.lstar),
 515                           mmu_idx, 0);
 516     env->cstar =
 517         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.cstar),
 518                           mmu_idx, 0);
 519     env->fmask =
 520         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sfmask),
 521                           mmu_idx, 0);
 522     svm_canonicalization(env, &env->kernelgsbase);
 523 #endif
 524     env->star =
 525         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.star),
 526                           mmu_idx, 0);
 527     env->sysenter_cs =
 528         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_cs),
 529                           mmu_idx, 0);
 530     env->sysenter_esp =
 531         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_esp),
 532                           mmu_idx, 0);
 533     env->sysenter_eip =
 534         cpu_ldq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_eip),
 535                           mmu_idx, 0);
 536 }
 537
 538 void helper_vmsave(CPUX86State *env, int aflag)
 539 {
 540     int mmu_idx = MMU_PHYS_IDX;
 541     target_ulong addr;
 542
 543     if (aflag == 2) {
 544         addr = env->regs[R_EAX];
 545     } else {
 546         addr = (uint32_t)env->regs[R_EAX];
 547     }
 548
 549     /* Exceptions are checked before the intercept.  */
 550     if (addr & (0xfff | ((~0ULL) << env_archcpu(env)->phys_bits))) {
 551         raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
 552     }
 553
 554     cpu_svm_check_intercept_param(env, SVM_EXIT_VMSAVE, 0, GETPC());
 555
 556     if (virtual_vm_load_save_enabled(env, SVM_EXIT_VMSAVE, GETPC())) {
 557         mmu_idx = MMU_NESTED_IDX;
 558     }
 559
 560     svm_save_seg(env, mmu_idx, addr + offsetof(struct vmcb, save.fs),
 561                  &env->segs[R_FS]);
 562     svm_save_seg(env, mmu_idx, addr + offsetof(struct vmcb, save.gs),
 563                  &env->segs[R_GS]);
 564     svm_save_seg(env, mmu_idx, addr + offsetof(struct vmcb, save.tr),
 565                  &env->tr);
 566     svm_save_seg(env, mmu_idx, addr + offsetof(struct vmcb, save.ldtr),
 567                  &env->ldt);
 568
 569 #ifdef TARGET_X86_64
 570     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.kernel_gs_base),
 571                       env->kernelgsbase, mmu_idx, 0);
 572     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.lstar),
 573                       env->lstar, mmu_idx, 0);
 574     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.cstar),
 575                       env->cstar, mmu_idx, 0);
 576     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sfmask),
 577                       env->fmask, mmu_idx, 0);
 578 #endif
 579     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.star),
 580                       env->star, mmu_idx, 0);
 581     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_cs),
 582                       env->sysenter_cs, mmu_idx, 0);
 583     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_esp),
 584                       env->sysenter_esp, mmu_idx, 0);
 585     cpu_stq_mmuidx_ra(env, addr + offsetof(struct vmcb, save.sysenter_eip),
 586                       env->sysenter_eip, mmu_idx, 0);
 587 }
 588
 589 void helper_stgi(CPUX86State *env)
 590 {
 591     cpu_svm_check_intercept_param(env, SVM_EXIT_STGI, 0, GETPC());
 592
 593     if (virtual_gif_enabled(env)) {
 594         env->int_ctl |= V_GIF_MASK;
 595         env->hflags2 |= HF2_VGIF_MASK;
 596     } else {
 597         env->hflags2 |= HF2_GIF_MASK;
 598     }
 599 }
 600
 601 void helper_clgi(CPUX86State *env)
 602 {
 603     cpu_svm_check_intercept_param(env, SVM_EXIT_CLGI, 0, GETPC());
 604
 605     if (virtual_gif_enabled(env)) {
 606         env->int_ctl &= ~V_GIF_MASK;
 607         env->hflags2 &= ~HF2_VGIF_MASK;
 608     } else {
 609         env->hflags2 &= ~HF2_GIF_MASK;
 610     }
 611 }
 612
 613 bool cpu_svm_has_intercept(CPUX86State *env, uint32_t type)
 614 {
 615     switch (type) {
 616     case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR0 + 8:
 617         if (env->intercept_cr_read & (1 << (type - SVM_EXIT_READ_CR0))) {
 618             return true;
 619         }
 620         break;
 621     case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR0 + 8:
 622         if (env->intercept_cr_write & (1 << (type - SVM_EXIT_WRITE_CR0))) {
 623             return true;
 624         }
 625         break;
 626     case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR0 + 7:
 627         if (env->intercept_dr_read & (1 << (type - SVM_EXIT_READ_DR0))) {
 628             return true;
 629         }
 630         break;
 631     case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR0 + 7:
 632         if (env->intercept_dr_write & (1 << (type - SVM_EXIT_WRITE_DR0))) {
 633             return true;
 634         }
 635         break;
 636     case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 31:
 637         if (env->intercept_exceptions & (1 << (type - SVM_EXIT_EXCP_BASE))) {
 638             return true;
 639         }
 640         break;
 641     default:
 642         if (env->intercept & (1ULL << (type - SVM_EXIT_INTR))) {
 643             return true;
 644         }
 645         break;
 646     }
 647     return false;
 648 }
 649
 650 void cpu_svm_check_intercept_param(CPUX86State *env, uint32_t type,
 651                                    uint64_t param, uintptr_t retaddr)
 652 {
 653     CPUState *cs = env_cpu(env);
 654
 655     if (likely(!(env->hflags & HF_GUEST_MASK))) {
 656         return;
 657     }
 658
 659     if (!cpu_svm_has_intercept(env, type)) {
 660         return;
 661     }
 662
 663     if (type == SVM_EXIT_MSR) {
 664         /* FIXME: this should be read in at vmrun (faster this way?) */
 665         uint64_t addr = x86_ldq_phys(cs, env->vm_vmcb +
 666                                     offsetof(struct vmcb,
 667                                             control.msrpm_base_pa));
 668         uint32_t t0, t1;
 669
 670         switch ((uint32_t)env->regs[R_ECX]) {
 671         case 0 ... 0x1fff:
 672             t0 = (env->regs[R_ECX] * 2) % 8;
 673             t1 = (env->regs[R_ECX] * 2) / 8;
 674             break;
 675         case 0xc0000000 ... 0xc0001fff:
 676             t0 = (8192 + env->regs[R_ECX] - 0xc0000000) * 2;
 677             t1 = (t0 / 8);
 678             t0 %= 8;
 679             break;
 680         case 0xc0010000 ... 0xc0011fff:
 681             t0 = (16384 + env->regs[R_ECX] - 0xc0010000) * 2;
 682             t1 = (t0 / 8);
 683             t0 %= 8;
 684             break;
 685         default:
 686             cpu_vmexit(env, type, param, retaddr);
 687             t0 = 0;
 688             t1 = 0;
 689             break;
 690         }
 691         if (x86_ldub_phys(cs, addr + t1) & ((1 << param) << t0)) {
 692             cpu_vmexit(env, type, param, retaddr);
 693         }
 694         return;
 695     }
 696
 697     cpu_vmexit(env, type, param, retaddr);
 698 }
 699
 700 void helper_svm_check_intercept(CPUX86State *env, uint32_t type)
 701 {
 702     cpu_svm_check_intercept_param(env, type, 0, GETPC());
 703 }
 704
 705 void helper_svm_check_io(CPUX86State *env, uint32_t port, uint32_t param,
 706                          uint32_t next_eip_addend)
 707 {
 708     CPUState *cs = env_cpu(env);
 709
 710     if (env->intercept & (1ULL << (SVM_EXIT_IOIO - SVM_EXIT_INTR))) {
 711         /* FIXME: this should be read in at vmrun (faster this way?) */
 712         uint64_t addr = x86_ldq_phys(cs, env->vm_vmcb +
 713                                  offsetof(struct vmcb, control.iopm_base_pa));
 714         uint16_t mask = (1 << ((param >> 4) & 7)) - 1;
 715
 716         if (x86_lduw_phys(cs, addr + port / 8) & (mask << (port & 7))) {
 717             /* next env->eip */
 718             x86_stq_phys(cs,
 719                      env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
 720                      env->eip + next_eip_addend);
 721             cpu_vmexit(env, SVM_EXIT_IOIO, param | (port << 16), GETPC());
 722         }
 723     }
 724 }
 725
 726 void cpu_vmexit(CPUX86State *env, uint32_t exit_code, uint64_t exit_info_1,
 727                 uintptr_t retaddr)
 728 {
 729     CPUState *cs = env_cpu(env);
 730
 731     cpu_restore_state(cs, retaddr);
 732
 733     qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmexit(%08x, %016" PRIx64 ", %016"
 734                   PRIx64 ", " TARGET_FMT_lx ")!\n",
 735                   exit_code, exit_info_1,
 736                   x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 737                                                    control.exit_info_2)),
 738                   env->eip);
 739
 740     cs->exception_index = EXCP_VMEXIT;
 741     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, control.exit_code),
 742              exit_code);
 743
 744     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 745                                              control.exit_info_1), exit_info_1),
 746
 747     /* remove any pending exception */
 748     env->old_exception = -1;
 749     cpu_loop_exit(cs);
 750 }
 751
 752 void do_vmexit(CPUX86State *env)
 753 {
 754     CPUState *cs = env_cpu(env);
 755
 756     if (env->hflags & HF_INHIBIT_IRQ_MASK) {
 757         x86_stl_phys(cs,
 758                  env->vm_vmcb + offsetof(struct vmcb, control.int_state),
 759                  SVM_INTERRUPT_SHADOW_MASK);
 760         env->hflags &= ~HF_INHIBIT_IRQ_MASK;
 761     } else {
 762         x86_stl_phys(cs,
 763                  env->vm_vmcb + offsetof(struct vmcb, control.int_state), 0);
 764     }
 765     env->hflags2 &= ~HF2_NPT_MASK;
 766     tlb_flush_by_mmuidx(cs, 1 << MMU_NESTED_IDX);
 767
 768     /* Save the VM state in the vmcb */
 769     svm_save_seg(env, MMU_PHYS_IDX,
 770                  env->vm_vmcb + offsetof(struct vmcb, save.es),
 771                  &env->segs[R_ES]);
 772     svm_save_seg(env, MMU_PHYS_IDX,
 773                  env->vm_vmcb + offsetof(struct vmcb, save.cs),
 774                  &env->segs[R_CS]);
 775     svm_save_seg(env, MMU_PHYS_IDX,
 776                  env->vm_vmcb + offsetof(struct vmcb, save.ss),
 777                  &env->segs[R_SS]);
 778     svm_save_seg(env, MMU_PHYS_IDX,
 779                  env->vm_vmcb + offsetof(struct vmcb, save.ds),
 780                  &env->segs[R_DS]);
 781
 782     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base),
 783              env->gdt.base);
 784     x86_stl_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit),
 785              env->gdt.limit);
 786
 787     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.idtr.base),
 788              env->idt.base);
 789     x86_stl_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.idtr.limit),
 790              env->idt.limit);
 791
 792     x86_stq_phys(cs,
 793              env->vm_vmcb + offsetof(struct vmcb, save.efer), env->efer);
 794     x86_stq_phys(cs,
 795              env->vm_vmcb + offsetof(struct vmcb, save.cr0), env->cr[0]);
 796     x86_stq_phys(cs,
 797              env->vm_vmcb + offsetof(struct vmcb, save.cr2), env->cr[2]);
 798     x86_stq_phys(cs,
 799              env->vm_vmcb + offsetof(struct vmcb, save.cr3), env->cr[3]);
 800     x86_stq_phys(cs,
 801              env->vm_vmcb + offsetof(struct vmcb, save.cr4), env->cr[4]);
 802     x86_stl_phys(cs,
 803              env->vm_vmcb + offsetof(struct vmcb, control.int_ctl), env->int_ctl);
 804
 805     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.rflags),
 806              cpu_compute_eflags(env));
 807     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.rip),
 808              env->eip);
 809     x86_stq_phys(cs,
 810              env->vm_vmcb + offsetof(struct vmcb, save.rsp), env->regs[R_ESP]);
 811     x86_stq_phys(cs,
 812              env->vm_vmcb + offsetof(struct vmcb, save.rax), env->regs[R_EAX]);
 813     x86_stq_phys(cs,
 814              env->vm_vmcb + offsetof(struct vmcb, save.dr7), env->dr[7]);
 815     x86_stq_phys(cs,
 816              env->vm_vmcb + offsetof(struct vmcb, save.dr6), env->dr[6]);
 817     x86_stb_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cpl),
 818              env->hflags & HF_CPL_MASK);
 819
 820     /* Reload the host state from vm_hsave */
 821     env->hflags2 &= ~(HF2_HIF_MASK | HF2_VINTR_MASK);
 822     env->hflags &= ~HF_GUEST_MASK;
 823     env->intercept = 0;
 824     env->intercept_exceptions = 0;
 825
 826     /* Clears the V_IRQ and V_INTR_MASKING bits inside the processor. */
 827     cs->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
 828     env->int_ctl = 0;
 829
 830     /* Clears the TSC_OFFSET inside the processor. */
 831     env->tsc_offset = 0;
 832
 833     env->gdt.base  = x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
 834                                                        save.gdtr.base));
 835     env->gdt.limit = x86_ldl_phys(cs, env->vm_hsave + offsetof(struct vmcb,
 836                                                        save.gdtr.limit));
 837
 838     env->idt.base  = x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
 839                                                        save.idtr.base));
 840     env->idt.limit = x86_ldl_phys(cs, env->vm_hsave + offsetof(struct vmcb,
 841                                                        save.idtr.limit));
 842
 843     cpu_x86_update_cr0(env, x86_ldq_phys(cs,
 844                                      env->vm_hsave + offsetof(struct vmcb,
 845                                                               save.cr0)) |
 846                        CR0_PE_MASK);
 847     cpu_x86_update_cr4(env, x86_ldq_phys(cs,
 848                                      env->vm_hsave + offsetof(struct vmcb,
 849                                                               save.cr4)));
 850
 851     /*
 852      * Resets the current ASID register to zero (host ASID; TLB flush).
 853      *
 854      * If the host is in PAE mode, the processor reloads the host's PDPEs
 855      * from the page table indicated the host's CR3. FIXME: If the PDPEs
 856      * contain illegal state, the processor causes a shutdown (QEMU does
 857      * not implement PDPTRs).
 858      */
 859     cpu_x86_update_cr3(env, x86_ldq_phys(cs,
 860                                      env->vm_hsave + offsetof(struct vmcb,
 861                                                               save.cr3)));
 862     /* we need to set the efer after the crs so the hidden flags get
 863        set properly */
 864     cpu_load_efer(env, x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
 865                                                          save.efer)));
 866
 867     /* Completion of the VMRUN instruction clears the host EFLAGS.RF bit.  */
 868     env->eflags = 0;
 869     cpu_load_eflags(env, x86_ldq_phys(cs,
 870                                   env->vm_hsave + offsetof(struct vmcb,
 871                                                            save.rflags)),
 872                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK |
 873                       RF_MASK | VM_MASK));
 874
 875     svm_load_seg_cache(env, MMU_PHYS_IDX,
 876                        env->vm_hsave + offsetof(struct vmcb, save.es), R_ES);
 877     svm_load_seg_cache(env, MMU_PHYS_IDX,
 878                        env->vm_hsave + offsetof(struct vmcb, save.cs), R_CS);
 879     svm_load_seg_cache(env, MMU_PHYS_IDX,
 880                        env->vm_hsave + offsetof(struct vmcb, save.ss), R_SS);
 881     svm_load_seg_cache(env, MMU_PHYS_IDX,
 882                        env->vm_hsave + offsetof(struct vmcb, save.ds), R_DS);
 883
 884     env->eip = x86_ldq_phys(cs,
 885                         env->vm_hsave + offsetof(struct vmcb, save.rip));
 886     env->regs[R_ESP] = x86_ldq_phys(cs, env->vm_hsave +
 887                                 offsetof(struct vmcb, save.rsp));
 888     env->regs[R_EAX] = x86_ldq_phys(cs, env->vm_hsave +
 889                                 offsetof(struct vmcb, save.rax));
 890
 891     env->dr[6] = x86_ldq_phys(cs,
 892                           env->vm_hsave + offsetof(struct vmcb, save.dr6));
 893
 894     /* Disables all breakpoints in the host DR7 register. */
 895     cpu_x86_update_dr7(env,
 896              x86_ldq_phys(cs,
 897                           env->vm_hsave + offsetof(struct vmcb, save.dr7)) & ~0xff);
 898
 899     /* other setups */
 900     x86_stl_phys(cs,
 901              env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info),
 902              x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 903                                               control.event_inj)));
 904     x86_stl_phys(cs,
 905              env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info_err),
 906              x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
 907                                               control.event_inj_err)));
 908     x86_stl_phys(cs,
 909              env->vm_vmcb + offsetof(struct vmcb, control.event_inj), 0);
 910
 911     env->hflags2 &= ~HF2_GIF_MASK;
 912     env->hflags2 &= ~HF2_VGIF_MASK;
 913
 914
 915     /* FIXME: Checks the reloaded host state for consistency. */
 916
 917     /*
 918      * EFLAGS.TF causes a #DB trap after the VMRUN completes on the host
 919      * side (i.e., after the #VMEXIT from the guest). Since we're running
 920      * in the main loop, call do_interrupt_all directly.
 921      */
 922     if ((env->eflags & TF_MASK) != 0) {
 923         env->dr[6] |= DR6_BS;
 924         do_interrupt_all(X86_CPU(cs), EXCP01_DB, 0, 0, env->eip, 0);
 925     }
 926 }