target/i386/excp_helper.c

   1 /*
   2  *  x86 exception helpers
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "cpu.h"
  22 #include "exec/exec-all.h"
  23 #include "qemu/log.h"
  24 #include "sysemu/runstate.h"
  25 #include "exec/helper-proto.h"
  26
  27 void helper_raise_interrupt(CPUX86State *env, int intno, int next_eip_addend)
  28 {
  29     raise_interrupt(env, intno, 1, 0, next_eip_addend);
  30 }
  31
  32 void helper_raise_exception(CPUX86State *env, int exception_index)
  33 {
  34     raise_exception(env, exception_index);
  35 }
  36
  37 /*
  38  * Check nested exceptions and change to double or triple fault if
  39  * needed. It should only be called, if this is not an interrupt.
  40  * Returns the new exception number.
  41  */
  42 static int check_exception(CPUX86State *env, int intno, int *error_code,
  43                            uintptr_t retaddr)
  44 {
  45     int first_contributory = env->old_exception == 0 ||
  46                               (env->old_exception >= 10 &&
  47                                env->old_exception <= 13);
  48     int second_contributory = intno == 0 ||
  49                                (intno >= 10 && intno <= 13);
  50
  51     qemu_log_mask(CPU_LOG_INT, "check_exception old: 0x%x new 0x%x\n",
  52                 env->old_exception, intno);
  53
  54 #if !defined(CONFIG_USER_ONLY)
  55     if (env->old_exception == EXCP08_DBLE) {
  56         if (env->hflags & HF_GUEST_MASK) {
  57             cpu_vmexit(env, SVM_EXIT_SHUTDOWN, 0, retaddr); /* does not return */
  58         }
  59
  60         qemu_log_mask(CPU_LOG_RESET, "Triple fault\n");
  61
  62         qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
  63         return EXCP_HLT;
  64     }
  65 #endif
  66
  67     if ((first_contributory && second_contributory)
  68         || (env->old_exception == EXCP0E_PAGE &&
  69             (second_contributory || (intno == EXCP0E_PAGE)))) {
  70         intno = EXCP08_DBLE;
  71         *error_code = 0;
  72     }
  73
  74     if (second_contributory || (intno == EXCP0E_PAGE) ||
  75         (intno == EXCP08_DBLE)) {
  76         env->old_exception = intno;
  77     }
  78
  79     return intno;
  80 }
  81
  82 /*
  83  * Signal an interruption. It is executed in the main CPU loop.
  84  * is_int is TRUE if coming from the int instruction. next_eip is the
  85  * env->eip value AFTER the interrupt instruction. It is only relevant if
  86  * is_int is TRUE.
  87  */
  88 static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
  89                                            int is_int, int error_code,
  90                                            int next_eip_addend,
  91                                            uintptr_t retaddr)
  92 {
  93     CPUState *cs = env_cpu(env);
  94
  95     if (!is_int) {
  96         cpu_svm_check_intercept_param(env, SVM_EXIT_EXCP_BASE + intno,
  97                                       error_code, retaddr);
  98         intno = check_exception(env, intno, &error_code, retaddr);
  99     } else {
 100         cpu_svm_check_intercept_param(env, SVM_EXIT_SWINT, 0, retaddr);
 101     }
 102
 103     cs->exception_index = intno;
 104     env->error_code = error_code;
 105     env->exception_is_int = is_int;
 106     env->exception_next_eip = env->eip + next_eip_addend;
 107     cpu_loop_exit_restore(cs, retaddr);
 108 }
 109
 110 /* shortcuts to generate exceptions */
 111
 112 void QEMU_NORETURN raise_interrupt(CPUX86State *env, int intno, int is_int,
 113                                    int error_code, int next_eip_addend)
 114 {
 115     raise_interrupt2(env, intno, is_int, error_code, next_eip_addend, 0);
 116 }
 117
 118 void raise_exception_err(CPUX86State *env, int exception_index,
 119                          int error_code)
 120 {
 121     raise_interrupt2(env, exception_index, 0, error_code, 0, 0);
 122 }
 123
 124 void raise_exception_err_ra(CPUX86State *env, int exception_index,
 125                             int error_code, uintptr_t retaddr)
 126 {
 127     raise_interrupt2(env, exception_index, 0, error_code, 0, retaddr);
 128 }
 129
 130 void raise_exception(CPUX86State *env, int exception_index)
 131 {
 132     raise_interrupt2(env, exception_index, 0, 0, 0, 0);
 133 }
 134
 135 void raise_exception_ra(CPUX86State *env, int exception_index, uintptr_t retaddr)
 136 {
 137     raise_interrupt2(env, exception_index, 0, 0, 0, retaddr);
 138 }
 139
 140 #if !defined(CONFIG_USER_ONLY)
 141 static hwaddr get_hphys(CPUState *cs, hwaddr gphys, MMUAccessType access_type,
 142                         int *prot)
 143 {
 144     CPUX86State *env = &X86_CPU(cs)->env;
 145     uint64_t rsvd_mask = PG_HI_RSVD_MASK;
 146     uint64_t ptep, pte;
 147     uint64_t exit_info_1 = 0;
 148     target_ulong pde_addr, pte_addr;
 149     uint32_t page_offset;
 150     int page_size;
 151
 152     if (likely(!(env->hflags2 & HF2_NPT_MASK))) {
 153         return gphys;
 154     }
 155
 156     if (!(env->nested_pg_mode & SVM_NPT_NXE)) {
 157         rsvd_mask |= PG_NX_MASK;
 158     }
 159
 160     if (env->nested_pg_mode & SVM_NPT_PAE) {
 161         uint64_t pde, pdpe;
 162         target_ulong pdpe_addr;
 163
 164 #ifdef TARGET_X86_64
 165         if (env->nested_pg_mode & SVM_NPT_LMA) {
 166             uint64_t pml5e;
 167             uint64_t pml4e_addr, pml4e;
 168
 169             pml5e = env->nested_cr3;
 170             ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
 171
 172             pml4e_addr = (pml5e & PG_ADDRESS_MASK) +
 173                     (((gphys >> 39) & 0x1ff) << 3);
 174             pml4e = x86_ldq_phys(cs, pml4e_addr);
 175             if (!(pml4e & PG_PRESENT_MASK)) {
 176                 goto do_fault;
 177             }
 178             if (pml4e & (rsvd_mask | PG_PSE_MASK)) {
 179                 goto do_fault_rsvd;
 180             }
 181             if (!(pml4e & PG_ACCESSED_MASK)) {
 182                 pml4e |= PG_ACCESSED_MASK;
 183                 x86_stl_phys_notdirty(cs, pml4e_addr, pml4e);
 184             }
 185             ptep &= pml4e ^ PG_NX_MASK;
 186             pdpe_addr = (pml4e & PG_ADDRESS_MASK) +
 187                     (((gphys >> 30) & 0x1ff) << 3);
 188             pdpe = x86_ldq_phys(cs, pdpe_addr);
 189             if (!(pdpe & PG_PRESENT_MASK)) {
 190                 goto do_fault;
 191             }
 192             if (pdpe & rsvd_mask) {
 193                 goto do_fault_rsvd;
 194             }
 195             ptep &= pdpe ^ PG_NX_MASK;
 196             if (!(pdpe & PG_ACCESSED_MASK)) {
 197                 pdpe |= PG_ACCESSED_MASK;
 198                 x86_stl_phys_notdirty(cs, pdpe_addr, pdpe);
 199             }
 200             if (pdpe & PG_PSE_MASK) {
 201                 /* 1 GB page */
 202                 page_size = 1024 * 1024 * 1024;
 203                 pte_addr = pdpe_addr;
 204                 pte = pdpe;
 205                 goto do_check_protect;
 206             }
 207         } else
 208 #endif
 209         {
 210             pdpe_addr = (env->nested_cr3 & ~0x1f) + ((gphys >> 27) & 0x18);
 211             pdpe = x86_ldq_phys(cs, pdpe_addr);
 212             if (!(pdpe & PG_PRESENT_MASK)) {
 213                 goto do_fault;
 214             }
 215             rsvd_mask |= PG_HI_USER_MASK;
 216             if (pdpe & (rsvd_mask | PG_NX_MASK)) {
 217                 goto do_fault_rsvd;
 218             }
 219             ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
 220         }
 221
 222         pde_addr = (pdpe & PG_ADDRESS_MASK) + (((gphys >> 21) & 0x1ff) << 3);
 223         pde = x86_ldq_phys(cs, pde_addr);
 224         if (!(pde & PG_PRESENT_MASK)) {
 225             goto do_fault;
 226         }
 227         if (pde & rsvd_mask) {
 228             goto do_fault_rsvd;
 229         }
 230         ptep &= pde ^ PG_NX_MASK;
 231         if (pde & PG_PSE_MASK) {
 232             /* 2 MB page */
 233             page_size = 2048 * 1024;
 234             pte_addr = pde_addr;
 235             pte = pde;
 236             goto do_check_protect;
 237         }
 238         /* 4 KB page */
 239         if (!(pde & PG_ACCESSED_MASK)) {
 240             pde |= PG_ACCESSED_MASK;
 241             x86_stl_phys_notdirty(cs, pde_addr, pde);
 242         }
 243         pte_addr = (pde & PG_ADDRESS_MASK) + (((gphys >> 12) & 0x1ff) << 3);
 244         pte = x86_ldq_phys(cs, pte_addr);
 245         if (!(pte & PG_PRESENT_MASK)) {
 246             goto do_fault;
 247         }
 248         if (pte & rsvd_mask) {
 249             goto do_fault_rsvd;
 250         }
 251         /* combine pde and pte nx, user and rw protections */
 252         ptep &= pte ^ PG_NX_MASK;
 253         page_size = 4096;
 254     } else {
 255         uint32_t pde;
 256
 257         /* page directory entry */
 258         pde_addr = (env->nested_cr3 & ~0xfff) + ((gphys >> 20) & 0xffc);
 259         pde = x86_ldl_phys(cs, pde_addr);
 260         if (!(pde & PG_PRESENT_MASK)) {
 261             goto do_fault;
 262         }
 263         ptep = pde | PG_NX_MASK;
 264
 265         /* if host cr4 PSE bit is set, then we use a 4MB page */
 266         if ((pde & PG_PSE_MASK) && (env->nested_pg_mode & SVM_NPT_PSE)) {
 267             page_size = 4096 * 1024;
 268             pte_addr = pde_addr;
 269
 270             /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
 271              * Leave bits 20-13 in place for setting accessed/dirty bits below.
 272              */
 273             pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
 274             rsvd_mask = 0x200000;
 275             goto do_check_protect_pse36;
 276         }
 277
 278         if (!(pde & PG_ACCESSED_MASK)) {
 279             pde |= PG_ACCESSED_MASK;
 280             x86_stl_phys_notdirty(cs, pde_addr, pde);
 281         }
 282
 283         /* page directory entry */
 284         pte_addr = (pde & ~0xfff) + ((gphys >> 10) & 0xffc);
 285         pte = x86_ldl_phys(cs, pte_addr);
 286         if (!(pte & PG_PRESENT_MASK)) {
 287             goto do_fault;
 288         }
 289         /* combine pde and pte user and rw protections */
 290         ptep &= pte | PG_NX_MASK;
 291         page_size = 4096;
 292         rsvd_mask = 0;
 293     }
 294
 295  do_check_protect:
 296     rsvd_mask |= (page_size - 1) & PG_ADDRESS_MASK & ~PG_PSE_PAT_MASK;
 297  do_check_protect_pse36:
 298     if (pte & rsvd_mask) {
 299         goto do_fault_rsvd;
 300     }
 301     ptep ^= PG_NX_MASK;
 302
 303     if (!(ptep & PG_USER_MASK)) {
 304         goto do_fault_protect;
 305     }
 306     if (ptep & PG_NX_MASK) {
 307         if (access_type == MMU_INST_FETCH) {
 308             goto do_fault_protect;
 309         }
 310         *prot &= ~PAGE_EXEC;
 311     }
 312     if (!(ptep & PG_RW_MASK)) {
 313         if (access_type == MMU_DATA_STORE) {
 314             goto do_fault_protect;
 315         }
 316         *prot &= ~PAGE_WRITE;
 317     }
 318
 319     pte &= PG_ADDRESS_MASK & ~(page_size - 1);
 320     page_offset = gphys & (page_size - 1);
 321     return pte + page_offset;
 322
 323  do_fault_rsvd:
 324     exit_info_1 |= SVM_NPTEXIT_RSVD;
 325  do_fault_protect:
 326     exit_info_1 |= SVM_NPTEXIT_P;
 327  do_fault:
 328     x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
 329                  gphys);
 330     exit_info_1 |= SVM_NPTEXIT_US;
 331     if (access_type == MMU_DATA_STORE) {
 332         exit_info_1 |= SVM_NPTEXIT_RW;
 333     } else if (access_type == MMU_INST_FETCH) {
 334         exit_info_1 |= SVM_NPTEXIT_ID;
 335     }
 336     if (prot) {
 337         exit_info_1 |= SVM_NPTEXIT_GPA;
 338     } else { /* page table access */
 339         exit_info_1 |= SVM_NPTEXIT_GPT;
 340     }
 341     cpu_vmexit(env, SVM_EXIT_NPF, exit_info_1, env->retaddr);
 342 }
 343
 344 /* return value:
 345  * -1 = cannot handle fault
 346  * 0  = nothing more to do
 347  * 1  = generate PF fault
 348  */
 349 static int handle_mmu_fault(CPUState *cs, vaddr addr, int size,
 350                             int is_write1, int mmu_idx)
 351 {
 352     X86CPU *cpu = X86_CPU(cs);
 353     CPUX86State *env = &cpu->env;
 354     uint64_t ptep, pte;
 355     int32_t a20_mask;
 356     target_ulong pde_addr, pte_addr;
 357     int error_code = 0;
 358     int is_dirty, prot, page_size, is_write, is_user;
 359     hwaddr paddr;
 360     uint64_t rsvd_mask = PG_HI_RSVD_MASK;
 361     uint32_t page_offset;
 362     target_ulong vaddr;
 363
 364     is_user = mmu_idx == MMU_USER_IDX;
 365 #if defined(DEBUG_MMU)
 366     printf("MMU fault: addr=%" VADDR_PRIx " w=%d u=%d eip=" TARGET_FMT_lx "\n",
 367            addr, is_write1, is_user, env->eip);
 368 #endif
 369     is_write = is_write1 & 1;
 370
 371     a20_mask = x86_get_a20_mask(env);
 372     if (!(env->cr[0] & CR0_PG_MASK)) {
 373         pte = addr;
 374 #ifdef TARGET_X86_64
 375         if (!(env->hflags & HF_LMA_MASK)) {
 376             /* Without long mode we can only address 32bits in real mode */
 377             pte = (uint32_t)pte;
 378         }
 379 #endif
 380         prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 381         page_size = 4096;
 382         goto do_mapping;
 383     }
 384
 385     if (!(env->efer & MSR_EFER_NXE)) {
 386         rsvd_mask |= PG_NX_MASK;
 387     }
 388
 389     if (env->cr[4] & CR4_PAE_MASK) {
 390         uint64_t pde, pdpe;
 391         target_ulong pdpe_addr;
 392
 393 #ifdef TARGET_X86_64
 394         if (env->hflags & HF_LMA_MASK) {
 395             bool la57 = env->cr[4] & CR4_LA57_MASK;
 396             uint64_t pml5e_addr, pml5e;
 397             uint64_t pml4e_addr, pml4e;
 398             int32_t sext;
 399
 400             /* test virtual address sign extension */
 401             sext = la57 ? (int64_t)addr >> 56 : (int64_t)addr >> 47;
 402             if (sext != 0 && sext != -1) {
 403                 env->error_code = 0;
 404                 cs->exception_index = EXCP0D_GPF;
 405                 return 1;
 406             }
 407
 408             if (la57) {
 409                 pml5e_addr = ((env->cr[3] & ~0xfff) +
 410                         (((addr >> 48) & 0x1ff) << 3)) & a20_mask;
 411                 pml5e_addr = get_hphys(cs, pml5e_addr, MMU_DATA_STORE, NULL);
 412                 pml5e = x86_ldq_phys(cs, pml5e_addr);
 413                 if (!(pml5e & PG_PRESENT_MASK)) {
 414                     goto do_fault;
 415                 }
 416                 if (pml5e & (rsvd_mask | PG_PSE_MASK)) {
 417                     goto do_fault_rsvd;
 418                 }
 419                 if (!(pml5e & PG_ACCESSED_MASK)) {
 420                     pml5e |= PG_ACCESSED_MASK;
 421                     x86_stl_phys_notdirty(cs, pml5e_addr, pml5e);
 422                 }
 423                 ptep = pml5e ^ PG_NX_MASK;
 424             } else {
 425                 pml5e = env->cr[3];
 426                 ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
 427             }
 428
 429             pml4e_addr = ((pml5e & PG_ADDRESS_MASK) +
 430                     (((addr >> 39) & 0x1ff) << 3)) & a20_mask;
 431             pml4e_addr = get_hphys(cs, pml4e_addr, MMU_DATA_STORE, false);
 432             pml4e = x86_ldq_phys(cs, pml4e_addr);
 433             if (!(pml4e & PG_PRESENT_MASK)) {
 434                 goto do_fault;
 435             }
 436             if (pml4e & (rsvd_mask | PG_PSE_MASK)) {
 437                 goto do_fault_rsvd;
 438             }
 439             if (!(pml4e & PG_ACCESSED_MASK)) {
 440                 pml4e |= PG_ACCESSED_MASK;
 441                 x86_stl_phys_notdirty(cs, pml4e_addr, pml4e);
 442             }
 443             ptep &= pml4e ^ PG_NX_MASK;
 444             pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) &
 445                 a20_mask;
 446             pdpe_addr = get_hphys(cs, pdpe_addr, MMU_DATA_STORE, NULL);
 447             pdpe = x86_ldq_phys(cs, pdpe_addr);
 448             if (!(pdpe & PG_PRESENT_MASK)) {
 449                 goto do_fault;
 450             }
 451             if (pdpe & rsvd_mask) {
 452                 goto do_fault_rsvd;
 453             }
 454             ptep &= pdpe ^ PG_NX_MASK;
 455             if (!(pdpe & PG_ACCESSED_MASK)) {
 456                 pdpe |= PG_ACCESSED_MASK;
 457                 x86_stl_phys_notdirty(cs, pdpe_addr, pdpe);
 458             }
 459             if (pdpe & PG_PSE_MASK) {
 460                 /* 1 GB page */
 461                 page_size = 1024 * 1024 * 1024;
 462                 pte_addr = pdpe_addr;
 463                 pte = pdpe;
 464                 goto do_check_protect;
 465             }
 466         } else
 467 #endif
 468         {
 469             /* XXX: load them when cr3 is loaded ? */
 470             pdpe_addr = ((env->cr[3] & ~0x1f) + ((addr >> 27) & 0x18)) &
 471                 a20_mask;
 472             pdpe_addr = get_hphys(cs, pdpe_addr, MMU_DATA_STORE, false);
 473             pdpe = x86_ldq_phys(cs, pdpe_addr);
 474             if (!(pdpe & PG_PRESENT_MASK)) {
 475                 goto do_fault;
 476             }
 477             rsvd_mask |= PG_HI_USER_MASK;
 478             if (pdpe & (rsvd_mask | PG_NX_MASK)) {
 479                 goto do_fault_rsvd;
 480             }
 481             ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
 482         }
 483
 484         pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) &
 485             a20_mask;
 486         pde_addr = get_hphys(cs, pde_addr, MMU_DATA_STORE, NULL);
 487         pde = x86_ldq_phys(cs, pde_addr);
 488         if (!(pde & PG_PRESENT_MASK)) {
 489             goto do_fault;
 490         }
 491         if (pde & rsvd_mask) {
 492             goto do_fault_rsvd;
 493         }
 494         ptep &= pde ^ PG_NX_MASK;
 495         if (pde & PG_PSE_MASK) {
 496             /* 2 MB page */
 497             page_size = 2048 * 1024;
 498             pte_addr = pde_addr;
 499             pte = pde;
 500             goto do_check_protect;
 501         }
 502         /* 4 KB page */
 503         if (!(pde & PG_ACCESSED_MASK)) {
 504             pde |= PG_ACCESSED_MASK;
 505             x86_stl_phys_notdirty(cs, pde_addr, pde);
 506         }
 507         pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) &
 508             a20_mask;
 509         pte_addr = get_hphys(cs, pte_addr, MMU_DATA_STORE, NULL);
 510         pte = x86_ldq_phys(cs, pte_addr);
 511         if (!(pte & PG_PRESENT_MASK)) {
 512             goto do_fault;
 513         }
 514         if (pte & rsvd_mask) {
 515             goto do_fault_rsvd;
 516         }
 517         /* combine pde and pte nx, user and rw protections */
 518         ptep &= pte ^ PG_NX_MASK;
 519         page_size = 4096;
 520     } else {
 521         uint32_t pde;
 522
 523         /* page directory entry */
 524         pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & 0xffc)) &
 525             a20_mask;
 526         pde_addr = get_hphys(cs, pde_addr, MMU_DATA_STORE, NULL);
 527         pde = x86_ldl_phys(cs, pde_addr);
 528         if (!(pde & PG_PRESENT_MASK)) {
 529             goto do_fault;
 530         }
 531         ptep = pde | PG_NX_MASK;
 532
 533         /* if PSE bit is set, then we use a 4MB page */
 534         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
 535             page_size = 4096 * 1024;
 536             pte_addr = pde_addr;
 537
 538             /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
 539              * Leave bits 20-13 in place for setting accessed/dirty bits below.
 540              */
 541             pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
 542             rsvd_mask = 0x200000;
 543             goto do_check_protect_pse36;
 544         }
 545
 546         if (!(pde & PG_ACCESSED_MASK)) {
 547             pde |= PG_ACCESSED_MASK;
 548             x86_stl_phys_notdirty(cs, pde_addr, pde);
 549         }
 550
 551         /* page directory entry */
 552         pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) &
 553             a20_mask;
 554         pte_addr = get_hphys(cs, pte_addr, MMU_DATA_STORE, NULL);
 555         pte = x86_ldl_phys(cs, pte_addr);
 556         if (!(pte & PG_PRESENT_MASK)) {
 557             goto do_fault;
 558         }
 559         /* combine pde and pte user and rw protections */
 560         ptep &= pte | PG_NX_MASK;
 561         page_size = 4096;
 562         rsvd_mask = 0;
 563     }
 564
 565 do_check_protect:
 566     rsvd_mask |= (page_size - 1) & PG_ADDRESS_MASK & ~PG_PSE_PAT_MASK;
 567 do_check_protect_pse36:
 568     if (pte & rsvd_mask) {
 569         goto do_fault_rsvd;
 570     }
 571     ptep ^= PG_NX_MASK;
 572
 573     /* can the page can be put in the TLB?  prot will tell us */
 574     if (is_user && !(ptep & PG_USER_MASK)) {
 575         goto do_fault_protect;
 576     }
 577
 578     prot = 0;
 579     if (mmu_idx != MMU_KSMAP_IDX || !(ptep & PG_USER_MASK)) {
 580         prot |= PAGE_READ;
 581         if ((ptep & PG_RW_MASK) || (!is_user && !(env->cr[0] & CR0_WP_MASK))) {
 582             prot |= PAGE_WRITE;
 583         }
 584     }
 585     if (!(ptep & PG_NX_MASK) &&
 586         (mmu_idx == MMU_USER_IDX ||
 587          !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK)))) {
 588         prot |= PAGE_EXEC;
 589     }
 590     if ((env->cr[4] & CR4_PKE_MASK) && (env->hflags & HF_LMA_MASK) &&
 591         (ptep & PG_USER_MASK) && env->pkru) {
 592         uint32_t pk = (pte & PG_PKRU_MASK) >> PG_PKRU_BIT;
 593         uint32_t pkru_ad = (env->pkru >> pk * 2) & 1;
 594         uint32_t pkru_wd = (env->pkru >> pk * 2) & 2;
 595         uint32_t pkru_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 596
 597         if (pkru_ad) {
 598             pkru_prot &= ~(PAGE_READ | PAGE_WRITE);
 599         } else if (pkru_wd && (is_user || env->cr[0] & CR0_WP_MASK)) {
 600             pkru_prot &= ~PAGE_WRITE;
 601         }
 602
 603         prot &= pkru_prot;
 604         if ((pkru_prot & (1 << is_write1)) == 0) {
 605             assert(is_write1 != 2);
 606             error_code |= PG_ERROR_PK_MASK;
 607             goto do_fault_protect;
 608         }
 609     }
 610
 611     if ((prot & (1 << is_write1)) == 0) {
 612         goto do_fault_protect;
 613     }
 614
 615     /* yes, it can! */
 616     is_dirty = is_write && !(pte & PG_DIRTY_MASK);
 617     if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
 618         pte |= PG_ACCESSED_MASK;
 619         if (is_dirty) {
 620             pte |= PG_DIRTY_MASK;
 621         }
 622         x86_stl_phys_notdirty(cs, pte_addr, pte);
 623     }
 624
 625     if (!(pte & PG_DIRTY_MASK)) {
 626         /* only set write access if already dirty... otherwise wait
 627            for dirty access */
 628         assert(!is_write);
 629         prot &= ~PAGE_WRITE;
 630     }
 631
 632  do_mapping:
 633     pte = pte & a20_mask;
 634
 635     /* align to page_size */
 636     pte &= PG_ADDRESS_MASK & ~(page_size - 1);
 637     page_offset = addr & (page_size - 1);
 638     paddr = get_hphys(cs, pte + page_offset, is_write1, &prot);
 639
 640     /* Even if 4MB pages, we map only one 4KB page in the cache to
 641        avoid filling it too fast */
 642     vaddr = addr & TARGET_PAGE_MASK;
 643     paddr &= TARGET_PAGE_MASK;
 644
 645     assert(prot & (1 << is_write1));
 646     tlb_set_page_with_attrs(cs, vaddr, paddr, cpu_get_mem_attrs(env),
 647                             prot, mmu_idx, page_size);
 648     return 0;
 649  do_fault_rsvd:
 650     error_code |= PG_ERROR_RSVD_MASK;
 651  do_fault_protect:
 652     error_code |= PG_ERROR_P_MASK;
 653  do_fault:
 654     error_code |= (is_write << PG_ERROR_W_BIT);
 655     if (is_user)
 656         error_code |= PG_ERROR_U_MASK;
 657     if (is_write1 == 2 &&
 658         (((env->efer & MSR_EFER_NXE) &&
 659           (env->cr[4] & CR4_PAE_MASK)) ||
 660          (env->cr[4] & CR4_SMEP_MASK)))
 661         error_code |= PG_ERROR_I_D_MASK;
 662     if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
 663         /* cr2 is not modified in case of exceptions */
 664         x86_stq_phys(cs,
 665                  env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
 666                  addr);
 667     } else {
 668         env->cr[2] = addr;
 669     }
 670     env->error_code = error_code;
 671     cs->exception_index = EXCP0E_PAGE;
 672     return 1;
 673 }
 674 #endif
 675
 676 bool x86_cpu_tlb_fill(CPUState *cs, vaddr addr, int size,
 677                       MMUAccessType access_type, int mmu_idx,
 678                       bool probe, uintptr_t retaddr)
 679 {
 680     X86CPU *cpu = X86_CPU(cs);
 681     CPUX86State *env = &cpu->env;
 682
 683 #ifdef CONFIG_USER_ONLY
 684     /* user mode only emulation */
 685     env->cr[2] = addr;
 686     env->error_code = (access_type == MMU_DATA_STORE) << PG_ERROR_W_BIT;
 687     env->error_code |= PG_ERROR_U_MASK;
 688     cs->exception_index = EXCP0E_PAGE;
 689     env->exception_is_int = 0;
 690     env->exception_next_eip = -1;
 691     cpu_loop_exit_restore(cs, retaddr);
 692 #else
 693     env->retaddr = retaddr;
 694     if (handle_mmu_fault(cs, addr, size, access_type, mmu_idx)) {
 695         /* FIXME: On error in get_hphys we have already jumped out.  */
 696         g_assert(!probe);
 697         raise_exception_err_ra(env, cs->exception_index,
 698                                env->error_code, retaddr);
 699     }
 700     return true;
 701 #endif
 702 }