accel/tcg/user-exec.c

   1 /*
   2  *  User emulator execution
   3  *
   4  *  Copyright (c) 2003-2005 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "disas/disas.h"
  22 #include "exec/exec-all.h"
  23 #include "tcg.h"
  24 #include "qemu/bitops.h"
  25 #include "exec/cpu_ldst.h"
  26 #include "translate-all.h"
  27 #include "exec/helper-proto.h"
  28
  29 #undef EAX
  30 #undef ECX
  31 #undef EDX
  32 #undef EBX
  33 #undef ESP
  34 #undef EBP
  35 #undef ESI
  36 #undef EDI
  37 #undef EIP
  38 #ifdef __linux__
  39 #include <sys/ucontext.h>
  40 #endif
  41
  42 __thread uintptr_t helper_retaddr;
  43
  44 //#define DEBUG_SIGNAL
  45
  46 /* exit the current TB from a signal handler. The host registers are
  47    restored in a state compatible with the CPU emulator
  48  */
  49 static void cpu_exit_tb_from_sighandler(CPUState *cpu, sigset_t *old_set)
  50 {
  51     /* XXX: use siglongjmp ? */
  52     sigprocmask(SIG_SETMASK, old_set, NULL);
  53     cpu_loop_exit_noexc(cpu);
  54 }
  55
  56 /* 'pc' is the host PC at which the exception was raised. 'address' is
  57    the effective address of the memory exception. 'is_write' is 1 if a
  58    write caused the exception and otherwise 0'. 'old_set' is the
  59    signal set which should be restored */
  60 static inline int handle_cpu_signal(uintptr_t pc, siginfo_t *info,
  61                                     int is_write, sigset_t *old_set)
  62 {
  63     CPUState *cpu = current_cpu;
  64     CPUClass *cc;
  65     int ret;
  66     unsigned long address = (unsigned long)info->si_addr;
  67
  68     /* We must handle PC addresses from two different sources:
  69      * a call return address and a signal frame address.
  70      *
  71      * Within cpu_restore_state_from_tb we assume the former and adjust
  72      * the address by -GETPC_ADJ so that the address is within the call
  73      * insn so that addr does not accidentally match the beginning of the
  74      * next guest insn.
  75      *
  76      * However, when the PC comes from the signal frame, it points to
  77      * the actual faulting host insn and not a call insn.  Subtracting
  78      * GETPC_ADJ in that case may accidentally match the previous guest insn.
  79      *
  80      * So for the later case, adjust forward to compensate for what
  81      * will be done later by cpu_restore_state_from_tb.
  82      */
  83     if (helper_retaddr) {
  84         pc = helper_retaddr;
  85     } else {
  86         pc += GETPC_ADJ;
  87     }
  88
  89     /* For synchronous signals we expect to be coming from the vCPU
  90      * thread (so current_cpu should be valid) and either from running
  91      * code or during translation which can fault as we cross pages.
  92      *
  93      * If neither is true then something has gone wrong and we should
  94      * abort rather than try and restart the vCPU execution.
  95      */
  96     if (!cpu || !cpu->running) {
  97         printf("qemu:%s received signal outside vCPU context @ pc=0x%"
  98                PRIxPTR "\n",  __func__, pc);
  99         abort();
 100     }
 101
 102 #if defined(DEBUG_SIGNAL)
 103     printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
 104            pc, address, is_write, *(unsigned long *)old_set);
 105 #endif
 106     /* XXX: locking issue */
 107     /* Note that it is important that we don't call page_unprotect() unless
 108      * this is really a "write to nonwriteable page" fault, because
 109      * page_unprotect() assumes that if it is called for an access to
 110      * a page that's writeable this means we had two threads racing and
 111      * another thread got there first and already made the page writeable;
 112      * so we will retry the access. If we were to call page_unprotect()
 113      * for some other kind of fault that should really be passed to the
 114      * guest, we'd end up in an infinite loop of retrying the faulting
 115      * access.
 116      */
 117     if (is_write && info->si_signo == SIGSEGV && info->si_code == SEGV_ACCERR &&
 118         h2g_valid(address)) {
 119         switch (page_unprotect(h2g(address), pc)) {
 120         case 0:
 121             /* Fault not caused by a page marked unwritable to protect
 122              * cached translations, must be the guest binary's problem.
 123              */
 124             break;
 125         case 1:
 126             /* Fault caused by protection of cached translation; TBs
 127              * invalidated, so resume execution.  Retain helper_retaddr
 128              * for a possible second fault.
 129              */
 130             return 1;
 131         case 2:
 132             /* Fault caused by protection of cached translation, and the
 133              * currently executing TB was modified and must be exited
 134              * immediately.  Clear helper_retaddr for next execution.
 135              */
 136             helper_retaddr = 0;
 137             cpu_exit_tb_from_sighandler(cpu, old_set);
 138             /* NORETURN */
 139
 140         default:
 141             g_assert_not_reached();
 142         }
 143     }
 144
 145     /* Convert forcefully to guest address space, invalid addresses
 146        are still valid segv ones */
 147     address = h2g_nocheck(address);
 148
 149     cc = CPU_GET_CLASS(cpu);
 150     /* see if it is an MMU fault */
 151     g_assert(cc->handle_mmu_fault);
 152     ret = cc->handle_mmu_fault(cpu, address, 0, is_write, MMU_USER_IDX);
 153
 154     if (ret == 0) {
 155         /* The MMU fault was handled without causing real CPU fault.
 156          *  Retain helper_retaddr for a possible second fault.
 157          */
 158         return 1;
 159     }
 160
 161     /* All other paths lead to cpu_exit; clear helper_retaddr
 162      * for next execution.
 163      */
 164     helper_retaddr = 0;
 165
 166     if (ret < 0) {
 167         return 0; /* not an MMU fault */
 168     }
 169
 170     /* Now we have a real cpu fault.  */
 171     cpu_restore_state(cpu, pc, true);
 172
 173     sigprocmask(SIG_SETMASK, old_set, NULL);
 174     cpu_loop_exit(cpu);
 175
 176     /* never comes here */
 177     return 1;
 178 }
 179
 180 #if defined(__i386__)
 181
 182 #if defined(__NetBSD__)
 183 #include <ucontext.h>
 184
 185 #define EIP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_EIP])
 186 #define TRAP_sig(context)    ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 187 #define ERROR_sig(context)   ((context)->uc_mcontext.__gregs[_REG_ERR])
 188 #define MASK_sig(context)    ((context)->uc_sigmask)
 189 #elif defined(__FreeBSD__) || defined(__DragonFly__)
 190 #include <ucontext.h>
 191
 192 #define EIP_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_eip))
 193 #define TRAP_sig(context)    ((context)->uc_mcontext.mc_trapno)
 194 #define ERROR_sig(context)   ((context)->uc_mcontext.mc_err)
 195 #define MASK_sig(context)    ((context)->uc_sigmask)
 196 #elif defined(__OpenBSD__)
 197 #define EIP_sig(context)     ((context)->sc_eip)
 198 #define TRAP_sig(context)    ((context)->sc_trapno)
 199 #define ERROR_sig(context)   ((context)->sc_err)
 200 #define MASK_sig(context)    ((context)->sc_mask)
 201 #else
 202 #define EIP_sig(context)     ((context)->uc_mcontext.gregs[REG_EIP])
 203 #define TRAP_sig(context)    ((context)->uc_mcontext.gregs[REG_TRAPNO])
 204 #define ERROR_sig(context)   ((context)->uc_mcontext.gregs[REG_ERR])
 205 #define MASK_sig(context)    ((context)->uc_sigmask)
 206 #endif
 207
 208 int cpu_signal_handler(int host_signum, void *pinfo,
 209                        void *puc)
 210 {
 211     siginfo_t *info = pinfo;
 212 #if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 213     ucontext_t *uc = puc;
 214 #elif defined(__OpenBSD__)
 215     struct sigcontext *uc = puc;
 216 #else
 217     ucontext_t *uc = puc;
 218 #endif
 219     unsigned long pc;
 220     int trapno;
 221
 222 #ifndef REG_EIP
 223 /* for glibc 2.1 */
 224 #define REG_EIP    EIP
 225 #define REG_ERR    ERR
 226 #define REG_TRAPNO TRAPNO
 227 #endif
 228     pc = EIP_sig(uc);
 229     trapno = TRAP_sig(uc);
 230     return handle_cpu_signal(pc, info,
 231                              trapno == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 232                              &MASK_sig(uc));
 233 }
 234
 235 #elif defined(__x86_64__)
 236
 237 #ifdef __NetBSD__
 238 #define PC_sig(context)       _UC_MACHINE_PC(context)
 239 #define TRAP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_TRAPNO])
 240 #define ERROR_sig(context)    ((context)->uc_mcontext.__gregs[_REG_ERR])
 241 #define MASK_sig(context)     ((context)->uc_sigmask)
 242 #elif defined(__OpenBSD__)
 243 #define PC_sig(context)       ((context)->sc_rip)
 244 #define TRAP_sig(context)     ((context)->sc_trapno)
 245 #define ERROR_sig(context)    ((context)->sc_err)
 246 #define MASK_sig(context)     ((context)->sc_mask)
 247 #elif defined(__FreeBSD__) || defined(__DragonFly__)
 248 #include <ucontext.h>
 249
 250 #define PC_sig(context)  (*((unsigned long *)&(context)->uc_mcontext.mc_rip))
 251 #define TRAP_sig(context)     ((context)->uc_mcontext.mc_trapno)
 252 #define ERROR_sig(context)    ((context)->uc_mcontext.mc_err)
 253 #define MASK_sig(context)     ((context)->uc_sigmask)
 254 #else
 255 #define PC_sig(context)       ((context)->uc_mcontext.gregs[REG_RIP])
 256 #define TRAP_sig(context)     ((context)->uc_mcontext.gregs[REG_TRAPNO])
 257 #define ERROR_sig(context)    ((context)->uc_mcontext.gregs[REG_ERR])
 258 #define MASK_sig(context)     ((context)->uc_sigmask)
 259 #endif
 260
 261 int cpu_signal_handler(int host_signum, void *pinfo,
 262                        void *puc)
 263 {
 264     siginfo_t *info = pinfo;
 265     unsigned long pc;
 266 #if defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__)
 267     ucontext_t *uc = puc;
 268 #elif defined(__OpenBSD__)
 269     struct sigcontext *uc = puc;
 270 #else
 271     ucontext_t *uc = puc;
 272 #endif
 273
 274     pc = PC_sig(uc);
 275     return handle_cpu_signal(pc, info,
 276                              TRAP_sig(uc) == 0xe ? (ERROR_sig(uc) >> 1) & 1 : 0,
 277                              &MASK_sig(uc));
 278 }
 279
 280 #elif defined(_ARCH_PPC)
 281
 282 /***********************************************************************
 283  * signal context platform-specific definitions
 284  * From Wine
 285  */
 286 #ifdef linux
 287 /* All Registers access - only for local access */
 288 #define REG_sig(reg_name, context)              \
 289     ((context)->uc_mcontext.regs->reg_name)
 290 /* Gpr Registers access  */
 291 #define GPR_sig(reg_num, context)              REG_sig(gpr[reg_num], context)
 292 /* Program counter */
 293 #define IAR_sig(context)                       REG_sig(nip, context)
 294 /* Machine State Register (Supervisor) */
 295 #define MSR_sig(context)                       REG_sig(msr, context)
 296 /* Count register */
 297 #define CTR_sig(context)                       REG_sig(ctr, context)
 298 /* User's integer exception register */
 299 #define XER_sig(context)                       REG_sig(xer, context)
 300 /* Link register */
 301 #define LR_sig(context)                        REG_sig(link, context)
 302 /* Condition register */
 303 #define CR_sig(context)                        REG_sig(ccr, context)
 304
 305 /* Float Registers access  */
 306 #define FLOAT_sig(reg_num, context)                                     \
 307     (((double *)((char *)((context)->uc_mcontext.regs + 48 * 4)))[reg_num])
 308 #define FPSCR_sig(context) \
 309     (*(int *)((char *)((context)->uc_mcontext.regs + (48 + 32 * 2) * 4)))
 310 /* Exception Registers access */
 311 #define DAR_sig(context)                       REG_sig(dar, context)
 312 #define DSISR_sig(context)                     REG_sig(dsisr, context)
 313 #define TRAP_sig(context)                      REG_sig(trap, context)
 314 #endif /* linux */
 315
 316 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 317 #include <ucontext.h>
 318 #define IAR_sig(context)               ((context)->uc_mcontext.mc_srr0)
 319 #define MSR_sig(context)               ((context)->uc_mcontext.mc_srr1)
 320 #define CTR_sig(context)               ((context)->uc_mcontext.mc_ctr)
 321 #define XER_sig(context)               ((context)->uc_mcontext.mc_xer)
 322 #define LR_sig(context)                ((context)->uc_mcontext.mc_lr)
 323 #define CR_sig(context)                ((context)->uc_mcontext.mc_cr)
 324 /* Exception Registers access */
 325 #define DAR_sig(context)               ((context)->uc_mcontext.mc_dar)
 326 #define DSISR_sig(context)             ((context)->uc_mcontext.mc_dsisr)
 327 #define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
 328 #endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 329
 330 int cpu_signal_handler(int host_signum, void *pinfo,
 331                        void *puc)
 332 {
 333     siginfo_t *info = pinfo;
 334 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 335     ucontext_t *uc = puc;
 336 #else
 337     ucontext_t *uc = puc;
 338 #endif
 339     unsigned long pc;
 340     int is_write;
 341
 342     pc = IAR_sig(uc);
 343     is_write = 0;
 344 #if 0
 345     /* ppc 4xx case */
 346     if (DSISR_sig(uc) & 0x00800000) {
 347         is_write = 1;
 348     }
 349 #else
 350     if (TRAP_sig(uc) != 0x400 && (DSISR_sig(uc) & 0x02000000)) {
 351         is_write = 1;
 352     }
 353 #endif
 354     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 355 }
 356
 357 #elif defined(__alpha__)
 358
 359 int cpu_signal_handler(int host_signum, void *pinfo,
 360                            void *puc)
 361 {
 362     siginfo_t *info = pinfo;
 363     ucontext_t *uc = puc;
 364     uint32_t *pc = uc->uc_mcontext.sc_pc;
 365     uint32_t insn = *pc;
 366     int is_write = 0;
 367
 368     /* XXX: need kernel patch to get write flag faster */
 369     switch (insn >> 26) {
 370     case 0x0d: /* stw */
 371     case 0x0e: /* stb */
 372     case 0x0f: /* stq_u */
 373     case 0x24: /* stf */
 374     case 0x25: /* stg */
 375     case 0x26: /* sts */
 376     case 0x27: /* stt */
 377     case 0x2c: /* stl */
 378     case 0x2d: /* stq */
 379     case 0x2e: /* stl_c */
 380     case 0x2f: /* stq_c */
 381         is_write = 1;
 382     }
 383
 384     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 385 }
 386 #elif defined(__sparc__)
 387
 388 int cpu_signal_handler(int host_signum, void *pinfo,
 389                        void *puc)
 390 {
 391     siginfo_t *info = pinfo;
 392     int is_write;
 393     uint32_t insn;
 394 #if !defined(__arch64__) || defined(CONFIG_SOLARIS)
 395     uint32_t *regs = (uint32_t *)(info + 1);
 396     void *sigmask = (regs + 20);
 397     /* XXX: is there a standard glibc define ? */
 398     unsigned long pc = regs[1];
 399 #else
 400 #ifdef __linux__
 401     struct sigcontext *sc = puc;
 402     unsigned long pc = sc->sigc_regs.tpc;
 403     void *sigmask = (void *)sc->sigc_mask;
 404 #elif defined(__OpenBSD__)
 405     struct sigcontext *uc = puc;
 406     unsigned long pc = uc->sc_pc;
 407     void *sigmask = (void *)(long)uc->sc_mask;
 408 #elif defined(__NetBSD__)
 409     ucontext_t *uc = puc;
 410     unsigned long pc = _UC_MACHINE_PC(uc);
 411     void *sigmask = (void *)&uc->uc_sigmask;
 412 #endif
 413 #endif
 414
 415     /* XXX: need kernel patch to get write flag faster */
 416     is_write = 0;
 417     insn = *(uint32_t *)pc;
 418     if ((insn >> 30) == 3) {
 419         switch ((insn >> 19) & 0x3f) {
 420         case 0x05: /* stb */
 421         case 0x15: /* stba */
 422         case 0x06: /* sth */
 423         case 0x16: /* stha */
 424         case 0x04: /* st */
 425         case 0x14: /* sta */
 426         case 0x07: /* std */
 427         case 0x17: /* stda */
 428         case 0x0e: /* stx */
 429         case 0x1e: /* stxa */
 430         case 0x24: /* stf */
 431         case 0x34: /* stfa */
 432         case 0x27: /* stdf */
 433         case 0x37: /* stdfa */
 434         case 0x26: /* stqf */
 435         case 0x36: /* stqfa */
 436         case 0x25: /* stfsr */
 437         case 0x3c: /* casa */
 438         case 0x3e: /* casxa */
 439             is_write = 1;
 440             break;
 441         }
 442     }
 443     return handle_cpu_signal(pc, info, is_write, sigmask);
 444 }
 445
 446 #elif defined(__arm__)
 447
 448 #if defined(__NetBSD__)
 449 #include <ucontext.h>
 450 #endif
 451
 452 int cpu_signal_handler(int host_signum, void *pinfo,
 453                        void *puc)
 454 {
 455     siginfo_t *info = pinfo;
 456 #if defined(__NetBSD__)
 457     ucontext_t *uc = puc;
 458 #else
 459     ucontext_t *uc = puc;
 460 #endif
 461     unsigned long pc;
 462     int is_write;
 463
 464 #if defined(__NetBSD__)
 465     pc = uc->uc_mcontext.__gregs[_REG_R15];
 466 #elif defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
 467     pc = uc->uc_mcontext.gregs[R15];
 468 #else
 469     pc = uc->uc_mcontext.arm_pc;
 470 #endif
 471
 472     /* error_code is the FSR value, in which bit 11 is WnR (assuming a v6 or
 473      * later processor; on v5 we will always report this as a read).
 474      */
 475     is_write = extract32(uc->uc_mcontext.error_code, 11, 1);
 476     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 477 }
 478
 479 #elif defined(__aarch64__)
 480
 481 int cpu_signal_handler(int host_signum, void *pinfo, void *puc)
 482 {
 483     siginfo_t *info = pinfo;
 484     ucontext_t *uc = puc;
 485     uintptr_t pc = uc->uc_mcontext.pc;
 486     uint32_t insn = *(uint32_t *)pc;
 487     bool is_write;
 488
 489     /* XXX: need kernel patch to get write flag faster.  */
 490     is_write = (   (insn & 0xbfff0000) == 0x0c000000   /* C3.3.1 */
 491                 || (insn & 0xbfe00000) == 0x0c800000   /* C3.3.2 */
 492                 || (insn & 0xbfdf0000) == 0x0d000000   /* C3.3.3 */
 493                 || (insn & 0xbfc00000) == 0x0d800000   /* C3.3.4 */
 494                 || (insn & 0x3f400000) == 0x08000000   /* C3.3.6 */
 495                 || (insn & 0x3bc00000) == 0x39000000   /* C3.3.13 */
 496                 || (insn & 0x3fc00000) == 0x3d800000   /* ... 128bit */
 497                 /* Ingore bits 10, 11 & 21, controlling indexing.  */
 498                 || (insn & 0x3bc00000) == 0x38000000   /* C3.3.8-12 */
 499                 || (insn & 0x3fe00000) == 0x3c800000   /* ... 128bit */
 500                 /* Ignore bits 23 & 24, controlling indexing.  */
 501                 || (insn & 0x3a400000) == 0x28000000); /* C3.3.7,14-16 */
 502
 503     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 504 }
 505
 506 #elif defined(__s390__)
 507
 508 int cpu_signal_handler(int host_signum, void *pinfo,
 509                        void *puc)
 510 {
 511     siginfo_t *info = pinfo;
 512     ucontext_t *uc = puc;
 513     unsigned long pc;
 514     uint16_t *pinsn;
 515     int is_write = 0;
 516
 517     pc = uc->uc_mcontext.psw.addr;
 518
 519     /* ??? On linux, the non-rt signal handler has 4 (!) arguments instead
 520        of the normal 2 arguments.  The 3rd argument contains the "int_code"
 521        from the hardware which does in fact contain the is_write value.
 522        The rt signal handler, as far as I can tell, does not give this value
 523        at all.  Not that we could get to it from here even if it were.  */
 524     /* ??? This is not even close to complete, since it ignores all
 525        of the read-modify-write instructions.  */
 526     pinsn = (uint16_t *)pc;
 527     switch (pinsn[0] >> 8) {
 528     case 0x50: /* ST */
 529     case 0x42: /* STC */
 530     case 0x40: /* STH */
 531         is_write = 1;
 532         break;
 533     case 0xc4: /* RIL format insns */
 534         switch (pinsn[0] & 0xf) {
 535         case 0xf: /* STRL */
 536         case 0xb: /* STGRL */
 537         case 0x7: /* STHRL */
 538             is_write = 1;
 539         }
 540         break;
 541     case 0xe3: /* RXY format insns */
 542         switch (pinsn[2] & 0xff) {
 543         case 0x50: /* STY */
 544         case 0x24: /* STG */
 545         case 0x72: /* STCY */
 546         case 0x70: /* STHY */
 547         case 0x8e: /* STPQ */
 548         case 0x3f: /* STRVH */
 549         case 0x3e: /* STRV */
 550         case 0x2f: /* STRVG */
 551             is_write = 1;
 552         }
 553         break;
 554     }
 555     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 556 }
 557
 558 #elif defined(__mips__)
 559
 560 int cpu_signal_handler(int host_signum, void *pinfo,
 561                        void *puc)
 562 {
 563     siginfo_t *info = pinfo;
 564     ucontext_t *uc = puc;
 565     greg_t pc = uc->uc_mcontext.pc;
 566     int is_write;
 567
 568     /* XXX: compute is_write */
 569     is_write = 0;
 570     return handle_cpu_signal(pc, info, is_write, &uc->uc_sigmask);
 571 }
 572
 573 #else
 574
 575 #error host CPU specific signal handler needed
 576
 577 #endif
 578
 579 /* The softmmu versions of these helpers are in cputlb.c.  */
 580
 581 /* Do not allow unaligned operations to proceed.  Return the host address.  */
 582 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
 583                                int size, uintptr_t retaddr)
 584 {
 585     /* Enforce qemu required alignment.  */
 586     if (unlikely(addr & (size - 1))) {
 587         cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
 588     }
 589     helper_retaddr = retaddr;
 590     return g2h(addr);
 591 }
 592
 593 /* Macro to call the above, with local variables from the use context.  */
 594 #define ATOMIC_MMU_DECLS do {} while (0)
 595 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, GETPC())
 596 #define ATOMIC_MMU_CLEANUP do { helper_retaddr = 0; } while (0)
 597
 598 #define ATOMIC_NAME(X)   HELPER(glue(glue(atomic_ ## X, SUFFIX), END))
 599 #define EXTRA_ARGS
 600
 601 #define DATA_SIZE 1
 602 #include "atomic_template.h"
 603
 604 #define DATA_SIZE 2
 605 #include "atomic_template.h"
 606
 607 #define DATA_SIZE 4
 608 #include "atomic_template.h"
 609
 610 #ifdef CONFIG_ATOMIC64
 611 #define DATA_SIZE 8
 612 #include "atomic_template.h"
 613 #endif
 614
 615 /* The following is only callable from other helpers, and matches up
 616    with the softmmu version.  */
 617
 618 #ifdef CONFIG_ATOMIC128
 619
 620 #undef EXTRA_ARGS
 621 #undef ATOMIC_NAME
 622 #undef ATOMIC_MMU_LOOKUP
 623
 624 #define EXTRA_ARGS     , TCGMemOpIdx oi, uintptr_t retaddr
 625 #define ATOMIC_NAME(X) \
 626     HELPER(glue(glue(glue(atomic_ ## X, SUFFIX), END), _mmu))
 627 #define ATOMIC_MMU_LOOKUP  atomic_mmu_lookup(env, addr, DATA_SIZE, retaddr)
 628
 629 #define DATA_SIZE 16
 630 #include "atomic_template.h"
 631 #endif /* CONFIG_ATOMIC128 */