qemu/exec.c

   1 /*
   2  *  virtual page mapping and translated block handling
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20 #include "config.h"
  21 #ifdef _WIN32
  22 #define WIN32_LEAN_AND_MEAN
  23 #include <windows.h>
  24 #else
  25 #include <sys/types.h>
  26 #include <sys/mman.h>
  27 #endif
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <stdarg.h>
  31 #include <string.h>
  32 #include <errno.h>
  33 #include <unistd.h>
  34 #include <inttypes.h>
  35
  36 #include "cpu.h"
  37 #include "exec-all.h"
  38 #include "qemu-common.h"
  39
  40 #if !defined(TARGET_IA64)
  41 #include "tcg.h"
  42 #endif
  43 #include "qemu-kvm.h"
  44
  45 #include "hw/hw.h"
  46 #include "osdep.h"
  47 #include "kvm.h"
  48 #if defined(CONFIG_USER_ONLY)
  49 #include <qemu.h>
  50 #endif
  51
  52 //#define DEBUG_TB_INVALIDATE
  53 //#define DEBUG_FLUSH
  54 //#define DEBUG_TLB
  55 //#define DEBUG_UNASSIGNED
  56
  57 /* make various TB consistency checks */
  58 //#define DEBUG_TB_CHECK
  59 //#define DEBUG_TLB_CHECK
  60
  61 //#define DEBUG_IOPORT
  62 //#define DEBUG_SUBPAGE
  63
  64 #if !defined(CONFIG_USER_ONLY)
  65 /* TB consistency checks only implemented for usermode emulation.  */
  66 #undef DEBUG_TB_CHECK
  67 #endif
  68
  69 #define SMC_BITMAP_USE_THRESHOLD 10
  70
  71 #define MMAP_AREA_START        0x00000000
  72 #define MMAP_AREA_END          0xa8000000
  73
  74 #if defined(TARGET_SPARC64)
  75 #define TARGET_PHYS_ADDR_SPACE_BITS 41
  76 #elif defined(TARGET_SPARC)
  77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
  78 #elif defined(TARGET_ALPHA)
  79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
  80 #define TARGET_VIRT_ADDR_SPACE_BITS 42
  81 #elif defined(TARGET_PPC64)
  82 #define TARGET_PHYS_ADDR_SPACE_BITS 42
  83 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
  84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
  85 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
  86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
  87 #elif defined(TARGET_IA64)
  88 #define TARGET_PHYS_ADDR_SPACE_BITS 36
  89 #else
  90 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
  91 #define TARGET_PHYS_ADDR_SPACE_BITS 32
  92 #endif
  93
  94 static TranslationBlock *tbs;
  95 int code_gen_max_blocks;
  96 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
  97 static int nb_tbs;
  98 /* any access to the tbs or the page table must use this lock */
  99 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
 100
 101 #if defined(__arm__) || defined(__sparc_v9__)
 102 /* The prologue must be reachable with a direct jump. ARM and Sparc64
 103  have limited branch ranges (possibly also PPC) so place it in a
 104  section close to code segment. */
 105 #define code_gen_section                                \
 106     __attribute__((__section__(".gen_code")))           \
 107     __attribute__((aligned (32)))
 108 #else
 109 #define code_gen_section                                \
 110     __attribute__((aligned (32)))
 111 #endif
 112
 113 uint8_t code_gen_prologue[1024] code_gen_section;
 114 static uint8_t *code_gen_buffer;
 115 static unsigned long code_gen_buffer_size;
 116 /* threshold to flush the translated code buffer */
 117 static unsigned long code_gen_buffer_max_size;
 118 uint8_t *code_gen_ptr;
 119
 120 #if !defined(CONFIG_USER_ONLY)
 121 ram_addr_t phys_ram_size;
 122 int phys_ram_fd;
 123 uint8_t *phys_ram_base;
 124 uint8_t *phys_ram_dirty;
 125 uint8_t *bios_mem;
 126 static int in_migration;
 127 static ram_addr_t phys_ram_alloc_offset = 0;
 128 #endif
 129
 130 CPUState *first_cpu;
 131 /* current CPU in the current thread. It is only valid inside
 132    cpu_exec() */
 133 CPUState *cpu_single_env;
 134 /* 0 = Do not count executed instructions.
 135    1 = Precise instruction counting.
 136    2 = Adaptive rate instruction counting.  */
 137 int use_icount = 0;
 138 /* Current instruction counter.  While executing translated code this may
 139    include some instructions that have not yet been executed.  */
 140 int64_t qemu_icount;
 141
 142 typedef struct PageDesc {
 143     /* list of TBs intersecting this ram page */
 144     TranslationBlock *first_tb;
 145     /* in order to optimize self modifying code, we count the number
 146        of lookups we do to a given page to use a bitmap */
 147     unsigned int code_write_count;
 148     uint8_t *code_bitmap;
 149 #if defined(CONFIG_USER_ONLY)
 150     unsigned long flags;
 151 #endif
 152 } PageDesc;
 153
 154 typedef struct PhysPageDesc {
 155     /* offset in host memory of the page + io_index in the low bits */
 156     ram_addr_t phys_offset;
 157     ram_addr_t region_offset;
 158 } PhysPageDesc;
 159
 160 #define L2_BITS 10
 161 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
 162 /* XXX: this is a temporary hack for alpha target.
 163  *      In the future, this is to be replaced by a multi-level table
 164  *      to actually be able to handle the complete 64 bits address space.
 165  */
 166 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
 167 #else
 168 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
 169 #endif
 170
 171 #define L1_SIZE (1 << L1_BITS)
 172 #define L2_SIZE (1 << L2_BITS)
 173
 174 unsigned long qemu_real_host_page_size;
 175 unsigned long qemu_host_page_bits;
 176 unsigned long qemu_host_page_size;
 177 unsigned long qemu_host_page_mask;
 178
 179 /* XXX: for system emulation, it could just be an array */
 180 static PageDesc *l1_map[L1_SIZE];
 181 static PhysPageDesc **l1_phys_map;
 182
 183 #if !defined(CONFIG_USER_ONLY)
 184 static void io_mem_init(void);
 185
 186 /* io memory support */
 187 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
 188 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
 189 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
 190 char io_mem_used[IO_MEM_NB_ENTRIES];
 191 static int io_mem_watch;
 192 #endif
 193
 194 /* log support */
 195 static const char *logfilename = "/tmp/qemu.log";
 196 FILE *logfile;
 197 int loglevel;
 198 static int log_append = 0;
 199
 200 /* statistics */
 201 static int tlb_flush_count;
 202 static int tb_flush_count;
 203 static int tb_phys_invalidate_count;
 204
 205 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 206 typedef struct subpage_t {
 207     target_phys_addr_t base;
 208     CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
 209     CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
 210     void *opaque[TARGET_PAGE_SIZE][2][4];
 211     ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
 212 } subpage_t;
 213
 214 #ifdef _WIN32
 215 static void map_exec(void *addr, long size)
 216 {
 217     DWORD old_protect;
 218     VirtualProtect(addr, size,
 219                    PAGE_EXECUTE_READWRITE, &old_protect);
 220
 221 }
 222 #else
 223 static void map_exec(void *addr, long size)
 224 {
 225     unsigned long start, end, page_size;
 226
 227     page_size = getpagesize();
 228     start = (unsigned long)addr;
 229     start &= ~(page_size - 1);
 230
 231     end = (unsigned long)addr + size;
 232     end += page_size - 1;
 233     end &= ~(page_size - 1);
 234
 235     mprotect((void *)start, end - start,
 236              PROT_READ | PROT_WRITE | PROT_EXEC);
 237 }
 238 #endif
 239
 240 static void page_init(void)
 241 {
 242     /* NOTE: we can always suppose that qemu_host_page_size >=
 243        TARGET_PAGE_SIZE */
 244 #ifdef _WIN32
 245     {
 246         SYSTEM_INFO system_info;
 247
 248         GetSystemInfo(&system_info);
 249         qemu_real_host_page_size = system_info.dwPageSize;
 250     }
 251 #else
 252     qemu_real_host_page_size = getpagesize();
 253 #endif
 254     if (qemu_host_page_size == 0)
 255         qemu_host_page_size = qemu_real_host_page_size;
 256     if (qemu_host_page_size < TARGET_PAGE_SIZE)
 257         qemu_host_page_size = TARGET_PAGE_SIZE;
 258     qemu_host_page_bits = 0;
 259     while ((1 << qemu_host_page_bits) < qemu_host_page_size)
 260         qemu_host_page_bits++;
 261     qemu_host_page_mask = ~(qemu_host_page_size - 1);
 262     l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
 263     memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
 264
 265 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
 266     {
 267         long long startaddr, endaddr;
 268         FILE *f;
 269         int n;
 270
 271         mmap_lock();
 272         last_brk = (unsigned long)sbrk(0);
 273         f = fopen("/proc/self/maps", "r");
 274         if (f) {
 275             do {
 276                 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
 277                 if (n == 2) {
 278                     startaddr = MIN(startaddr,
 279                                     (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
 280                     endaddr = MIN(endaddr,
 281                                     (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
 282                     page_set_flags(startaddr & TARGET_PAGE_MASK,
 283                                    TARGET_PAGE_ALIGN(endaddr),
 284                                    PAGE_RESERVED);
 285                 }
 286             } while (!feof(f));
 287             fclose(f);
 288         }
 289         mmap_unlock();
 290     }
 291 #endif
 292 }
 293
 294 static inline PageDesc **page_l1_map(target_ulong index)
 295 {
 296 #if TARGET_LONG_BITS > 32
 297     /* Host memory outside guest VM.  For 32-bit targets we have already
 298        excluded high addresses.  */
 299     if (index > ((target_ulong)L2_SIZE * L1_SIZE))
 300         return NULL;
 301 #endif
 302     return &l1_map[index >> L2_BITS];
 303 }
 304
 305 static inline PageDesc *page_find_alloc(target_ulong index)
 306 {
 307     PageDesc **lp, *p;
 308     lp = page_l1_map(index);
 309     if (!lp)
 310         return NULL;
 311
 312     p = *lp;
 313     if (!p) {
 314         /* allocate if not found */
 315 #if defined(CONFIG_USER_ONLY)
 316         size_t len = sizeof(PageDesc) * L2_SIZE;
 317         /* Don't use qemu_malloc because it may recurse.  */
 318         p = mmap(0, len, PROT_READ | PROT_WRITE,
 319                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 320         *lp = p;
 321         if (h2g_valid(p)) {
 322             unsigned long addr = h2g(p);
 323             page_set_flags(addr & TARGET_PAGE_MASK,
 324                            TARGET_PAGE_ALIGN(addr + len),
 325                            PAGE_RESERVED);
 326         }
 327 #else
 328         p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
 329         *lp = p;
 330 #endif
 331     }
 332     return p + (index & (L2_SIZE - 1));
 333 }
 334
 335 static inline PageDesc *page_find(target_ulong index)
 336 {
 337     PageDesc **lp, *p;
 338     lp = page_l1_map(index);
 339     if (!lp)
 340         return NULL;
 341
 342     p = *lp;
 343     if (!p)
 344         return 0;
 345     return p + (index & (L2_SIZE - 1));
 346 }
 347
 348 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
 349 {
 350     void **lp, **p;
 351     PhysPageDesc *pd;
 352
 353     p = (void **)l1_phys_map;
 354 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
 355
 356 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
 357 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
 358 #endif
 359     lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
 360     p = *lp;
 361     if (!p) {
 362         /* allocate if not found */
 363         if (!alloc)
 364             return NULL;
 365         p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
 366         memset(p, 0, sizeof(void *) * L1_SIZE);
 367         *lp = p;
 368     }
 369 #endif
 370     lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
 371     pd = *lp;
 372     if (!pd) {
 373         int i;
 374         /* allocate if not found */
 375         if (!alloc)
 376             return NULL;
 377         pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
 378         *lp = pd;
 379         for (i = 0; i < L2_SIZE; i++)
 380           pd[i].phys_offset = IO_MEM_UNASSIGNED;
 381     }
 382     return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
 383 }
 384
 385 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
 386 {
 387     return phys_page_find_alloc(index, 0);
 388 }
 389
 390 #if !defined(CONFIG_USER_ONLY)
 391 static void tlb_protect_code(ram_addr_t ram_addr);
 392 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
 393                                     target_ulong vaddr);
 394 #define mmap_lock() do { } while(0)
 395 #define mmap_unlock() do { } while(0)
 396 #endif
 397
 398 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
 399
 400 #if defined(CONFIG_USER_ONLY)
 401 /* Currently it is not recommanded to allocate big chunks of data in
 402    user mode. It will change when a dedicated libc will be used */
 403 #define USE_STATIC_CODE_GEN_BUFFER
 404 #endif
 405
 406 #ifdef USE_STATIC_CODE_GEN_BUFFER
 407 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
 408 #endif
 409
 410 static void code_gen_alloc(unsigned long tb_size)
 411 {
 412     if (kvm_enabled())
 413         return;
 414
 415 #ifdef USE_STATIC_CODE_GEN_BUFFER
 416     code_gen_buffer = static_code_gen_buffer;
 417     code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
 418     map_exec(code_gen_buffer, code_gen_buffer_size);
 419 #else
 420     code_gen_buffer_size = tb_size;
 421     if (code_gen_buffer_size == 0) {
 422 #if defined(CONFIG_USER_ONLY)
 423         /* in user mode, phys_ram_size is not meaningful */
 424         code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
 425 #else
 426         /* XXX: needs ajustments */
 427         code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
 428 #endif
 429     }
 430     if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
 431         code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
 432     /* The code gen buffer location may have constraints depending on
 433        the host cpu and OS */
 434 #if defined(__linux__)
 435     {
 436         int flags;
 437         void *start = NULL;
 438
 439         flags = MAP_PRIVATE | MAP_ANONYMOUS;
 440 #if defined(__x86_64__)
 441         flags |= MAP_32BIT;
 442         /* Cannot map more than that */
 443         if (code_gen_buffer_size > (800 * 1024 * 1024))
 444             code_gen_buffer_size = (800 * 1024 * 1024);
 445 #elif defined(__sparc_v9__)
 446         // Map the buffer below 2G, so we can use direct calls and branches
 447         flags |= MAP_FIXED;
 448         start = (void *) 0x60000000UL;
 449         if (code_gen_buffer_size > (512 * 1024 * 1024))
 450             code_gen_buffer_size = (512 * 1024 * 1024);
 451 #elif defined(__arm__)
 452         /* Map the buffer below 32M, so we can use direct calls and branches */
 453         flags |= MAP_FIXED;
 454         start = (void *) 0x01000000UL;
 455         if (code_gen_buffer_size > 16 * 1024 * 1024)
 456             code_gen_buffer_size = 16 * 1024 * 1024;
 457 #endif
 458         code_gen_buffer = mmap(start, code_gen_buffer_size,
 459                                PROT_WRITE | PROT_READ | PROT_EXEC,
 460                                flags, -1, 0);
 461         if (code_gen_buffer == MAP_FAILED) {
 462             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
 463             exit(1);
 464         }
 465     }
 466 #elif defined(__FreeBSD__)
 467     {
 468         int flags;
 469         void *addr = NULL;
 470         flags = MAP_PRIVATE | MAP_ANONYMOUS;
 471 #if defined(__x86_64__)
 472         /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
 473          * 0x40000000 is free */
 474         flags |= MAP_FIXED;
 475         addr = (void *)0x40000000;
 476         /* Cannot map more than that */
 477         if (code_gen_buffer_size > (800 * 1024 * 1024))
 478             code_gen_buffer_size = (800 * 1024 * 1024);
 479 #endif
 480         code_gen_buffer = mmap(addr, code_gen_buffer_size,
 481                                PROT_WRITE | PROT_READ | PROT_EXEC,
 482                                flags, -1, 0);
 483         if (code_gen_buffer == MAP_FAILED) {
 484             fprintf(stderr, "Could not allocate dynamic translator buffer\n");
 485             exit(1);
 486         }
 487     }
 488 #else
 489     code_gen_buffer = qemu_malloc(code_gen_buffer_size);
 490     if (!code_gen_buffer) {
 491         fprintf(stderr, "Could not allocate dynamic translator buffer\n");
 492         exit(1);
 493     }
 494     map_exec(code_gen_buffer, code_gen_buffer_size);
 495 #endif
 496 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
 497     map_exec(code_gen_prologue, sizeof(code_gen_prologue));
 498     code_gen_buffer_max_size = code_gen_buffer_size -
 499         code_gen_max_block_size();
 500     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
 501     tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
 502 }
 503
 504 /* Must be called before using the QEMU cpus. 'tb_size' is the size
 505    (in bytes) allocated to the translation buffer. Zero means default
 506    size. */
 507 void cpu_exec_init_all(unsigned long tb_size)
 508 {
 509     cpu_gen_init();
 510     code_gen_alloc(tb_size);
 511     code_gen_ptr = code_gen_buffer;
 512     page_init();
 513 #if !defined(CONFIG_USER_ONLY)
 514     io_mem_init();
 515 #endif
 516 }
 517
 518 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 519
 520 #define CPU_COMMON_SAVE_VERSION 1
 521
 522 static void cpu_common_save(QEMUFile *f, void *opaque)
 523 {
 524     CPUState *env = opaque;
 525
 526     qemu_put_be32s(f, &env->halted);
 527     qemu_put_be32s(f, &env->interrupt_request);
 528 }
 529
 530 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
 531 {
 532     CPUState *env = opaque;
 533
 534     if (version_id != CPU_COMMON_SAVE_VERSION)
 535         return -EINVAL;
 536
 537     qemu_get_be32s(f, &env->halted);
 538     qemu_get_be32s(f, &env->interrupt_request);
 539     tlb_flush(env, 1);
 540
 541     return 0;
 542 }
 543 #endif
 544
 545 void cpu_exec_init(CPUState *env)
 546 {
 547     CPUState **penv;
 548     int cpu_index;
 549
 550     env->next_cpu = NULL;
 551     penv = &first_cpu;
 552     cpu_index = 0;
 553     while (*penv != NULL) {
 554         penv = (CPUState **)&(*penv)->next_cpu;
 555         cpu_index++;
 556     }
 557     env->cpu_index = cpu_index;
 558     TAILQ_INIT(&env->breakpoints);
 559     TAILQ_INIT(&env->watchpoints);
 560 #ifdef __WIN32
 561     env->thread_id = GetCurrentProcessId();
 562 #else
 563     env->thread_id = getpid();
 564 #endif
 565     *penv = env;
 566 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
 567     register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
 568                     cpu_common_save, cpu_common_load, env);
 569     register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
 570                     cpu_save, cpu_load, env);
 571 #endif
 572 }
 573
 574 static inline void invalidate_page_bitmap(PageDesc *p)
 575 {
 576     if (p->code_bitmap) {
 577         qemu_free(p->code_bitmap);
 578         p->code_bitmap = NULL;
 579     }
 580     p->code_write_count = 0;
 581 }
 582
 583 /* set to NULL all the 'first_tb' fields in all PageDescs */
 584 static void page_flush_tb(void)
 585 {
 586     int i, j;
 587     PageDesc *p;
 588
 589     for(i = 0; i < L1_SIZE; i++) {
 590         p = l1_map[i];
 591         if (p) {
 592             for(j = 0; j < L2_SIZE; j++) {
 593                 p->first_tb = NULL;
 594                 invalidate_page_bitmap(p);
 595                 p++;
 596             }
 597         }
 598     }
 599 }
 600
 601 /* flush all the translation blocks */
 602 /* XXX: tb_flush is currently not thread safe */
 603 void tb_flush(CPUState *env1)
 604 {
 605     CPUState *env;
 606 #if defined(DEBUG_FLUSH)
 607     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
 608            (unsigned long)(code_gen_ptr - code_gen_buffer),
 609            nb_tbs, nb_tbs > 0 ?
 610            ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
 611 #endif
 612     if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
 613         cpu_abort(env1, "Internal error: code buffer overflow\n");
 614
 615     nb_tbs = 0;
 616
 617     for(env = first_cpu; env != NULL; env = env->next_cpu) {
 618         memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
 619     }
 620
 621     memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
 622     page_flush_tb();
 623
 624     code_gen_ptr = code_gen_buffer;
 625     /* XXX: flush processor icache at this point if cache flush is
 626        expensive */
 627     tb_flush_count++;
 628 }
 629
 630 #ifdef DEBUG_TB_CHECK
 631
 632 static void tb_invalidate_check(target_ulong address)
 633 {
 634     TranslationBlock *tb;
 635     int i;
 636     address &= TARGET_PAGE_MASK;
 637     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
 638         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
 639             if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
 640                   address >= tb->pc + tb->size)) {
 641                 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
 642                        address, (long)tb->pc, tb->size);
 643             }
 644         }
 645     }
 646 }
 647
 648 /* verify that all the pages have correct rights for code */
 649 static void tb_page_check(void)
 650 {
 651     TranslationBlock *tb;
 652     int i, flags1, flags2;
 653
 654     for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
 655         for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
 656             flags1 = page_get_flags(tb->pc);
 657             flags2 = page_get_flags(tb->pc + tb->size - 1);
 658             if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
 659                 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
 660                        (long)tb->pc, tb->size, flags1, flags2);
 661             }
 662         }
 663     }
 664 }
 665
 666 static void tb_jmp_check(TranslationBlock *tb)
 667 {
 668     TranslationBlock *tb1;
 669     unsigned int n1;
 670
 671     /* suppress any remaining jumps to this TB */
 672     tb1 = tb->jmp_first;
 673     for(;;) {
 674         n1 = (long)tb1 & 3;
 675         tb1 = (TranslationBlock *)((long)tb1 & ~3);
 676         if (n1 == 2)
 677             break;
 678         tb1 = tb1->jmp_next[n1];
 679     }
 680     /* check end of list */
 681     if (tb1 != tb) {
 682         printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
 683     }
 684 }
 685
 686 #endif
 687
 688 /* invalidate one TB */
 689 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
 690                              int next_offset)
 691 {
 692     TranslationBlock *tb1;
 693     for(;;) {
 694         tb1 = *ptb;
 695         if (tb1 == tb) {
 696             *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
 697             break;
 698         }
 699         ptb = (TranslationBlock **)((char *)tb1 + next_offset);
 700     }
 701 }
 702
 703 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
 704 {
 705     TranslationBlock *tb1;
 706     unsigned int n1;
 707
 708     for(;;) {
 709         tb1 = *ptb;
 710         n1 = (long)tb1 & 3;
 711         tb1 = (TranslationBlock *)((long)tb1 & ~3);
 712         if (tb1 == tb) {
 713             *ptb = tb1->page_next[n1];
 714             break;
 715         }
 716         ptb = &tb1->page_next[n1];
 717     }
 718 }
 719
 720 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
 721 {
 722     TranslationBlock *tb1, **ptb;
 723     unsigned int n1;
 724
 725     ptb = &tb->jmp_next[n];
 726     tb1 = *ptb;
 727     if (tb1) {
 728         /* find tb(n) in circular list */
 729         for(;;) {
 730             tb1 = *ptb;
 731             n1 = (long)tb1 & 3;
 732             tb1 = (TranslationBlock *)((long)tb1 & ~3);
 733             if (n1 == n && tb1 == tb)
 734                 break;
 735             if (n1 == 2) {
 736                 ptb = &tb1->jmp_first;
 737             } else {
 738                 ptb = &tb1->jmp_next[n1];
 739             }
 740         }
 741         /* now we can suppress tb(n) from the list */
 742         *ptb = tb->jmp_next[n];
 743
 744         tb->jmp_next[n] = NULL;
 745     }
 746 }
 747
 748 /* reset the jump entry 'n' of a TB so that it is not chained to
 749    another TB */
 750 static inline void tb_reset_jump(TranslationBlock *tb, int n)
 751 {
 752     tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
 753 }
 754
 755 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
 756 {
 757     CPUState *env;
 758     PageDesc *p;
 759     unsigned int h, n1;
 760     target_phys_addr_t phys_pc;
 761     TranslationBlock *tb1, *tb2;
 762
 763     /* remove the TB from the hash list */
 764     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
 765     h = tb_phys_hash_func(phys_pc);
 766     tb_remove(&tb_phys_hash[h], tb,
 767               offsetof(TranslationBlock, phys_hash_next));
 768
 769     /* remove the TB from the page list */
 770     if (tb->page_addr[0] != page_addr) {
 771         p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
 772         tb_page_remove(&p->first_tb, tb);
 773         invalidate_page_bitmap(p);
 774     }
 775     if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
 776         p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
 777         tb_page_remove(&p->first_tb, tb);
 778         invalidate_page_bitmap(p);
 779     }
 780
 781     tb_invalidated_flag = 1;
 782
 783     /* remove the TB from the hash list */
 784     h = tb_jmp_cache_hash_func(tb->pc);
 785     for(env = first_cpu; env != NULL; env = env->next_cpu) {
 786         if (env->tb_jmp_cache[h] == tb)
 787             env->tb_jmp_cache[h] = NULL;
 788     }
 789
 790     /* suppress this TB from the two jump lists */
 791     tb_jmp_remove(tb, 0);
 792     tb_jmp_remove(tb, 1);
 793
 794     /* suppress any remaining jumps to this TB */
 795     tb1 = tb->jmp_first;
 796     for(;;) {
 797         n1 = (long)tb1 & 3;
 798         if (n1 == 2)
 799             break;
 800         tb1 = (TranslationBlock *)((long)tb1 & ~3);
 801         tb2 = tb1->jmp_next[n1];
 802         tb_reset_jump(tb1, n1);
 803         tb1->jmp_next[n1] = NULL;
 804         tb1 = tb2;
 805     }
 806     tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
 807
 808     tb_phys_invalidate_count++;
 809 }
 810
 811 static inline void set_bits(uint8_t *tab, int start, int len)
 812 {
 813     int end, mask, end1;
 814
 815     end = start + len;
 816     tab += start >> 3;
 817     mask = 0xff << (start & 7);
 818     if ((start & ~7) == (end & ~7)) {
 819         if (start < end) {
 820             mask &= ~(0xff << (end & 7));
 821             *tab |= mask;
 822         }
 823     } else {
 824         *tab++ |= mask;
 825         start = (start + 8) & ~7;
 826         end1 = end & ~7;
 827         while (start < end1) {
 828             *tab++ = 0xff;
 829             start += 8;
 830         }
 831         if (start < end) {
 832             mask = ~(0xff << (end & 7));
 833             *tab |= mask;
 834         }
 835     }
 836 }
 837
 838 static void build_page_bitmap(PageDesc *p)
 839 {
 840     int n, tb_start, tb_end;
 841     TranslationBlock *tb;
 842
 843     p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
 844     if (!p->code_bitmap)
 845         return;
 846
 847     tb = p->first_tb;
 848     while (tb != NULL) {
 849         n = (long)tb & 3;
 850         tb = (TranslationBlock *)((long)tb & ~3);
 851         /* NOTE: this is subtle as a TB may span two physical pages */
 852         if (n == 0) {
 853             /* NOTE: tb_end may be after the end of the page, but
 854                it is not a problem */
 855             tb_start = tb->pc & ~TARGET_PAGE_MASK;
 856             tb_end = tb_start + tb->size;
 857             if (tb_end > TARGET_PAGE_SIZE)
 858                 tb_end = TARGET_PAGE_SIZE;
 859         } else {
 860             tb_start = 0;
 861             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
 862         }
 863         set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
 864         tb = tb->page_next[n];
 865     }
 866 }
 867
 868 TranslationBlock *tb_gen_code(CPUState *env,
 869                               target_ulong pc, target_ulong cs_base,
 870                               int flags, int cflags)
 871 {
 872     TranslationBlock *tb;
 873     uint8_t *tc_ptr;
 874     target_ulong phys_pc, phys_page2, virt_page2;
 875     int code_gen_size;
 876
 877     phys_pc = get_phys_addr_code(env, pc);
 878     tb = tb_alloc(pc);
 879     if (!tb) {
 880         /* flush must be done */
 881         tb_flush(env);
 882         /* cannot fail at this point */
 883         tb = tb_alloc(pc);
 884         /* Don't forget to invalidate previous TB info.  */
 885         tb_invalidated_flag = 1;
 886     }
 887     tc_ptr = code_gen_ptr;
 888     tb->tc_ptr = tc_ptr;
 889     tb->cs_base = cs_base;
 890     tb->flags = flags;
 891     tb->cflags = cflags;
 892     cpu_gen_code(env, tb, &code_gen_size);
 893     code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
 894
 895     /* check next page if needed */
 896     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
 897     phys_page2 = -1;
 898     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
 899         phys_page2 = get_phys_addr_code(env, virt_page2);
 900     }
 901     tb_link_phys(tb, phys_pc, phys_page2);
 902     return tb;
 903 }
 904
 905 /* invalidate all TBs which intersect with the target physical page
 906    starting in range [start;end[. NOTE: start and end must refer to
 907    the same physical page. 'is_cpu_write_access' should be true if called
 908    from a real cpu write access: the virtual CPU will exit the current
 909    TB if code is modified inside this TB. */
 910 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
 911                                    int is_cpu_write_access)
 912 {
 913     TranslationBlock *tb, *tb_next, *saved_tb;
 914     CPUState *env = cpu_single_env;
 915     target_ulong tb_start, tb_end;
 916     PageDesc *p;
 917     int n;
 918 #ifdef TARGET_HAS_PRECISE_SMC
 919     int current_tb_not_found = is_cpu_write_access;
 920     TranslationBlock *current_tb = NULL;
 921     int current_tb_modified = 0;
 922     target_ulong current_pc = 0;
 923     target_ulong current_cs_base = 0;
 924     int current_flags = 0;
 925 #endif /* TARGET_HAS_PRECISE_SMC */
 926
 927     p = page_find(start >> TARGET_PAGE_BITS);
 928     if (!p)
 929         return;
 930     if (!p->code_bitmap &&
 931         ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
 932         is_cpu_write_access) {
 933         /* build code bitmap */
 934         build_page_bitmap(p);
 935     }
 936
 937     /* we remove all the TBs in the range [start, end[ */
 938     /* XXX: see if in some cases it could be faster to invalidate all the code */
 939     tb = p->first_tb;
 940     while (tb != NULL) {
 941         n = (long)tb & 3;
 942         tb = (TranslationBlock *)((long)tb & ~3);
 943         tb_next = tb->page_next[n];
 944         /* NOTE: this is subtle as a TB may span two physical pages */
 945         if (n == 0) {
 946             /* NOTE: tb_end may be after the end of the page, but
 947                it is not a problem */
 948             tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
 949             tb_end = tb_start + tb->size;
 950         } else {
 951             tb_start = tb->page_addr[1];
 952             tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
 953         }
 954         if (!(tb_end <= start || tb_start >= end)) {
 955 #ifdef TARGET_HAS_PRECISE_SMC
 956             if (current_tb_not_found) {
 957                 current_tb_not_found = 0;
 958                 current_tb = NULL;
 959                 if (env->mem_io_pc) {
 960                     /* now we have a real cpu fault */
 961                     current_tb = tb_find_pc(env->mem_io_pc);
 962                 }
 963             }
 964             if (current_tb == tb &&
 965                 (current_tb->cflags & CF_COUNT_MASK) != 1) {
 966                 /* If we are modifying the current TB, we must stop
 967                 its execution. We could be more precise by checking
 968                 that the modification is after the current PC, but it
 969                 would require a specialized function to partially
 970                 restore the CPU state */
 971
 972                 current_tb_modified = 1;
 973                 cpu_restore_state(current_tb, env,
 974                                   env->mem_io_pc, NULL);
 975                 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
 976                                      &current_flags);
 977             }
 978 #endif /* TARGET_HAS_PRECISE_SMC */
 979             /* we need to do that to handle the case where a signal
 980                occurs while doing tb_phys_invalidate() */
 981             saved_tb = NULL;
 982             if (env) {
 983                 saved_tb = env->current_tb;
 984                 env->current_tb = NULL;
 985             }
 986             tb_phys_invalidate(tb, -1);
 987             if (env) {
 988                 env->current_tb = saved_tb;
 989                 if (env->interrupt_request && env->current_tb)
 990                     cpu_interrupt(env, env->interrupt_request);
 991             }
 992         }
 993         tb = tb_next;
 994     }
 995 #if !defined(CONFIG_USER_ONLY)
 996     /* if no code remaining, no need to continue to use slow writes */
 997     if (!p->first_tb) {
 998         invalidate_page_bitmap(p);
 999         if (is_cpu_write_access) {
1000             tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1001         }
1002     }
1003 #endif
1004 #ifdef TARGET_HAS_PRECISE_SMC
1005     if (current_tb_modified) {
1006         /* we generate a block containing just the instruction
1007            modifying the memory. It will ensure that it cannot modify
1008            itself */
1009         env->current_tb = NULL;
1010         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1011         cpu_resume_from_signal(env, NULL);
1012     }
1013 #endif
1014 }
1015
1016 /* len must be <= 8 and start must be a multiple of len */
1017 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1018 {
1019     PageDesc *p;
1020     int offset, b;
1021 #if 0
1022     if (1) {
1023         if (loglevel) {
1024             fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1025                    cpu_single_env->mem_io_vaddr, len,
1026                    cpu_single_env->eip,
1027                    cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1028         }
1029     }
1030 #endif
1031     p = page_find(start >> TARGET_PAGE_BITS);
1032     if (!p)
1033         return;
1034     if (p->code_bitmap) {
1035         offset = start & ~TARGET_PAGE_MASK;
1036         b = p->code_bitmap[offset >> 3] >> (offset & 7);
1037         if (b & ((1 << len) - 1))
1038             goto do_invalidate;
1039     } else {
1040     do_invalidate:
1041         tb_invalidate_phys_page_range(start, start + len, 1);
1042     }
1043 }
1044
1045 #if !defined(CONFIG_SOFTMMU)
1046 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1047                                     unsigned long pc, void *puc)
1048 {
1049     TranslationBlock *tb;
1050     PageDesc *p;
1051     int n;
1052 #ifdef TARGET_HAS_PRECISE_SMC
1053     TranslationBlock *current_tb = NULL;
1054     CPUState *env = cpu_single_env;
1055     int current_tb_modified = 0;
1056     target_ulong current_pc = 0;
1057     target_ulong current_cs_base = 0;
1058     int current_flags = 0;
1059 #endif
1060
1061     addr &= TARGET_PAGE_MASK;
1062     p = page_find(addr >> TARGET_PAGE_BITS);
1063     if (!p)
1064         return;
1065     tb = p->first_tb;
1066 #ifdef TARGET_HAS_PRECISE_SMC
1067     if (tb && pc != 0) {
1068         current_tb = tb_find_pc(pc);
1069     }
1070 #endif
1071     while (tb != NULL) {
1072         n = (long)tb & 3;
1073         tb = (TranslationBlock *)((long)tb & ~3);
1074 #ifdef TARGET_HAS_PRECISE_SMC
1075         if (current_tb == tb &&
1076             (current_tb->cflags & CF_COUNT_MASK) != 1) {
1077                 /* If we are modifying the current TB, we must stop
1078                    its execution. We could be more precise by checking
1079                    that the modification is after the current PC, but it
1080                    would require a specialized function to partially
1081                    restore the CPU state */
1082
1083             current_tb_modified = 1;
1084             cpu_restore_state(current_tb, env, pc, puc);
1085             cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1086                                  &current_flags);
1087         }
1088 #endif /* TARGET_HAS_PRECISE_SMC */
1089         tb_phys_invalidate(tb, addr);
1090         tb = tb->page_next[n];
1091     }
1092     p->first_tb = NULL;
1093 #ifdef TARGET_HAS_PRECISE_SMC
1094     if (current_tb_modified) {
1095         /* we generate a block containing just the instruction
1096            modifying the memory. It will ensure that it cannot modify
1097            itself */
1098         env->current_tb = NULL;
1099         tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1100         cpu_resume_from_signal(env, puc);
1101     }
1102 #endif
1103 }
1104 #endif
1105
1106 /* add the tb in the target page and protect it if necessary */
1107 static inline void tb_alloc_page(TranslationBlock *tb,
1108                                  unsigned int n, target_ulong page_addr)
1109 {
1110     PageDesc *p;
1111     TranslationBlock *last_first_tb;
1112
1113     tb->page_addr[n] = page_addr;
1114     p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1115     tb->page_next[n] = p->first_tb;
1116     last_first_tb = p->first_tb;
1117     p->first_tb = (TranslationBlock *)((long)tb | n);
1118     invalidate_page_bitmap(p);
1119
1120 #if defined(TARGET_HAS_SMC) || 1
1121
1122 #if defined(CONFIG_USER_ONLY)
1123     if (p->flags & PAGE_WRITE) {
1124         target_ulong addr;
1125         PageDesc *p2;
1126         int prot;
1127
1128         /* force the host page as non writable (writes will have a
1129            page fault + mprotect overhead) */
1130         page_addr &= qemu_host_page_mask;
1131         prot = 0;
1132         for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1133             addr += TARGET_PAGE_SIZE) {
1134
1135             p2 = page_find (addr >> TARGET_PAGE_BITS);
1136             if (!p2)
1137                 continue;
1138             prot |= p2->flags;
1139             p2->flags &= ~PAGE_WRITE;
1140             page_get_flags(addr);
1141           }
1142         mprotect(g2h(page_addr), qemu_host_page_size,
1143                  (prot & PAGE_BITS) & ~PAGE_WRITE);
1144 #ifdef DEBUG_TB_INVALIDATE
1145         printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1146                page_addr);
1147 #endif
1148     }
1149 #else
1150     /* if some code is already present, then the pages are already
1151        protected. So we handle the case where only the first TB is
1152        allocated in a physical page */
1153     if (!last_first_tb) {
1154         tlb_protect_code(page_addr);
1155     }
1156 #endif
1157
1158 #endif /* TARGET_HAS_SMC */
1159 }
1160
1161 /* Allocate a new translation block. Flush the translation buffer if
1162    too many translation blocks or too much generated code. */
1163 TranslationBlock *tb_alloc(target_ulong pc)
1164 {
1165     TranslationBlock *tb;
1166
1167     if (nb_tbs >= code_gen_max_blocks ||
1168         (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1169         return NULL;
1170     tb = &tbs[nb_tbs++];
1171     tb->pc = pc;
1172     tb->cflags = 0;
1173     return tb;
1174 }
1175
1176 void tb_free(TranslationBlock *tb)
1177 {
1178     /* In practice this is mostly used for single use temporary TB
1179        Ignore the hard cases and just back up if this TB happens to
1180        be the last one generated.  */
1181     if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1182         code_gen_ptr = tb->tc_ptr;
1183         nb_tbs--;
1184     }
1185 }
1186
1187 /* add a new TB and link it to the physical page tables. phys_page2 is
1188    (-1) to indicate that only one page contains the TB. */
1189 void tb_link_phys(TranslationBlock *tb,
1190                   target_ulong phys_pc, target_ulong phys_page2)
1191 {
1192     unsigned int h;
1193     TranslationBlock **ptb;
1194
1195     /* Grab the mmap lock to stop another thread invalidating this TB
1196        before we are done.  */
1197     mmap_lock();
1198     /* add in the physical hash table */
1199     h = tb_phys_hash_func(phys_pc);
1200     ptb = &tb_phys_hash[h];
1201     tb->phys_hash_next = *ptb;
1202     *ptb = tb;
1203
1204     /* add in the page list */
1205     tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1206     if (phys_page2 != -1)
1207         tb_alloc_page(tb, 1, phys_page2);
1208     else
1209         tb->page_addr[1] = -1;
1210
1211     tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1212     tb->jmp_next[0] = NULL;
1213     tb->jmp_next[1] = NULL;
1214
1215     /* init original jump addresses */
1216     if (tb->tb_next_offset[0] != 0xffff)
1217         tb_reset_jump(tb, 0);
1218     if (tb->tb_next_offset[1] != 0xffff)
1219         tb_reset_jump(tb, 1);
1220
1221 #ifdef DEBUG_TB_CHECK
1222     tb_page_check();
1223 #endif
1224     mmap_unlock();
1225 }
1226
1227 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1228    tb[1].tc_ptr. Return NULL if not found */
1229 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1230 {
1231     int m_min, m_max, m;
1232     unsigned long v;
1233     TranslationBlock *tb;
1234
1235     if (nb_tbs <= 0)
1236         return NULL;
1237     if (tc_ptr < (unsigned long)code_gen_buffer ||
1238         tc_ptr >= (unsigned long)code_gen_ptr)
1239         return NULL;
1240     /* binary search (cf Knuth) */
1241     m_min = 0;
1242     m_max = nb_tbs - 1;
1243     while (m_min <= m_max) {
1244         m = (m_min + m_max) >> 1;
1245         tb = &tbs[m];
1246         v = (unsigned long)tb->tc_ptr;
1247         if (v == tc_ptr)
1248             return tb;
1249         else if (tc_ptr < v) {
1250             m_max = m - 1;
1251         } else {
1252             m_min = m + 1;
1253         }
1254     }
1255     return &tbs[m_max];
1256 }
1257
1258 static void tb_reset_jump_recursive(TranslationBlock *tb);
1259
1260 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1261 {
1262     TranslationBlock *tb1, *tb_next, **ptb;
1263     unsigned int n1;
1264
1265     tb1 = tb->jmp_next[n];
1266     if (tb1 != NULL) {
1267         /* find head of list */
1268         for(;;) {
1269             n1 = (long)tb1 & 3;
1270             tb1 = (TranslationBlock *)((long)tb1 & ~3);
1271             if (n1 == 2)
1272                 break;
1273             tb1 = tb1->jmp_next[n1];
1274         }
1275         /* we are now sure now that tb jumps to tb1 */
1276         tb_next = tb1;
1277
1278         /* remove tb from the jmp_first list */
1279         ptb = &tb_next->jmp_first;
1280         for(;;) {
1281             tb1 = *ptb;
1282             n1 = (long)tb1 & 3;
1283             tb1 = (TranslationBlock *)((long)tb1 & ~3);
1284             if (n1 == n && tb1 == tb)
1285                 break;
1286             ptb = &tb1->jmp_next[n1];
1287         }
1288         *ptb = tb->jmp_next[n];
1289         tb->jmp_next[n] = NULL;
1290
1291         /* suppress the jump to next tb in generated code */
1292         tb_reset_jump(tb, n);
1293
1294         /* suppress jumps in the tb on which we could have jumped */
1295         tb_reset_jump_recursive(tb_next);
1296     }
1297 }
1298
1299 static void tb_reset_jump_recursive(TranslationBlock *tb)
1300 {
1301     tb_reset_jump_recursive2(tb, 0);
1302     tb_reset_jump_recursive2(tb, 1);
1303 }
1304
1305 #if defined(TARGET_HAS_ICE)
1306 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1307 {
1308     target_phys_addr_t addr;
1309     target_ulong pd;
1310     ram_addr_t ram_addr;
1311     PhysPageDesc *p;
1312
1313     addr = cpu_get_phys_page_debug(env, pc);
1314     p = phys_page_find(addr >> TARGET_PAGE_BITS);
1315     if (!p) {
1316         pd = IO_MEM_UNASSIGNED;
1317     } else {
1318         pd = p->phys_offset;
1319     }
1320     ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1321     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1322 }
1323 #endif
1324
1325 /* Add a watchpoint.  */
1326 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1327                           int flags, CPUWatchpoint **watchpoint)
1328 {
1329     target_ulong len_mask = ~(len - 1);
1330     CPUWatchpoint *wp;
1331
1332     /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1333     if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1334         fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1335                 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1336         return -EINVAL;
1337     }
1338     wp = qemu_malloc(sizeof(*wp));
1339     if (!wp)
1340         return -ENOMEM;
1341
1342     wp->vaddr = addr;
1343     wp->len_mask = len_mask;
1344     wp->flags = flags;
1345
1346     /* keep all GDB-injected watchpoints in front */
1347     if (flags & BP_GDB)
1348         TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1349     else
1350         TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1351
1352     tlb_flush_page(env, addr);
1353
1354     if (watchpoint)
1355         *watchpoint = wp;
1356     return 0;
1357 }
1358
1359 /* Remove a specific watchpoint.  */
1360 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1361                           int flags)
1362 {
1363     target_ulong len_mask = ~(len - 1);
1364     CPUWatchpoint *wp;
1365
1366     TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1367         if (addr == wp->vaddr && len_mask == wp->len_mask
1368                 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1369             cpu_watchpoint_remove_by_ref(env, wp);
1370             return 0;
1371         }
1372     }
1373     return -ENOENT;
1374 }
1375
1376 /* Remove a specific watchpoint by reference.  */
1377 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1378 {
1379     TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1380
1381     tlb_flush_page(env, watchpoint->vaddr);
1382
1383     qemu_free(watchpoint);
1384 }
1385
1386 /* Remove all matching watchpoints.  */
1387 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1388 {
1389     CPUWatchpoint *wp, *next;
1390
1391     TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1392         if (wp->flags & mask)
1393             cpu_watchpoint_remove_by_ref(env, wp);
1394     }
1395 }
1396
1397 /* Add a breakpoint.  */
1398 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1399                           CPUBreakpoint **breakpoint)
1400 {
1401 #if defined(TARGET_HAS_ICE)
1402     CPUBreakpoint *bp;
1403
1404     bp = qemu_malloc(sizeof(*bp));
1405     if (!bp)
1406         return -ENOMEM;
1407
1408     bp->pc = pc;
1409     bp->flags = flags;
1410
1411     /* keep all GDB-injected breakpoints in front */
1412     if (flags & BP_GDB)
1413         TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1414     else
1415         TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1416
1417     breakpoint_invalidate(env, pc);
1418
1419     if (breakpoint)
1420         *breakpoint = bp;
1421     return 0;
1422 #else
1423     return -ENOSYS;
1424 #endif
1425 }
1426
1427 /* Remove a specific breakpoint.  */
1428 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1429 {
1430 #if defined(TARGET_HAS_ICE)
1431     CPUBreakpoint *bp;
1432
1433     TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1434         if (bp->pc == pc && bp->flags == flags) {
1435             cpu_breakpoint_remove_by_ref(env, bp);
1436             return 0;
1437         }
1438     }
1439     return -ENOENT;
1440 #else
1441     return -ENOSYS;
1442 #endif
1443 }
1444
1445 /* Remove a specific breakpoint by reference.  */
1446 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1447 {
1448 #if defined(TARGET_HAS_ICE)
1449     TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1450
1451     breakpoint_invalidate(env, breakpoint->pc);
1452
1453     qemu_free(breakpoint);
1454 #endif
1455 }
1456
1457 /* Remove all matching breakpoints. */
1458 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1459 {
1460 #if defined(TARGET_HAS_ICE)
1461     CPUBreakpoint *bp, *next;
1462
1463     TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1464         if (bp->flags & mask)
1465             cpu_breakpoint_remove_by_ref(env, bp);
1466     }
1467 #endif
1468 }
1469
1470 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1471    CPU loop after each instruction */
1472 void cpu_single_step(CPUState *env, int enabled)
1473 {
1474 #if defined(TARGET_HAS_ICE)
1475     if (env->singlestep_enabled != enabled) {
1476         env->singlestep_enabled = enabled;
1477         if (kvm_enabled())
1478             kvm_update_guest_debug(env, 0);
1479         else {
1480             /* must flush all the translated code to avoid inconsistancies */
1481             /* XXX: only flush what is necessary */
1482             tb_flush(env);
1483         }
1484     }
1485 #endif
1486 }
1487
1488 /* enable or disable low levels log */
1489 void cpu_set_log(int log_flags)
1490 {
1491     loglevel = log_flags;
1492     if (loglevel && !logfile) {
1493         logfile = fopen(logfilename, log_append ? "a" : "w");
1494         if (!logfile) {
1495             perror(logfilename);
1496             _exit(1);
1497         }
1498 #if !defined(CONFIG_SOFTMMU)
1499         /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1500         {
1501             static char logfile_buf[4096];
1502             setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1503         }
1504 #else
1505         setvbuf(logfile, NULL, _IOLBF, 0);
1506 #endif
1507         log_append = 1;
1508     }
1509     if (!loglevel && logfile) {
1510         fclose(logfile);
1511         logfile = NULL;
1512     }
1513 }
1514
1515 void cpu_set_log_filename(const char *filename)
1516 {
1517     logfilename = strdup(filename);
1518     if (logfile) {
1519         fclose(logfile);
1520         logfile = NULL;
1521     }
1522     cpu_set_log(loglevel);
1523 }
1524
1525 /* mask must never be zero, except for A20 change call */
1526 void cpu_interrupt(CPUState *env, int mask)
1527 {
1528 #if !defined(USE_NPTL)
1529     TranslationBlock *tb;
1530     static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1531 #endif
1532     int old_mask;
1533
1534     old_mask = env->interrupt_request;
1535     /* FIXME: This is probably not threadsafe.  A different thread could
1536        be in the middle of a read-modify-write operation.  */
1537     env->interrupt_request |= mask;
1538     if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1539         kvm_update_interrupt_request(env);
1540 #if defined(USE_NPTL)
1541     /* FIXME: TB unchaining isn't SMP safe.  For now just ignore the
1542        problem and hope the cpu will stop of its own accord.  For userspace
1543        emulation this often isn't actually as bad as it sounds.  Often
1544        signals are used primarily to interrupt blocking syscalls.  */
1545 #else
1546     if (use_icount) {
1547         env->icount_decr.u16.high = 0xffff;
1548 #ifndef CONFIG_USER_ONLY
1549         /* CPU_INTERRUPT_EXIT isn't a real interrupt.  It just means
1550            an async event happened and we need to process it.  */
1551         if (!can_do_io(env)
1552             && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1553             cpu_abort(env, "Raised interrupt while not in I/O function");
1554         }
1555 #endif
1556     } else {
1557         tb = env->current_tb;
1558         /* if the cpu is currently executing code, we must unlink it and
1559            all the potentially executing TB */
1560         if (tb && !testandset(&interrupt_lock)) {
1561             env->current_tb = NULL;
1562             tb_reset_jump_recursive(tb);
1563             resetlock(&interrupt_lock);
1564         }
1565     }
1566 #endif
1567 }
1568
1569 void cpu_reset_interrupt(CPUState *env, int mask)
1570 {
1571     env->interrupt_request &= ~mask;
1572 }
1573
1574 const CPULogItem cpu_log_items[] = {
1575     { CPU_LOG_TB_OUT_ASM, "out_asm",
1576       "show generated host assembly code for each compiled TB" },
1577     { CPU_LOG_TB_IN_ASM, "in_asm",
1578       "show target assembly code for each compiled TB" },
1579     { CPU_LOG_TB_OP, "op",
1580       "show micro ops for each compiled TB" },
1581     { CPU_LOG_TB_OP_OPT, "op_opt",
1582       "show micro ops "
1583 #ifdef TARGET_I386
1584       "before eflags optimization and "
1585 #endif
1586       "after liveness analysis" },
1587     { CPU_LOG_INT, "int",
1588       "show interrupts/exceptions in short format" },
1589     { CPU_LOG_EXEC, "exec",
1590       "show trace before each executed TB (lots of logs)" },
1591     { CPU_LOG_TB_CPU, "cpu",
1592       "show CPU state before block translation" },
1593 #ifdef TARGET_I386
1594     { CPU_LOG_PCALL, "pcall",
1595       "show protected mode far calls/returns/exceptions" },
1596 #endif
1597 #ifdef DEBUG_IOPORT
1598     { CPU_LOG_IOPORT, "ioport",
1599       "show all i/o ports accesses" },
1600 #endif
1601     { 0, NULL, NULL },
1602 };
1603
1604 static int cmp1(const char *s1, int n, const char *s2)
1605 {
1606     if (strlen(s2) != n)
1607         return 0;
1608     return memcmp(s1, s2, n) == 0;
1609 }
1610
1611 /* takes a comma separated list of log masks. Return 0 if error. */
1612 int cpu_str_to_log_mask(const char *str)
1613 {
1614     const CPULogItem *item;
1615     int mask;
1616     const char *p, *p1;
1617
1618     p = str;
1619     mask = 0;
1620     for(;;) {
1621         p1 = strchr(p, ',');
1622         if (!p1)
1623             p1 = p + strlen(p);
1624         if(cmp1(p,p1-p,"all")) {
1625                 for(item = cpu_log_items; item->mask != 0; item++) {
1626                         mask |= item->mask;
1627                 }
1628         } else {
1629         for(item = cpu_log_items; item->mask != 0; item++) {
1630             if (cmp1(p, p1 - p, item->name))
1631                 goto found;
1632         }
1633         return 0;
1634         }
1635     found:
1636         mask |= item->mask;
1637         if (*p1 != ',')
1638             break;
1639         p = p1 + 1;
1640     }
1641     return mask;
1642 }
1643
1644 void cpu_abort(CPUState *env, const char *fmt, ...)
1645 {
1646     va_list ap;
1647     va_list ap2;
1648
1649     va_start(ap, fmt);
1650     va_copy(ap2, ap);
1651     fprintf(stderr, "qemu: fatal: ");
1652     vfprintf(stderr, fmt, ap);
1653     fprintf(stderr, "\n");
1654 #ifdef TARGET_I386
1655     cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1656 #else
1657     cpu_dump_state(env, stderr, fprintf, 0);
1658 #endif
1659     if (logfile) {
1660         fprintf(logfile, "qemu: fatal: ");
1661         vfprintf(logfile, fmt, ap2);
1662         fprintf(logfile, "\n");
1663 #ifdef TARGET_I386
1664         cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1665 #else
1666         cpu_dump_state(env, logfile, fprintf, 0);
1667 #endif
1668         fflush(logfile);
1669         fclose(logfile);
1670     }
1671     va_end(ap2);
1672     va_end(ap);
1673     abort();
1674 }
1675
1676 CPUState *cpu_copy(CPUState *env)
1677 {
1678     CPUState *new_env = cpu_init(env->cpu_model_str);
1679     /* preserve chaining and index */
1680     CPUState *next_cpu = new_env->next_cpu;
1681     int cpu_index = new_env->cpu_index;
1682     memcpy(new_env, env, sizeof(CPUState));
1683     new_env->next_cpu = next_cpu;
1684     new_env->cpu_index = cpu_index;
1685     return new_env;
1686 }
1687
1688 #if !defined(CONFIG_USER_ONLY)
1689
1690 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1691 {
1692     unsigned int i;
1693
1694     /* Discard jump cache entries for any tb which might potentially
1695        overlap the flushed page.  */
1696     i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1697     memset (&env->tb_jmp_cache[i], 0,
1698             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1699
1700     i = tb_jmp_cache_hash_page(addr);
1701     memset (&env->tb_jmp_cache[i], 0,
1702             TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1703 }
1704
1705 /* NOTE: if flush_global is true, also flush global entries (not
1706    implemented yet) */
1707 void tlb_flush(CPUState *env, int flush_global)
1708 {
1709     int i;
1710
1711 #if defined(DEBUG_TLB)
1712     printf("tlb_flush:\n");
1713 #endif
1714     /* must reset current TB so that interrupts cannot modify the
1715        links while we are modifying them */
1716     env->current_tb = NULL;
1717
1718     for(i = 0; i < CPU_TLB_SIZE; i++) {
1719         env->tlb_table[0][i].addr_read = -1;
1720         env->tlb_table[0][i].addr_write = -1;
1721         env->tlb_table[0][i].addr_code = -1;
1722         env->tlb_table[1][i].addr_read = -1;
1723         env->tlb_table[1][i].addr_write = -1;
1724         env->tlb_table[1][i].addr_code = -1;
1725 #if (NB_MMU_MODES >= 3)
1726         env->tlb_table[2][i].addr_read = -1;
1727         env->tlb_table[2][i].addr_write = -1;
1728         env->tlb_table[2][i].addr_code = -1;
1729 #if (NB_MMU_MODES == 4)
1730         env->tlb_table[3][i].addr_read = -1;
1731         env->tlb_table[3][i].addr_write = -1;
1732         env->tlb_table[3][i].addr_code = -1;
1733 #endif
1734 #endif
1735     }
1736
1737     memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1738
1739 #ifdef USE_KQEMU
1740     if (env->kqemu_enabled) {
1741         kqemu_flush(env, flush_global);
1742     }
1743 #endif
1744     tlb_flush_count++;
1745 }
1746
1747 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1748 {
1749     if (addr == (tlb_entry->addr_read &
1750                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1751         addr == (tlb_entry->addr_write &
1752                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1753         addr == (tlb_entry->addr_code &
1754                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1755         tlb_entry->addr_read = -1;
1756         tlb_entry->addr_write = -1;
1757         tlb_entry->addr_code = -1;
1758     }
1759 }
1760
1761 void tlb_flush_page(CPUState *env, target_ulong addr)
1762 {
1763     int i;
1764
1765 #if defined(DEBUG_TLB)
1766     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1767 #endif
1768     /* must reset current TB so that interrupts cannot modify the
1769        links while we are modifying them */
1770     env->current_tb = NULL;
1771
1772     addr &= TARGET_PAGE_MASK;
1773     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1774     tlb_flush_entry(&env->tlb_table[0][i], addr);
1775     tlb_flush_entry(&env->tlb_table[1][i], addr);
1776 #if (NB_MMU_MODES >= 3)
1777     tlb_flush_entry(&env->tlb_table[2][i], addr);
1778 #if (NB_MMU_MODES == 4)
1779     tlb_flush_entry(&env->tlb_table[3][i], addr);
1780 #endif
1781 #endif
1782
1783     tlb_flush_jmp_cache(env, addr);
1784
1785 #ifdef USE_KQEMU
1786     if (env->kqemu_enabled) {
1787         kqemu_flush_page(env, addr);
1788     }
1789 #endif
1790 }
1791
1792 /* update the TLBs so that writes to code in the virtual page 'addr'
1793    can be detected */
1794 static void tlb_protect_code(ram_addr_t ram_addr)
1795 {
1796     cpu_physical_memory_reset_dirty(ram_addr,
1797                                     ram_addr + TARGET_PAGE_SIZE,
1798                                     CODE_DIRTY_FLAG);
1799 }
1800
1801 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1802    tested for self modifying code */
1803 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1804                                     target_ulong vaddr)
1805 {
1806     phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1807 }
1808
1809 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1810                                          unsigned long start, unsigned long length)
1811 {
1812     unsigned long addr;
1813     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1814         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1815         if ((addr - start) < length) {
1816             tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1817         }
1818     }
1819 }
1820
1821 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1822                                      int dirty_flags)
1823 {
1824     CPUState *env;
1825     unsigned long length, start1;
1826     int i, mask, len;
1827     uint8_t *p;
1828
1829     start &= TARGET_PAGE_MASK;
1830     end = TARGET_PAGE_ALIGN(end);
1831
1832     length = end - start;
1833     if (length == 0)
1834         return;
1835     len = length >> TARGET_PAGE_BITS;
1836 #ifdef USE_KQEMU
1837     /* XXX: should not depend on cpu context */
1838     env = first_cpu;
1839     if (env->kqemu_enabled) {
1840         ram_addr_t addr;
1841         addr = start;
1842         for(i = 0; i < len; i++) {
1843             kqemu_set_notdirty(env, addr);
1844             addr += TARGET_PAGE_SIZE;
1845         }
1846     }
1847 #endif
1848     mask = ~dirty_flags;
1849     p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1850     for(i = 0; i < len; i++)
1851         p[i] &= mask;
1852
1853     /* we modify the TLB cache so that the dirty bit will be set again
1854        when accessing the range */
1855     start1 = start + (unsigned long)phys_ram_base;
1856     for(env = first_cpu; env != NULL; env = env->next_cpu) {
1857         for(i = 0; i < CPU_TLB_SIZE; i++)
1858             tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1859         for(i = 0; i < CPU_TLB_SIZE; i++)
1860             tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1861 #if (NB_MMU_MODES >= 3)
1862         for(i = 0; i < CPU_TLB_SIZE; i++)
1863             tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1864 #if (NB_MMU_MODES == 4)
1865         for(i = 0; i < CPU_TLB_SIZE; i++)
1866             tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1867 #endif
1868 #endif
1869     }
1870 }
1871
1872 int cpu_physical_memory_set_dirty_tracking(int enable)
1873 {
1874     int r=0;
1875
1876     if (kvm_enabled())
1877         r = kvm_physical_memory_set_dirty_tracking(enable);
1878     in_migration = enable;
1879     return r;
1880 }
1881
1882 int cpu_physical_memory_get_dirty_tracking(void)
1883 {
1884     return in_migration;
1885 }
1886
1887 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1888 {
1889     if (kvm_enabled())
1890         kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1891 }
1892
1893 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1894 {
1895     ram_addr_t ram_addr;
1896
1897     if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1898         ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1899             tlb_entry->addend - (unsigned long)phys_ram_base;
1900         if (!cpu_physical_memory_is_dirty(ram_addr)) {
1901             tlb_entry->addr_write |= TLB_NOTDIRTY;
1902         }
1903     }
1904 }
1905
1906 /* update the TLB according to the current state of the dirty bits */
1907 void cpu_tlb_update_dirty(CPUState *env)
1908 {
1909     int i;
1910     for(i = 0; i < CPU_TLB_SIZE; i++)
1911         tlb_update_dirty(&env->tlb_table[0][i]);
1912     for(i = 0; i < CPU_TLB_SIZE; i++)
1913         tlb_update_dirty(&env->tlb_table[1][i]);
1914 #if (NB_MMU_MODES >= 3)
1915     for(i = 0; i < CPU_TLB_SIZE; i++)
1916         tlb_update_dirty(&env->tlb_table[2][i]);
1917 #if (NB_MMU_MODES == 4)
1918     for(i = 0; i < CPU_TLB_SIZE; i++)
1919         tlb_update_dirty(&env->tlb_table[3][i]);
1920 #endif
1921 #endif
1922 }
1923
1924 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1925 {
1926     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1927         tlb_entry->addr_write = vaddr;
1928 }
1929
1930 /* update the TLB corresponding to virtual page vaddr
1931    so that it is no longer dirty */
1932 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1933 {
1934     int i;
1935
1936     vaddr &= TARGET_PAGE_MASK;
1937     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1938     tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1939     tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1940 #if (NB_MMU_MODES >= 3)
1941     tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1942 #if (NB_MMU_MODES == 4)
1943     tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1944 #endif
1945 #endif
1946 }
1947
1948 /* add a new TLB entry. At most one entry for a given virtual address
1949    is permitted. Return 0 if OK or 2 if the page could not be mapped
1950    (can only happen in non SOFTMMU mode for I/O pages or pages
1951    conflicting with the host address space). */
1952 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1953                       target_phys_addr_t paddr, int prot,
1954                       int mmu_idx, int is_softmmu)
1955 {
1956     PhysPageDesc *p;
1957     unsigned long pd;
1958     unsigned int index;
1959     target_ulong address;
1960     target_ulong code_address;
1961     target_phys_addr_t addend;
1962     int ret;
1963     CPUTLBEntry *te;
1964     CPUWatchpoint *wp;
1965     target_phys_addr_t iotlb;
1966
1967     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1968     if (!p) {
1969         pd = IO_MEM_UNASSIGNED;
1970     } else {
1971         pd = p->phys_offset;
1972     }
1973 #if defined(DEBUG_TLB)
1974     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1975            vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1976 #endif
1977
1978     ret = 0;
1979     address = vaddr;
1980     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1981         /* IO memory case (romd handled later) */
1982         address |= TLB_MMIO;
1983     }
1984     addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1985     if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1986         /* Normal RAM.  */
1987         iotlb = pd & TARGET_PAGE_MASK;
1988         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1989             iotlb |= IO_MEM_NOTDIRTY;
1990         else
1991             iotlb |= IO_MEM_ROM;
1992     } else {
1993         /* IO handlers are currently passed a phsical address.
1994            It would be nice to pass an offset from the base address
1995            of that region.  This would avoid having to special case RAM,
1996            and avoid full address decoding in every device.
1997            We can't use the high bits of pd for this because
1998            IO_MEM_ROMD uses these as a ram address.  */
1999         iotlb = (pd & ~TARGET_PAGE_MASK);
2000         if (p) {
2001             iotlb += p->region_offset;
2002         } else {
2003             iotlb += paddr;
2004         }
2005     }
2006
2007     code_address = address;
2008     /* Make accesses to pages with watchpoints go via the
2009        watchpoint trap routines.  */
2010     TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2011         if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2012             iotlb = io_mem_watch + paddr;
2013             /* TODO: The memory case can be optimized by not trapping
2014                reads of pages with a write breakpoint.  */
2015             address |= TLB_MMIO;
2016         }
2017     }
2018
2019     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2020     env->iotlb[mmu_idx][index] = iotlb - vaddr;
2021     te = &env->tlb_table[mmu_idx][index];
2022     te->addend = addend - vaddr;
2023     if (prot & PAGE_READ) {
2024         te->addr_read = address;
2025     } else {
2026         te->addr_read = -1;
2027     }
2028
2029     if (prot & PAGE_EXEC) {
2030         te->addr_code = code_address;
2031     } else {
2032         te->addr_code = -1;
2033     }
2034     if (prot & PAGE_WRITE) {
2035         if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2036             (pd & IO_MEM_ROMD)) {
2037             /* Write access calls the I/O callback.  */
2038             te->addr_write = address | TLB_MMIO;
2039         } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2040                    !cpu_physical_memory_is_dirty(pd)) {
2041             te->addr_write = address | TLB_NOTDIRTY;
2042         } else {
2043             te->addr_write = address;
2044         }
2045     } else {
2046         te->addr_write = -1;
2047     }
2048     return ret;
2049 }
2050
2051 #else
2052
2053 void tlb_flush(CPUState *env, int flush_global)
2054 {
2055 }
2056
2057 void tlb_flush_page(CPUState *env, target_ulong addr)
2058 {
2059 }
2060
2061 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2062                       target_phys_addr_t paddr, int prot,
2063                       int mmu_idx, int is_softmmu)
2064 {
2065     return 0;
2066 }
2067
2068 /* dump memory mappings */
2069 void page_dump(FILE *f)
2070 {
2071     unsigned long start, end;
2072     int i, j, prot, prot1;
2073     PageDesc *p;
2074
2075     fprintf(f, "%-8s %-8s %-8s %s\n",
2076             "start", "end", "size", "prot");
2077     start = -1;
2078     end = -1;
2079     prot = 0;
2080     for(i = 0; i <= L1_SIZE; i++) {
2081         if (i < L1_SIZE)
2082             p = l1_map[i];
2083         else
2084             p = NULL;
2085         for(j = 0;j < L2_SIZE; j++) {
2086             if (!p)
2087                 prot1 = 0;
2088             else
2089                 prot1 = p[j].flags;
2090             if (prot1 != prot) {
2091                 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2092                 if (start != -1) {
2093                     fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2094                             start, end, end - start,
2095                             prot & PAGE_READ ? 'r' : '-',
2096                             prot & PAGE_WRITE ? 'w' : '-',
2097                             prot & PAGE_EXEC ? 'x' : '-');
2098                 }
2099                 if (prot1 != 0)
2100                     start = end;
2101                 else
2102                     start = -1;
2103                 prot = prot1;
2104             }
2105             if (!p)
2106                 break;
2107         }
2108     }
2109 }
2110
2111 int page_get_flags(target_ulong address)
2112 {
2113     PageDesc *p;
2114
2115     p = page_find(address >> TARGET_PAGE_BITS);
2116     if (!p)
2117         return 0;
2118     return p->flags;
2119 }
2120
2121 /* modify the flags of a page and invalidate the code if
2122    necessary. The flag PAGE_WRITE_ORG is positionned automatically
2123    depending on PAGE_WRITE */
2124 void page_set_flags(target_ulong start, target_ulong end, int flags)
2125 {
2126     PageDesc *p;
2127     target_ulong addr;
2128
2129     /* mmap_lock should already be held.  */
2130     start = start & TARGET_PAGE_MASK;
2131     end = TARGET_PAGE_ALIGN(end);
2132     if (flags & PAGE_WRITE)
2133         flags |= PAGE_WRITE_ORG;
2134     for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2135         p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2136         /* We may be called for host regions that are outside guest
2137            address space.  */
2138         if (!p)
2139             return;
2140         /* if the write protection is set, then we invalidate the code
2141            inside */
2142         if (!(p->flags & PAGE_WRITE) &&
2143             (flags & PAGE_WRITE) &&
2144             p->first_tb) {
2145             tb_invalidate_phys_page(addr, 0, NULL);
2146         }
2147         p->flags = flags;
2148     }
2149 }
2150
2151 int page_check_range(target_ulong start, target_ulong len, int flags)
2152 {
2153     PageDesc *p;
2154     target_ulong end;
2155     target_ulong addr;
2156
2157     if (start + len < start)
2158         /* we've wrapped around */
2159         return -1;
2160
2161     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2162     start = start & TARGET_PAGE_MASK;
2163
2164     for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2165         p = page_find(addr >> TARGET_PAGE_BITS);
2166         if( !p )
2167             return -1;
2168         if( !(p->flags & PAGE_VALID) )
2169             return -1;
2170
2171         if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2172             return -1;
2173         if (flags & PAGE_WRITE) {
2174             if (!(p->flags & PAGE_WRITE_ORG))
2175                 return -1;
2176             /* unprotect the page if it was put read-only because it
2177                contains translated code */
2178             if (!(p->flags & PAGE_WRITE)) {
2179                 if (!page_unprotect(addr, 0, NULL))
2180                     return -1;
2181             }
2182             return 0;
2183         }
2184     }
2185     return 0;
2186 }
2187
2188 /* called from signal handler: invalidate the code and unprotect the
2189    page. Return TRUE if the fault was succesfully handled. */
2190 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2191 {
2192     unsigned int page_index, prot, pindex;
2193     PageDesc *p, *p1;
2194     target_ulong host_start, host_end, addr;
2195
2196     /* Technically this isn't safe inside a signal handler.  However we
2197        know this only ever happens in a synchronous SEGV handler, so in
2198        practice it seems to be ok.  */
2199     mmap_lock();
2200
2201     host_start = address & qemu_host_page_mask;
2202     page_index = host_start >> TARGET_PAGE_BITS;
2203     p1 = page_find(page_index);
2204     if (!p1) {
2205         mmap_unlock();
2206         return 0;
2207     }
2208     host_end = host_start + qemu_host_page_size;
2209     p = p1;
2210     prot = 0;
2211     for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2212         prot |= p->flags;
2213         p++;
2214     }
2215     /* if the page was really writable, then we change its
2216        protection back to writable */
2217     if (prot & PAGE_WRITE_ORG) {
2218         pindex = (address - host_start) >> TARGET_PAGE_BITS;
2219         if (!(p1[pindex].flags & PAGE_WRITE)) {
2220             mprotect((void *)g2h(host_start), qemu_host_page_size,
2221                      (prot & PAGE_BITS) | PAGE_WRITE);
2222             p1[pindex].flags |= PAGE_WRITE;
2223             /* and since the content will be modified, we must invalidate
2224                the corresponding translated code. */
2225             tb_invalidate_phys_page(address, pc, puc);
2226 #ifdef DEBUG_TB_CHECK
2227             tb_invalidate_check(address);
2228 #endif
2229             mmap_unlock();
2230             return 1;
2231         }
2232     }
2233     mmap_unlock();
2234     return 0;
2235 }
2236
2237 static inline void tlb_set_dirty(CPUState *env,
2238                                  unsigned long addr, target_ulong vaddr)
2239 {
2240 }
2241 #endif /* defined(CONFIG_USER_ONLY) */
2242
2243 #if !defined(CONFIG_USER_ONLY)
2244
2245 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2246                              ram_addr_t memory, ram_addr_t region_offset);
2247 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2248                            ram_addr_t orig_memory, ram_addr_t region_offset);
2249 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2250                       need_subpage)                                     \
2251     do {                                                                \
2252         if (addr > start_addr)                                          \
2253             start_addr2 = 0;                                            \
2254         else {                                                          \
2255             start_addr2 = start_addr & ~TARGET_PAGE_MASK;               \
2256             if (start_addr2 > 0)                                        \
2257                 need_subpage = 1;                                       \
2258         }                                                               \
2259                                                                         \
2260         if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE)        \
2261             end_addr2 = TARGET_PAGE_SIZE - 1;                           \
2262         else {                                                          \
2263             end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2264             if (end_addr2 < TARGET_PAGE_SIZE - 1)                       \
2265                 need_subpage = 1;                                       \
2266         }                                                               \
2267     } while (0)
2268
2269 /* register physical memory. 'size' must be a multiple of the target
2270    page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2271    io memory page.  The address used when calling the IO function is
2272    the offset from the start of the region, plus region_offset.  Both
2273    start_region and regon_offset are rounded down to a page boundary
2274    before calculating this offset.  This should not be a problem unless
2275    the low bits of start_addr and region_offset differ.  */
2276 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2277                                          ram_addr_t size,
2278                                          ram_addr_t phys_offset,
2279                                          ram_addr_t region_offset)
2280 {
2281     target_phys_addr_t addr, end_addr;
2282     PhysPageDesc *p;
2283     CPUState *env;
2284     ram_addr_t orig_size = size;
2285     void *subpage;
2286
2287 #ifdef USE_KQEMU
2288     /* XXX: should not depend on cpu context */
2289     env = first_cpu;
2290     if (env->kqemu_enabled) {
2291         kqemu_set_phys_mem(start_addr, size, phys_offset);
2292     }
2293 #endif
2294     if (kvm_enabled())
2295         kvm_set_phys_mem(start_addr, size, phys_offset);
2296
2297     region_offset &= TARGET_PAGE_MASK;
2298     size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2299     end_addr = start_addr + (target_phys_addr_t)size;
2300     for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2301         p = phys_page_find(addr >> TARGET_PAGE_BITS);
2302         if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2303             ram_addr_t orig_memory = p->phys_offset;
2304             target_phys_addr_t start_addr2, end_addr2;
2305             int need_subpage = 0;
2306
2307             CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2308                           need_subpage);
2309             if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2310                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2311                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
2312                                            &p->phys_offset, orig_memory,
2313                                            p->region_offset);
2314                 } else {
2315                     subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2316                                             >> IO_MEM_SHIFT];
2317                 }
2318                 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2319                                  region_offset);
2320                 p->region_offset = 0;
2321             } else {
2322                 p->phys_offset = phys_offset;
2323                 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2324                     (phys_offset & IO_MEM_ROMD))
2325                     phys_offset += TARGET_PAGE_SIZE;
2326             }
2327         } else {
2328             p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2329             p->phys_offset = phys_offset;
2330             p->region_offset = region_offset;
2331             if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2332                 (phys_offset & IO_MEM_ROMD)) {
2333                 phys_offset += TARGET_PAGE_SIZE;
2334             } else {
2335                 target_phys_addr_t start_addr2, end_addr2;
2336                 int need_subpage = 0;
2337
2338                 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2339                               end_addr2, need_subpage);
2340
2341                 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2342                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
2343                                            &p->phys_offset, IO_MEM_UNASSIGNED,
2344                                            0);
2345                     subpage_register(subpage, start_addr2, end_addr2,
2346                                      phys_offset, region_offset);
2347                     p->region_offset = 0;
2348                 }
2349             }
2350         }
2351         region_offset += TARGET_PAGE_SIZE;
2352     }
2353
2354     /* since each CPU stores ram addresses in its TLB cache, we must
2355        reset the modified entries */
2356     /* XXX: slow ! */
2357     for(env = first_cpu; env != NULL; env = env->next_cpu) {
2358         tlb_flush(env, 1);
2359     }
2360 }
2361
2362 /* XXX: temporary until new memory mapping API */
2363 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2364 {
2365     PhysPageDesc *p;
2366
2367     p = phys_page_find(addr >> TARGET_PAGE_BITS);
2368     if (!p)
2369         return IO_MEM_UNASSIGNED;
2370     return p->phys_offset;
2371 }
2372
2373 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2374 {
2375     if (kvm_enabled())
2376         kvm_coalesce_mmio_region(addr, size);
2377 }
2378
2379 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2380 {
2381     if (kvm_enabled())
2382         kvm_uncoalesce_mmio_region(addr, size);
2383 }
2384
2385 /* XXX: better than nothing */
2386 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2387 {
2388     ram_addr_t addr;
2389     if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2390         fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2391                 (uint64_t)size, (uint64_t)phys_ram_size);
2392         abort();
2393     }
2394     addr = phys_ram_alloc_offset;
2395     phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2396     return addr;
2397 }
2398
2399 void qemu_ram_free(ram_addr_t addr)
2400 {
2401 }
2402
2403 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2404 {
2405 #ifdef DEBUG_UNASSIGNED
2406     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2407 #endif
2408 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2409     do_unassigned_access(addr, 0, 0, 0, 1);
2410 #endif
2411     return 0;
2412 }
2413
2414 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2415 {
2416 #ifdef DEBUG_UNASSIGNED
2417     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2418 #endif
2419 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2420     do_unassigned_access(addr, 0, 0, 0, 2);
2421 #endif
2422     return 0;
2423 }
2424
2425 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2426 {
2427 #ifdef DEBUG_UNASSIGNED
2428     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2429 #endif
2430 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2431     do_unassigned_access(addr, 0, 0, 0, 4);
2432 #endif
2433     return 0;
2434 }
2435
2436 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2437 {
2438 #ifdef DEBUG_UNASSIGNED
2439     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2440 #endif
2441 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2442     do_unassigned_access(addr, 1, 0, 0, 1);
2443 #endif
2444 }
2445
2446 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2447 {
2448 #ifdef DEBUG_UNASSIGNED
2449     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2450 #endif
2451 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2452     do_unassigned_access(addr, 1, 0, 0, 2);
2453 #endif
2454 }
2455
2456 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2457 {
2458 #ifdef DEBUG_UNASSIGNED
2459     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2460 #endif
2461 #if defined(TARGET_SPARC) || defined(TARGET_CRIS)
2462     do_unassigned_access(addr, 1, 0, 0, 4);
2463 #endif
2464 }
2465
2466 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2467     unassigned_mem_readb,
2468     unassigned_mem_readw,
2469     unassigned_mem_readl,
2470 };
2471
2472 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2473     unassigned_mem_writeb,
2474     unassigned_mem_writew,
2475     unassigned_mem_writel,
2476 };
2477
2478 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2479                                 uint32_t val)
2480 {
2481     int dirty_flags;
2482     dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2483     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2484 #if !defined(CONFIG_USER_ONLY)
2485         tb_invalidate_phys_page_fast(ram_addr, 1);
2486         dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2487 #endif
2488     }
2489     stb_p(phys_ram_base + ram_addr, val);
2490 #ifdef USE_KQEMU
2491     if (cpu_single_env->kqemu_enabled &&
2492         (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2493         kqemu_modify_page(cpu_single_env, ram_addr);
2494 #endif
2495     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2496     phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2497     /* we remove the notdirty callback only if the code has been
2498        flushed */
2499     if (dirty_flags == 0xff)
2500         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2501 }
2502
2503 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2504                                 uint32_t val)
2505 {
2506     int dirty_flags;
2507     dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2508     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2509 #if !defined(CONFIG_USER_ONLY)
2510         tb_invalidate_phys_page_fast(ram_addr, 2);
2511         dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2512 #endif
2513     }
2514     stw_p(phys_ram_base + ram_addr, val);
2515 #ifdef USE_KQEMU
2516     if (cpu_single_env->kqemu_enabled &&
2517         (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2518         kqemu_modify_page(cpu_single_env, ram_addr);
2519 #endif
2520     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2521     phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2522     /* we remove the notdirty callback only if the code has been
2523        flushed */
2524     if (dirty_flags == 0xff)
2525         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2526 }
2527
2528 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2529                                 uint32_t val)
2530 {
2531     int dirty_flags;
2532     dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2533     if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2534 #if !defined(CONFIG_USER_ONLY)
2535         tb_invalidate_phys_page_fast(ram_addr, 4);
2536         dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2537 #endif
2538     }
2539     stl_p(phys_ram_base + ram_addr, val);
2540 #ifdef USE_KQEMU
2541     if (cpu_single_env->kqemu_enabled &&
2542         (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2543         kqemu_modify_page(cpu_single_env, ram_addr);
2544 #endif
2545     dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2546     phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2547     /* we remove the notdirty callback only if the code has been
2548        flushed */
2549     if (dirty_flags == 0xff)
2550         tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2551 }
2552
2553 static CPUReadMemoryFunc *error_mem_read[3] = {
2554     NULL, /* never used */
2555     NULL, /* never used */
2556     NULL, /* never used */
2557 };
2558
2559 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2560     notdirty_mem_writeb,
2561     notdirty_mem_writew,
2562     notdirty_mem_writel,
2563 };
2564
2565 /* Generate a debug exception if a watchpoint has been hit.  */
2566 static void check_watchpoint(int offset, int len_mask, int flags)
2567 {
2568     CPUState *env = cpu_single_env;
2569     target_ulong pc, cs_base;
2570     TranslationBlock *tb;
2571     target_ulong vaddr;
2572     CPUWatchpoint *wp;
2573     int cpu_flags;
2574
2575     if (env->watchpoint_hit) {
2576         /* We re-entered the check after replacing the TB. Now raise
2577          * the debug interrupt so that is will trigger after the
2578          * current instruction. */
2579         cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2580         return;
2581     }
2582     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2583     TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2584         if ((vaddr == (wp->vaddr & len_mask) ||
2585              (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2586             wp->flags |= BP_WATCHPOINT_HIT;
2587             if (!env->watchpoint_hit) {
2588                 env->watchpoint_hit = wp;
2589                 tb = tb_find_pc(env->mem_io_pc);
2590                 if (!tb) {
2591                     cpu_abort(env, "check_watchpoint: could not find TB for "
2592                               "pc=%p", (void *)env->mem_io_pc);
2593                 }
2594                 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2595                 tb_phys_invalidate(tb, -1);
2596                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2597                     env->exception_index = EXCP_DEBUG;
2598                 } else {
2599                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2600                     tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2601                 }
2602                 cpu_resume_from_signal(env, NULL);
2603             }
2604         } else {
2605             wp->flags &= ~BP_WATCHPOINT_HIT;
2606         }
2607     }
2608 }
2609
2610 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
2611    so these check for a hit then pass through to the normal out-of-line
2612    phys routines.  */
2613 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2614 {
2615     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2616     return ldub_phys(addr);
2617 }
2618
2619 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2620 {
2621     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2622     return lduw_phys(addr);
2623 }
2624
2625 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2626 {
2627     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2628     return ldl_phys(addr);
2629 }
2630
2631 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2632                              uint32_t val)
2633 {
2634     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2635     stb_phys(addr, val);
2636 }
2637
2638 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2639                              uint32_t val)
2640 {
2641     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2642     stw_phys(addr, val);
2643 }
2644
2645 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2646                              uint32_t val)
2647 {
2648     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2649     stl_phys(addr, val);
2650 }
2651
2652 static CPUReadMemoryFunc *watch_mem_read[3] = {
2653     watch_mem_readb,
2654     watch_mem_readw,
2655     watch_mem_readl,
2656 };
2657
2658 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2659     watch_mem_writeb,
2660     watch_mem_writew,
2661     watch_mem_writel,
2662 };
2663
2664 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2665                                  unsigned int len)
2666 {
2667     uint32_t ret;
2668     unsigned int idx;
2669
2670     idx = SUBPAGE_IDX(addr);
2671 #if defined(DEBUG_SUBPAGE)
2672     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2673            mmio, len, addr, idx);
2674 #endif
2675     ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2676                                        addr + mmio->region_offset[idx][0][len]);
2677
2678     return ret;
2679 }
2680
2681 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2682                               uint32_t value, unsigned int len)
2683 {
2684     unsigned int idx;
2685
2686     idx = SUBPAGE_IDX(addr);
2687 #if defined(DEBUG_SUBPAGE)
2688     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2689            mmio, len, addr, idx, value);
2690 #endif
2691     (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2692                                   addr + mmio->region_offset[idx][1][len],
2693                                   value);
2694 }
2695
2696 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2697 {
2698 #if defined(DEBUG_SUBPAGE)
2699     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2700 #endif
2701
2702     return subpage_readlen(opaque, addr, 0);
2703 }
2704
2705 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2706                             uint32_t value)
2707 {
2708 #if defined(DEBUG_SUBPAGE)
2709     printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2710 #endif
2711     subpage_writelen(opaque, addr, value, 0);
2712 }
2713
2714 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2715 {
2716 #if defined(DEBUG_SUBPAGE)
2717     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2718 #endif
2719
2720     return subpage_readlen(opaque, addr, 1);
2721 }
2722
2723 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2724                             uint32_t value)
2725 {
2726 #if defined(DEBUG_SUBPAGE)
2727     printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2728 #endif
2729     subpage_writelen(opaque, addr, value, 1);
2730 }
2731
2732 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2733 {
2734 #if defined(DEBUG_SUBPAGE)
2735     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2736 #endif
2737
2738     return subpage_readlen(opaque, addr, 2);
2739 }
2740
2741 static void subpage_writel (void *opaque,
2742                          target_phys_addr_t addr, uint32_t value)
2743 {
2744 #if defined(DEBUG_SUBPAGE)
2745     printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2746 #endif
2747     subpage_writelen(opaque, addr, value, 2);
2748 }
2749
2750 static CPUReadMemoryFunc *subpage_read[] = {
2751     &subpage_readb,
2752     &subpage_readw,
2753     &subpage_readl,
2754 };
2755
2756 static CPUWriteMemoryFunc *subpage_write[] = {
2757     &subpage_writeb,
2758     &subpage_writew,
2759     &subpage_writel,
2760 };
2761
2762 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2763                              ram_addr_t memory, ram_addr_t region_offset)
2764 {
2765     int idx, eidx;
2766     unsigned int i;
2767
2768     if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2769         return -1;
2770     idx = SUBPAGE_IDX(start);
2771     eidx = SUBPAGE_IDX(end);
2772 #if defined(DEBUG_SUBPAGE)
2773     printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2774            mmio, start, end, idx, eidx, memory);
2775 #endif
2776     memory >>= IO_MEM_SHIFT;
2777     for (; idx <= eidx; idx++) {
2778         for (i = 0; i < 4; i++) {
2779             if (io_mem_read[memory][i]) {
2780                 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2781                 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2782                 mmio->region_offset[idx][0][i] = region_offset;
2783             }
2784             if (io_mem_write[memory][i]) {
2785                 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2786                 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2787                 mmio->region_offset[idx][1][i] = region_offset;
2788             }
2789         }
2790     }
2791
2792     return 0;
2793 }
2794
2795 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2796                            ram_addr_t orig_memory, ram_addr_t region_offset)
2797 {
2798     subpage_t *mmio;
2799     int subpage_memory;
2800
2801     mmio = qemu_mallocz(sizeof(subpage_t));
2802     if (mmio != NULL) {
2803         mmio->base = base;
2804         subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2805 #if defined(DEBUG_SUBPAGE)
2806         printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2807                mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2808 #endif
2809         *phys = subpage_memory | IO_MEM_SUBPAGE;
2810         subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2811                          region_offset);
2812     }
2813
2814     return mmio;
2815 }
2816
2817 static int get_free_io_mem_idx(void)
2818 {
2819     int i;
2820
2821     for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2822         if (!io_mem_used[i]) {
2823             io_mem_used[i] = 1;
2824             return i;
2825         }
2826
2827     return -1;
2828 }
2829
2830 static void io_mem_init(void)
2831 {
2832     int i;
2833
2834     cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2835     cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2836     cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2837     for (i=0; i<5; i++)
2838         io_mem_used[i] = 1;
2839
2840     io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2841                                           watch_mem_write, NULL);
2842     /* alloc dirty bits array */
2843     phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2844     memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2845 }
2846
2847 /* mem_read and mem_write are arrays of functions containing the
2848    function to access byte (index 0), word (index 1) and dword (index
2849    2). Functions can be omitted with a NULL function pointer. The
2850    registered functions may be modified dynamically later.
2851    If io_index is non zero, the corresponding io zone is
2852    modified. If it is zero, a new io zone is allocated. The return
2853    value can be used with cpu_register_physical_memory(). (-1) is
2854    returned if error. */
2855 int cpu_register_io_memory(int io_index,
2856                            CPUReadMemoryFunc **mem_read,
2857                            CPUWriteMemoryFunc **mem_write,
2858                            void *opaque)
2859 {
2860     int i, subwidth = 0;
2861
2862     if (io_index <= 0) {
2863         io_index = get_free_io_mem_idx();
2864         if (io_index == -1)
2865             return io_index;
2866     } else {
2867         if (io_index >= IO_MEM_NB_ENTRIES)
2868             return -1;
2869     }
2870
2871     for(i = 0;i < 3; i++) {
2872         if (!mem_read[i] || !mem_write[i])
2873             subwidth = IO_MEM_SUBWIDTH;
2874         io_mem_read[io_index][i] = mem_read[i];
2875         io_mem_write[io_index][i] = mem_write[i];
2876     }
2877     io_mem_opaque[io_index] = opaque;
2878     return (io_index << IO_MEM_SHIFT) | subwidth;
2879 }
2880
2881 void cpu_unregister_io_memory(int io_table_address)
2882 {
2883     int i;
2884     int io_index = io_table_address >> IO_MEM_SHIFT;
2885
2886     for (i=0;i < 3; i++) {
2887         io_mem_read[io_index][i] = unassigned_mem_read[i];
2888         io_mem_write[io_index][i] = unassigned_mem_write[i];
2889     }
2890     io_mem_opaque[io_index] = NULL;
2891     io_mem_used[io_index] = 0;
2892 }
2893
2894 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2895 {
2896     return io_mem_write[io_index >> IO_MEM_SHIFT];
2897 }
2898
2899 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2900 {
2901     return io_mem_read[io_index >> IO_MEM_SHIFT];
2902 }
2903
2904 #endif /* !defined(CONFIG_USER_ONLY) */
2905
2906 /* physical memory access (slow version, mainly for debug) */
2907 #if defined(CONFIG_USER_ONLY)
2908 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2909                             int len, int is_write)
2910 {
2911     int l, flags;
2912     target_ulong page;
2913     void * p;
2914
2915     while (len > 0) {
2916         page = addr & TARGET_PAGE_MASK;
2917         l = (page + TARGET_PAGE_SIZE) - addr;
2918         if (l > len)
2919             l = len;
2920         flags = page_get_flags(page);
2921         if (!(flags & PAGE_VALID))
2922             return;
2923         if (is_write) {
2924             if (!(flags & PAGE_WRITE))
2925                 return;
2926             /* XXX: this code should not depend on lock_user */
2927             if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2928                 /* FIXME - should this return an error rather than just fail? */
2929                 return;
2930             memcpy(p, buf, l);
2931             unlock_user(p, addr, l);
2932         } else {
2933             if (!(flags & PAGE_READ))
2934                 return;
2935             /* XXX: this code should not depend on lock_user */
2936             if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2937                 /* FIXME - should this return an error rather than just fail? */
2938                 return;
2939             memcpy(buf, p, l);
2940             unlock_user(p, addr, 0);
2941         }
2942         len -= l;
2943         buf += l;
2944         addr += l;
2945     }
2946 }
2947
2948 #else
2949 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2950                             int len, int is_write)
2951 {
2952     int l, io_index;
2953     uint8_t *ptr;
2954     uint32_t val;
2955     target_phys_addr_t page;
2956     unsigned long pd;
2957     PhysPageDesc *p;
2958
2959     while (len > 0) {
2960         page = addr & TARGET_PAGE_MASK;
2961         l = (page + TARGET_PAGE_SIZE) - addr;
2962         if (l > len)
2963             l = len;
2964         p = phys_page_find(page >> TARGET_PAGE_BITS);
2965         if (!p) {
2966             pd = IO_MEM_UNASSIGNED;
2967         } else {
2968             pd = p->phys_offset;
2969         }
2970
2971         if (is_write) {
2972             if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2973                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2974                 if (p)
2975                     addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2976                 /* XXX: could force cpu_single_env to NULL to avoid
2977                    potential bugs */
2978                 if (l >= 4 && ((addr & 3) == 0)) {
2979                     /* 32 bit write access */
2980                     val = ldl_p(buf);
2981                     io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
2982                     l = 4;
2983                 } else if (l >= 2 && ((addr & 1) == 0)) {
2984                     /* 16 bit write access */
2985                     val = lduw_p(buf);
2986                     io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
2987                     l = 2;
2988                 } else {
2989                     /* 8 bit write access */
2990                     val = ldub_p(buf);
2991                     io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
2992                     l = 1;
2993                 }
2994             } else {
2995                 unsigned long addr1;
2996                 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
2997                 /* RAM case */
2998                 ptr = phys_ram_base + addr1;
2999                 memcpy(ptr, buf, l);
3000                 if (!cpu_physical_memory_is_dirty(addr1)) {
3001                     /* invalidate code */
3002                     tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3003                     /* set dirty bit */
3004                     phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3005                         (0xff & ~CODE_DIRTY_FLAG);
3006                 }
3007                 /* qemu doesn't execute guest code directly, but kvm does
3008                    therefore fluch instruction caches */
3009                 if (kvm_enabled())
3010                     flush_icache_range((unsigned long)ptr,
3011                                        ((unsigned long)ptr)+l);
3012             }
3013         } else {
3014             if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3015                 !(pd & IO_MEM_ROMD)) {
3016                 /* I/O case */
3017                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3018                 if (p)
3019                     addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3020                 if (l >= 4 && ((addr & 3) == 0)) {
3021                     /* 32 bit read access */
3022                     val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3023                     stl_p(buf, val);
3024                     l = 4;
3025                 } else if (l >= 2 && ((addr & 1) == 0)) {
3026                     /* 16 bit read access */
3027                     val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3028                     stw_p(buf, val);
3029                     l = 2;
3030                 } else {
3031                     /* 8 bit read access */
3032                     val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
3033                     stb_p(buf, val);
3034                     l = 1;
3035                 }
3036             } else {
3037                 /* RAM case */
3038                 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3039                     (addr & ~TARGET_PAGE_MASK);
3040                 memcpy(buf, ptr, l);
3041             }
3042         }
3043         len -= l;
3044         buf += l;
3045         addr += l;
3046     }
3047 }
3048
3049 /* used for ROM loading : can write in RAM and ROM */
3050 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3051                                    const uint8_t *buf, int len)
3052 {
3053     int l;
3054     uint8_t *ptr;
3055     target_phys_addr_t page;
3056     unsigned long pd;
3057     PhysPageDesc *p;
3058
3059     while (len > 0) {
3060         page = addr & TARGET_PAGE_MASK;
3061         l = (page + TARGET_PAGE_SIZE) - addr;
3062         if (l > len)
3063             l = len;
3064         p = phys_page_find(page >> TARGET_PAGE_BITS);
3065         if (!p) {
3066             pd = IO_MEM_UNASSIGNED;
3067         } else {
3068             pd = p->phys_offset;
3069         }
3070
3071         if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3072             (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3073             !(pd & IO_MEM_ROMD)) {
3074             /* do nothing */
3075         } else {
3076             unsigned long addr1;
3077             addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3078             /* ROM/RAM case */
3079             ptr = phys_ram_base + addr1;
3080             memcpy(ptr, buf, l);
3081         }
3082         len -= l;
3083         buf += l;
3084         addr += l;
3085     }
3086 }
3087
3088
3089 /* warning: addr must be aligned */
3090 uint32_t ldl_phys(target_phys_addr_t addr)
3091 {
3092     int io_index;
3093     uint8_t *ptr;
3094     uint32_t val;
3095     unsigned long pd;
3096     PhysPageDesc *p;
3097
3098     p = phys_page_find(addr >> TARGET_PAGE_BITS);
3099     if (!p) {
3100         pd = IO_MEM_UNASSIGNED;
3101     } else {
3102         pd = p->phys_offset;
3103     }
3104
3105     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3106         !(pd & IO_MEM_ROMD)) {
3107         /* I/O case */
3108         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3109         if (p)
3110             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3111         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3112     } else {
3113         /* RAM case */
3114         ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3115             (addr & ~TARGET_PAGE_MASK);
3116         val = ldl_p(ptr);
3117     }
3118     return val;
3119 }
3120
3121 /* warning: addr must be aligned */
3122 uint64_t ldq_phys(target_phys_addr_t addr)
3123 {
3124     int io_index;
3125     uint8_t *ptr;
3126     uint64_t val;
3127     unsigned long pd;
3128     PhysPageDesc *p;
3129
3130     p = phys_page_find(addr >> TARGET_PAGE_BITS);
3131     if (!p) {
3132         pd = IO_MEM_UNASSIGNED;
3133     } else {
3134         pd = p->phys_offset;
3135     }
3136
3137     if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3138         !(pd & IO_MEM_ROMD)) {
3139         /* I/O case */
3140         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3141         if (p)
3142             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3143 #ifdef TARGET_WORDS_BIGENDIAN
3144         val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3145         val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3146 #else
3147         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3148         val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3149 #endif
3150     } else {
3151         /* RAM case */
3152         ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3153             (addr & ~TARGET_PAGE_MASK);
3154         val = ldq_p(ptr);
3155     }
3156     return val;
3157 }
3158
3159 /* XXX: optimize */
3160 uint32_t ldub_phys(target_phys_addr_t addr)
3161 {
3162     uint8_t val;
3163     cpu_physical_memory_read(addr, &val, 1);
3164     return val;
3165 }
3166
3167 /* XXX: optimize */
3168 uint32_t lduw_phys(target_phys_addr_t addr)
3169 {
3170     uint16_t val;
3171     cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3172     return tswap16(val);
3173 }
3174
3175 #ifdef __GNUC__
3176 #define likely(x) __builtin_expect(!!(x), 1)
3177 #define unlikely(x) __builtin_expect(!!(x), 0)
3178 #else
3179 #define likely(x) x
3180 #define unlikely(x) x
3181 #endif
3182
3183 /* warning: addr must be aligned. The ram page is not masked as dirty
3184    and the code inside is not invalidated. It is useful if the dirty
3185    bits are used to track modified PTEs */
3186 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3187 {
3188     int io_index;
3189     uint8_t *ptr;
3190     unsigned long pd;
3191     PhysPageDesc *p;
3192
3193     p = phys_page_find(addr >> TARGET_PAGE_BITS);
3194     if (!p) {
3195         pd = IO_MEM_UNASSIGNED;
3196     } else {
3197         pd = p->phys_offset;
3198     }
3199
3200     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3201         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3202         if (p)
3203             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3204         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3205     } else {
3206         unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3207         ptr = phys_ram_base + addr1;
3208         stl_p(ptr, val);
3209
3210         if (unlikely(in_migration)) {
3211             if (!cpu_physical_memory_is_dirty(addr1)) {
3212                 /* invalidate code */
3213                 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3214                 /* set dirty bit */
3215                 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3216                     (0xff & ~CODE_DIRTY_FLAG);
3217             }
3218         }
3219     }
3220 }
3221
3222 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3223 {
3224     int io_index;
3225     uint8_t *ptr;
3226     unsigned long pd;
3227     PhysPageDesc *p;
3228
3229     p = phys_page_find(addr >> TARGET_PAGE_BITS);
3230     if (!p) {
3231         pd = IO_MEM_UNASSIGNED;
3232     } else {
3233         pd = p->phys_offset;
3234     }
3235
3236     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3237         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3238         if (p)
3239             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3240 #ifdef TARGET_WORDS_BIGENDIAN
3241         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3242         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3243 #else
3244         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3245         io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3246 #endif
3247     } else {
3248         ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3249             (addr & ~TARGET_PAGE_MASK);
3250         stq_p(ptr, val);
3251     }
3252 }
3253
3254 /* warning: addr must be aligned */
3255 void stl_phys(target_phys_addr_t addr, uint32_t val)
3256 {
3257     int io_index;
3258     uint8_t *ptr;
3259     unsigned long pd;
3260     PhysPageDesc *p;
3261
3262     p = phys_page_find(addr >> TARGET_PAGE_BITS);
3263     if (!p) {
3264         pd = IO_MEM_UNASSIGNED;
3265     } else {
3266         pd = p->phys_offset;
3267     }
3268
3269     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3270         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3271         if (p)
3272             addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3273         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3274     } else {
3275         unsigned long addr1;
3276         addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3277         /* RAM case */
3278         ptr = phys_ram_base + addr1;
3279         stl_p(ptr, val);
3280         if (!cpu_physical_memory_is_dirty(addr1)) {
3281             /* invalidate code */
3282             tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3283             /* set dirty bit */
3284             phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3285                 (0xff & ~CODE_DIRTY_FLAG);
3286         }
3287     }
3288 }
3289
3290 /* XXX: optimize */
3291 void stb_phys(target_phys_addr_t addr, uint32_t val)
3292 {
3293     uint8_t v = val;
3294     cpu_physical_memory_write(addr, &v, 1);
3295 }
3296
3297 /* XXX: optimize */
3298 void stw_phys(target_phys_addr_t addr, uint32_t val)
3299 {
3300     uint16_t v = tswap16(val);
3301     cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3302 }
3303
3304 /* XXX: optimize */
3305 void stq_phys(target_phys_addr_t addr, uint64_t val)
3306 {
3307     val = tswap64(val);
3308     cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3309 }
3310
3311 #endif
3312
3313 /* virtual memory access for debug */
3314 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3315                         uint8_t *buf, int len, int is_write)
3316 {
3317     int l;
3318     target_phys_addr_t phys_addr;
3319     target_ulong page;
3320
3321     while (len > 0) {
3322         page = addr & TARGET_PAGE_MASK;
3323         phys_addr = cpu_get_phys_page_debug(env, page);
3324         /* if no physical page mapped, return an error */
3325         if (phys_addr == -1)
3326             return -1;
3327         l = (page + TARGET_PAGE_SIZE) - addr;
3328         if (l > len)
3329             l = len;
3330         cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3331                                buf, l, is_write);
3332         len -= l;
3333         buf += l;
3334         addr += l;
3335     }
3336     return 0;
3337 }
3338
3339 /* in deterministic execution mode, instructions doing device I/Os
3340    must be at the end of the TB */
3341 void cpu_io_recompile(CPUState *env, void *retaddr)
3342 {
3343     TranslationBlock *tb;
3344     uint32_t n, cflags;
3345     target_ulong pc, cs_base;
3346     uint64_t flags;
3347
3348     tb = tb_find_pc((unsigned long)retaddr);
3349     if (!tb) {
3350         cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3351                   retaddr);
3352     }
3353     n = env->icount_decr.u16.low + tb->icount;
3354     cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3355     /* Calculate how many instructions had been executed before the fault
3356        occurred.  */
3357     n = n - env->icount_decr.u16.low;
3358     /* Generate a new TB ending on the I/O insn.  */
3359     n++;
3360     /* On MIPS and SH, delay slot instructions can only be restarted if
3361        they were already the first instruction in the TB.  If this is not
3362        the first instruction in a TB then re-execute the preceding
3363        branch.  */
3364 #if defined(TARGET_MIPS)
3365     if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3366         env->active_tc.PC -= 4;
3367         env->icount_decr.u16.low++;
3368         env->hflags &= ~MIPS_HFLAG_BMASK;
3369     }
3370 #elif defined(TARGET_SH4)
3371     if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3372             && n > 1) {
3373         env->pc -= 2;
3374         env->icount_decr.u16.low++;
3375         env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3376     }
3377 #endif
3378     /* This should never happen.  */
3379     if (n > CF_COUNT_MASK)
3380         cpu_abort(env, "TB too big during recompile");
3381
3382     cflags = n | CF_LAST_IO;
3383     pc = tb->pc;
3384     cs_base = tb->cs_base;
3385     flags = tb->flags;
3386     tb_phys_invalidate(tb, -1);
3387     /* FIXME: In theory this could raise an exception.  In practice
3388        we have already translated the block once so it's probably ok.  */
3389     tb_gen_code(env, pc, cs_base, flags, cflags);
3390     /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3391        the first in the TB) then we end up generating a whole new TB and
3392        repeating the fault, which is horribly inefficient.
3393        Better would be to execute just this insn uncached, or generate a
3394        second new TB.  */
3395     cpu_resume_from_signal(env, NULL);
3396 }
3397
3398 void dump_exec_info(FILE *f,
3399                     int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3400 {
3401     int i, target_code_size, max_target_code_size;
3402     int direct_jmp_count, direct_jmp2_count, cross_page;
3403     TranslationBlock *tb;
3404
3405     target_code_size = 0;
3406     max_target_code_size = 0;
3407     cross_page = 0;
3408     direct_jmp_count = 0;
3409     direct_jmp2_count = 0;
3410     for(i = 0; i < nb_tbs; i++) {
3411         tb = &tbs[i];
3412         target_code_size += tb->size;
3413         if (tb->size > max_target_code_size)
3414             max_target_code_size = tb->size;
3415         if (tb->page_addr[1] != -1)
3416             cross_page++;
3417         if (tb->tb_next_offset[0] != 0xffff) {
3418             direct_jmp_count++;
3419             if (tb->tb_next_offset[1] != 0xffff) {
3420                 direct_jmp2_count++;
3421             }
3422         }
3423     }
3424     /* XXX: avoid using doubles ? */
3425     cpu_fprintf(f, "Translation buffer state:\n");
3426     cpu_fprintf(f, "gen code size       %ld/%ld\n",
3427                 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3428     cpu_fprintf(f, "TB count            %d/%d\n",
3429                 nb_tbs, code_gen_max_blocks);
3430     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
3431                 nb_tbs ? target_code_size / nb_tbs : 0,
3432                 max_target_code_size);
3433     cpu_fprintf(f, "TB avg host size    %d bytes (expansion ratio: %0.1f)\n",
3434                 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3435                 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3436     cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3437             cross_page,
3438             nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3439     cpu_fprintf(f, "direct jump count   %d (%d%%) (2 jumps=%d %d%%)\n",
3440                 direct_jmp_count,
3441                 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3442                 direct_jmp2_count,
3443                 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3444     cpu_fprintf(f, "\nStatistics:\n");
3445     cpu_fprintf(f, "TB flush count      %d\n", tb_flush_count);
3446     cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3447     cpu_fprintf(f, "TLB flush count     %d\n", tlb_flush_count);
3448     tcg_dump_info(f, cpu_fprintf);
3449 }
3450
3451 #if !defined(CONFIG_USER_ONLY)
3452
3453 #define MMUSUFFIX _cmmu
3454 #define GETPC() NULL
3455 #define env cpu_single_env
3456 #define SOFTMMU_CODE_ACCESS
3457
3458 #define SHIFT 0
3459 #include "softmmu_template.h"
3460
3461 #define SHIFT 1
3462 #include "softmmu_template.h"
3463
3464 #define SHIFT 2
3465 #include "softmmu_template.h"
3466
3467 #define SHIFT 3
3468 #include "softmmu_template.h"
3469
3470 #undef env
3471
3472 #endif