?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Fix qemu_send_packet
[qemu-kvm/fedora.git] / exec.c
blobe2660f11cdc8ba2638b8d096e32ba2fdf20a5029
1 /*
2 * virtual page mapping and translated block handling
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
19 */
20 #include "config.h"
21 #ifdef _WIN32
22 #define WIN32_LEAN_AND_MEAN
23 #include <windows.h>
24 #else
25 #include <sys/types.h>
26 #include <sys/mman.h>
27 #endif
28 #include <stdlib.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <unistd.h>
34 #include <inttypes.h>
35
36 #include "cpu.h"
37 #include "exec-all.h"
38 #include "qemu-common.h"
39
40 #if !defined(TARGET_IA64)
41 #include "tcg.h"
42 #endif
43 #include "qemu-kvm.h"
44
45 #include "hw/hw.h"
46 #include "osdep.h"
47 #include "kvm.h"
48 #if defined(CONFIG_USER_ONLY)
49 #include <qemu.h>
50 #endif
51
52 //#define DEBUG_TB_INVALIDATE
53 //#define DEBUG_FLUSH
54 //#define DEBUG_TLB
55 //#define DEBUG_UNASSIGNED
56
57 /* make various TB consistency checks */
58 //#define DEBUG_TB_CHECK
59 //#define DEBUG_TLB_CHECK
60
61 //#define DEBUG_IOPORT
62 //#define DEBUG_SUBPAGE
63
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
68
69 #define SMC_BITMAP_USE_THRESHOLD 10
70
71 #define MMAP_AREA_START 0x00000000
72 #define MMAP_AREA_END 0xa8000000
73
74 #if defined(TARGET_SPARC64)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 41
76 #elif defined(TARGET_SPARC)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #elif defined(TARGET_ALPHA)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
80 #define TARGET_VIRT_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_PPC64)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 42
83 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
85 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
87 #elif defined(TARGET_IA64)
88 #define TARGET_PHYS_ADDR_SPACE_BITS 36
89 #else
90 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
91 #define TARGET_PHYS_ADDR_SPACE_BITS 32
92 #endif
93
94 static TranslationBlock *tbs;
95 int code_gen_max_blocks;
96 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
97 static int nb_tbs;
98 /* any access to the tbs or the page table must use this lock */
99 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
100
101 #if defined(__arm__) || defined(__sparc_v9__)
102 /* The prologue must be reachable with a direct jump. ARM and Sparc64
103 have limited branch ranges (possibly also PPC) so place it in a
104 section close to code segment. */
105 #define code_gen_section \
106 __attribute__((__section__(".gen_code"))) \
107 __attribute__((aligned (32)))
108 #else
109 #define code_gen_section \
110 __attribute__((aligned (32)))
111 #endif
112
113 uint8_t code_gen_prologue[1024] code_gen_section;
114 static uint8_t *code_gen_buffer;
115 static unsigned long code_gen_buffer_size;
116 /* threshold to flush the translated code buffer */
117 static unsigned long code_gen_buffer_max_size;
118 uint8_t *code_gen_ptr;
119
120 #if !defined(CONFIG_USER_ONLY)
121 ram_addr_t phys_ram_size;
122 int phys_ram_fd;
123 uint8_t *phys_ram_base;
124 uint8_t *phys_ram_dirty;
125 uint8_t *bios_mem;
126 static int in_migration;
127 static ram_addr_t phys_ram_alloc_offset = 0;
128 #endif
129
130 CPUState *first_cpu;
131 /* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133 CPUState *cpu_single_env;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137 int use_icount = 0;
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140 int64_t qemu_icount;
141
142 typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149 #if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151 #endif
152 } PageDesc;
153
154 typedef struct PhysPageDesc {
155 /* offset in host memory of the page + io_index in the low bits */
156 ram_addr_t phys_offset;
157 ram_addr_t region_offset;
158 } PhysPageDesc;
159
160 #define L2_BITS 10
161 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
162 /* XXX: this is a temporary hack for alpha target.
163 * In the future, this is to be replaced by a multi-level table
164 * to actually be able to handle the complete 64 bits address space.
165 */
166 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
167 #else
168 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
169 #endif
170
171 #define L1_SIZE (1 << L1_BITS)
172 #define L2_SIZE (1 << L2_BITS)
173
174 unsigned long qemu_real_host_page_size;
175 unsigned long qemu_host_page_bits;
176 unsigned long qemu_host_page_size;
177 unsigned long qemu_host_page_mask;
178
179 /* XXX: for system emulation, it could just be an array */
180 static PageDesc *l1_map[L1_SIZE];
181 static PhysPageDesc **l1_phys_map;
182
183 #if !defined(CONFIG_USER_ONLY)
184 static void io_mem_init(void);
185
186 /* io memory support */
187 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
188 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
189 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
190 char io_mem_used[IO_MEM_NB_ENTRIES];
191 static int io_mem_watch;
192 #endif
193
194 /* log support */
195 static const char *logfilename = "/tmp/qemu.log";
196 FILE *logfile;
197 int loglevel;
198 static int log_append = 0;
199
200 /* statistics */
201 static int tlb_flush_count;
202 static int tb_flush_count;
203 static int tb_phys_invalidate_count;
204
205 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
206 typedef struct subpage_t {
207 target_phys_addr_t base;
208 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
209 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
210 void *opaque[TARGET_PAGE_SIZE][2][4];
211 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
212 } subpage_t;
213
214 #ifdef _WIN32
215 static void map_exec(void *addr, long size)
216 {
217 DWORD old_protect;
218 VirtualProtect(addr, size,
219 PAGE_EXECUTE_READWRITE, &old_protect);
220
221 }
222 #else
223 static void map_exec(void *addr, long size)
224 {
225 unsigned long start, end, page_size;
226
227 page_size = getpagesize();
228 start = (unsigned long)addr;
229 start &= ~(page_size - 1);
230
231 end = (unsigned long)addr + size;
232 end += page_size - 1;
233 end &= ~(page_size - 1);
234
235 mprotect((void *)start, end - start,
236 PROT_READ | PROT_WRITE | PROT_EXEC);
237 }
238 #endif
239
240 static void page_init(void)
241 {
242 /* NOTE: we can always suppose that qemu_host_page_size >=
243 TARGET_PAGE_SIZE */
244 #ifdef _WIN32
245 {
246 SYSTEM_INFO system_info;
247
248 GetSystemInfo(&system_info);
249 qemu_real_host_page_size = system_info.dwPageSize;
250 }
251 #else
252 qemu_real_host_page_size = getpagesize();
253 #endif
254 if (qemu_host_page_size == 0)
255 qemu_host_page_size = qemu_real_host_page_size;
256 if (qemu_host_page_size < TARGET_PAGE_SIZE)
257 qemu_host_page_size = TARGET_PAGE_SIZE;
258 qemu_host_page_bits = 0;
259 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
260 qemu_host_page_bits++;
261 qemu_host_page_mask = ~(qemu_host_page_size - 1);
262 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
263 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
264
265 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
266 {
267 long long startaddr, endaddr;
268 FILE *f;
269 int n;
270
271 mmap_lock();
272 last_brk = (unsigned long)sbrk(0);
273 f = fopen("/proc/self/maps", "r");
274 if (f) {
275 do {
276 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
277 if (n == 2) {
278 startaddr = MIN(startaddr,
279 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
280 endaddr = MIN(endaddr,
281 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
282 page_set_flags(startaddr & TARGET_PAGE_MASK,
283 TARGET_PAGE_ALIGN(endaddr),
284 PAGE_RESERVED);
285 }
286 } while (!feof(f));
287 fclose(f);
288 }
289 mmap_unlock();
290 }
291 #endif
292 }
293
294 static inline PageDesc **page_l1_map(target_ulong index)
295 {
296 #if TARGET_LONG_BITS > 32
297 /* Host memory outside guest VM. For 32-bit targets we have already
298 excluded high addresses. */
299 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
300 return NULL;
301 #endif
302 return &l1_map[index >> L2_BITS];
303 }
304
305 static inline PageDesc *page_find_alloc(target_ulong index)
306 {
307 PageDesc **lp, *p;
308 lp = page_l1_map(index);
309 if (!lp)
310 return NULL;
311
312 p = *lp;
313 if (!p) {
314 /* allocate if not found */
315 #if defined(CONFIG_USER_ONLY)
316 size_t len = sizeof(PageDesc) * L2_SIZE;
317 /* Don't use qemu_malloc because it may recurse. */
318 p = mmap(0, len, PROT_READ | PROT_WRITE,
319 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
320 *lp = p;
321 if (h2g_valid(p)) {
322 unsigned long addr = h2g(p);
323 page_set_flags(addr & TARGET_PAGE_MASK,
324 TARGET_PAGE_ALIGN(addr + len),
325 PAGE_RESERVED);
326 }
327 #else
328 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
329 *lp = p;
330 #endif
331 }
332 return p + (index & (L2_SIZE - 1));
333 }
334
335 static inline PageDesc *page_find(target_ulong index)
336 {
337 PageDesc **lp, *p;
338 lp = page_l1_map(index);
339 if (!lp)
340 return NULL;
341
342 p = *lp;
343 if (!p)
344 return 0;
345 return p + (index & (L2_SIZE - 1));
346 }
347
348 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
349 {
350 void **lp, **p;
351 PhysPageDesc *pd;
352
353 p = (void **)l1_phys_map;
354 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
355
356 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
357 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
358 #endif
359 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
360 p = *lp;
361 if (!p) {
362 /* allocate if not found */
363 if (!alloc)
364 return NULL;
365 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
366 memset(p, 0, sizeof(void *) * L1_SIZE);
367 *lp = p;
368 }
369 #endif
370 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
371 pd = *lp;
372 if (!pd) {
373 int i;
374 /* allocate if not found */
375 if (!alloc)
376 return NULL;
377 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
378 *lp = pd;
379 for (i = 0; i < L2_SIZE; i++)
380 pd[i].phys_offset = IO_MEM_UNASSIGNED;
381 }
382 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
383 }
384
385 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
386 {
387 return phys_page_find_alloc(index, 0);
388 }
389
390 #if !defined(CONFIG_USER_ONLY)
391 static void tlb_protect_code(ram_addr_t ram_addr);
392 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
393 target_ulong vaddr);
394 #define mmap_lock() do { } while(0)
395 #define mmap_unlock() do { } while(0)
396 #endif
397
398 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
399
400 #if defined(CONFIG_USER_ONLY)
401 /* Currently it is not recommanded to allocate big chunks of data in
402 user mode. It will change when a dedicated libc will be used */
403 #define USE_STATIC_CODE_GEN_BUFFER
404 #endif
405
406 #ifdef USE_STATIC_CODE_GEN_BUFFER
407 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
408 #endif
409
410 static void code_gen_alloc(unsigned long tb_size)
411 {
412 if (kvm_enabled())
413 return;
414
415 #ifdef USE_STATIC_CODE_GEN_BUFFER
416 code_gen_buffer = static_code_gen_buffer;
417 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
418 map_exec(code_gen_buffer, code_gen_buffer_size);
419 #else
420 code_gen_buffer_size = tb_size;
421 if (code_gen_buffer_size == 0) {
422 #if defined(CONFIG_USER_ONLY)
423 /* in user mode, phys_ram_size is not meaningful */
424 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
425 #else
426 /* XXX: needs ajustments */
427 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
428 #endif
429 }
430 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
431 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
432 /* The code gen buffer location may have constraints depending on
433 the host cpu and OS */
434 #if defined(__linux__)
435 {
436 int flags;
437 void *start = NULL;
438
439 flags = MAP_PRIVATE | MAP_ANONYMOUS;
440 #if defined(__x86_64__)
441 flags |= MAP_32BIT;
442 /* Cannot map more than that */
443 if (code_gen_buffer_size > (800 * 1024 * 1024))
444 code_gen_buffer_size = (800 * 1024 * 1024);
445 #elif defined(__sparc_v9__)
446 // Map the buffer below 2G, so we can use direct calls and branches
447 flags |= MAP_FIXED;
448 start = (void *) 0x60000000UL;
449 if (code_gen_buffer_size > (512 * 1024 * 1024))
450 code_gen_buffer_size = (512 * 1024 * 1024);
451 #elif defined(__arm__)
452 /* Map the buffer below 32M, so we can use direct calls and branches */
453 flags |= MAP_FIXED;
454 start = (void *) 0x01000000UL;
455 if (code_gen_buffer_size > 16 * 1024 * 1024)
456 code_gen_buffer_size = 16 * 1024 * 1024;
457 #endif
458 code_gen_buffer = mmap(start, code_gen_buffer_size,
459 PROT_WRITE | PROT_READ | PROT_EXEC,
460 flags, -1, 0);
461 if (code_gen_buffer == MAP_FAILED) {
462 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
463 exit(1);
464 }
465 }
466 #elif defined(__FreeBSD__)
467 {
468 int flags;
469 void *addr = NULL;
470 flags = MAP_PRIVATE | MAP_ANONYMOUS;
471 #if defined(__x86_64__)
472 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
473 * 0x40000000 is free */
474 flags |= MAP_FIXED;
475 addr = (void *)0x40000000;
476 /* Cannot map more than that */
477 if (code_gen_buffer_size > (800 * 1024 * 1024))
478 code_gen_buffer_size = (800 * 1024 * 1024);
479 #endif
480 code_gen_buffer = mmap(addr, code_gen_buffer_size,
481 PROT_WRITE | PROT_READ | PROT_EXEC,
482 flags, -1, 0);
483 if (code_gen_buffer == MAP_FAILED) {
484 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
485 exit(1);
486 }
487 }
488 #else
489 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
490 if (!code_gen_buffer) {
491 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
492 exit(1);
493 }
494 map_exec(code_gen_buffer, code_gen_buffer_size);
495 #endif
496 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
497 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
498 code_gen_buffer_max_size = code_gen_buffer_size -
499 code_gen_max_block_size();
500 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
501 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
502 }
503
504 /* Must be called before using the QEMU cpus. 'tb_size' is the size
505 (in bytes) allocated to the translation buffer. Zero means default
506 size. */
507 void cpu_exec_init_all(unsigned long tb_size)
508 {
509 cpu_gen_init();
510 code_gen_alloc(tb_size);
511 code_gen_ptr = code_gen_buffer;
512 page_init();
513 #if !defined(CONFIG_USER_ONLY)
514 io_mem_init();
515 #endif
516 }
517
518 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
519
520 #define CPU_COMMON_SAVE_VERSION 1
521
522 static void cpu_common_save(QEMUFile *f, void *opaque)
523 {
524 CPUState *env = opaque;
525
526 qemu_put_be32s(f, &env->halted);
527 qemu_put_be32s(f, &env->interrupt_request);
528 }
529
530 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
531 {
532 CPUState *env = opaque;
533
534 if (version_id != CPU_COMMON_SAVE_VERSION)
535 return -EINVAL;
536
537 qemu_get_be32s(f, &env->halted);
538 qemu_get_be32s(f, &env->interrupt_request);
539 tlb_flush(env, 1);
540
541 return 0;
542 }
543 #endif
544
545 void cpu_exec_init(CPUState *env)
546 {
547 CPUState **penv;
548 int cpu_index;
549
550 env->next_cpu = NULL;
551 penv = &first_cpu;
552 cpu_index = 0;
553 while (*penv != NULL) {
554 penv = (CPUState **)&(*penv)->next_cpu;
555 cpu_index++;
556 }
557 env->cpu_index = cpu_index;
558 TAILQ_INIT(&env->breakpoints);
559 TAILQ_INIT(&env->watchpoints);
560 #ifdef __WIN32
561 env->thread_id = GetCurrentProcessId();
562 #else
563 env->thread_id = getpid();
564 #endif
565 *penv = env;
566 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
567 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
568 cpu_common_save, cpu_common_load, env);
569 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
570 cpu_save, cpu_load, env);
571 #endif
572 }
573
574 static inline void invalidate_page_bitmap(PageDesc *p)
575 {
576 if (p->code_bitmap) {
577 qemu_free(p->code_bitmap);
578 p->code_bitmap = NULL;
579 }
580 p->code_write_count = 0;
581 }
582
583 /* set to NULL all the 'first_tb' fields in all PageDescs */
584 static void page_flush_tb(void)
585 {
586 int i, j;
587 PageDesc *p;
588
589 for(i = 0; i < L1_SIZE; i++) {
590 p = l1_map[i];
591 if (p) {
592 for(j = 0; j < L2_SIZE; j++) {
593 p->first_tb = NULL;
594 invalidate_page_bitmap(p);
595 p++;
596 }
597 }
598 }
599 }
600
601 /* flush all the translation blocks */
602 /* XXX: tb_flush is currently not thread safe */
603 void tb_flush(CPUState *env1)
604 {
605 CPUState *env;
606 #if defined(DEBUG_FLUSH)
607 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
608 (unsigned long)(code_gen_ptr - code_gen_buffer),
609 nb_tbs, nb_tbs > 0 ?
610 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
611 #endif
612 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
613 cpu_abort(env1, "Internal error: code buffer overflow\n");
614
615 nb_tbs = 0;
616
617 for(env = first_cpu; env != NULL; env = env->next_cpu) {
618 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
619 }
620
621 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
622 page_flush_tb();
623
624 code_gen_ptr = code_gen_buffer;
625 /* XXX: flush processor icache at this point if cache flush is
626 expensive */
627 tb_flush_count++;
628 }
629
630 #ifdef DEBUG_TB_CHECK
631
632 static void tb_invalidate_check(target_ulong address)
633 {
634 TranslationBlock *tb;
635 int i;
636 address &= TARGET_PAGE_MASK;
637 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
638 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
639 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
640 address >= tb->pc + tb->size)) {
641 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
642 address, (long)tb->pc, tb->size);
643 }
644 }
645 }
646 }
647
648 /* verify that all the pages have correct rights for code */
649 static void tb_page_check(void)
650 {
651 TranslationBlock *tb;
652 int i, flags1, flags2;
653
654 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
655 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
656 flags1 = page_get_flags(tb->pc);
657 flags2 = page_get_flags(tb->pc + tb->size - 1);
658 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
659 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
660 (long)tb->pc, tb->size, flags1, flags2);
661 }
662 }
663 }
664 }
665
666 static void tb_jmp_check(TranslationBlock *tb)
667 {
668 TranslationBlock *tb1;
669 unsigned int n1;
670
671 /* suppress any remaining jumps to this TB */
672 tb1 = tb->jmp_first;
673 for(;;) {
674 n1 = (long)tb1 & 3;
675 tb1 = (TranslationBlock *)((long)tb1 & ~3);
676 if (n1 == 2)
677 break;
678 tb1 = tb1->jmp_next[n1];
679 }
680 /* check end of list */
681 if (tb1 != tb) {
682 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
683 }
684 }
685
686 #endif
687
688 /* invalidate one TB */
689 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
690 int next_offset)
691 {
692 TranslationBlock *tb1;
693 for(;;) {
694 tb1 = *ptb;
695 if (tb1 == tb) {
696 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
697 break;
698 }
699 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
700 }
701 }
702
703 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
704 {
705 TranslationBlock *tb1;
706 unsigned int n1;
707
708 for(;;) {
709 tb1 = *ptb;
710 n1 = (long)tb1 & 3;
711 tb1 = (TranslationBlock *)((long)tb1 & ~3);
712 if (tb1 == tb) {
713 *ptb = tb1->page_next[n1];
714 break;
715 }
716 ptb = &tb1->page_next[n1];
717 }
718 }
719
720 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
721 {
722 TranslationBlock *tb1, **ptb;
723 unsigned int n1;
724
725 ptb = &tb->jmp_next[n];
726 tb1 = *ptb;
727 if (tb1) {
728 /* find tb(n) in circular list */
729 for(;;) {
730 tb1 = *ptb;
731 n1 = (long)tb1 & 3;
732 tb1 = (TranslationBlock *)((long)tb1 & ~3);
733 if (n1 == n && tb1 == tb)
734 break;
735 if (n1 == 2) {
736 ptb = &tb1->jmp_first;
737 } else {
738 ptb = &tb1->jmp_next[n1];
739 }
740 }
741 /* now we can suppress tb(n) from the list */
742 *ptb = tb->jmp_next[n];
743
744 tb->jmp_next[n] = NULL;
745 }
746 }
747
748 /* reset the jump entry 'n' of a TB so that it is not chained to
749 another TB */
750 static inline void tb_reset_jump(TranslationBlock *tb, int n)
751 {
752 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
753 }
754
755 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
756 {
757 CPUState *env;
758 PageDesc *p;
759 unsigned int h, n1;
760 target_phys_addr_t phys_pc;
761 TranslationBlock *tb1, *tb2;
762
763 /* remove the TB from the hash list */
764 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
765 h = tb_phys_hash_func(phys_pc);
766 tb_remove(&tb_phys_hash[h], tb,
767 offsetof(TranslationBlock, phys_hash_next));
768
769 /* remove the TB from the page list */
770 if (tb->page_addr[0] != page_addr) {
771 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
772 tb_page_remove(&p->first_tb, tb);
773 invalidate_page_bitmap(p);
774 }
775 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
776 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
777 tb_page_remove(&p->first_tb, tb);
778 invalidate_page_bitmap(p);
779 }
780
781 tb_invalidated_flag = 1;
782
783 /* remove the TB from the hash list */
784 h = tb_jmp_cache_hash_func(tb->pc);
785 for(env = first_cpu; env != NULL; env = env->next_cpu) {
786 if (env->tb_jmp_cache[h] == tb)
787 env->tb_jmp_cache[h] = NULL;
788 }
789
790 /* suppress this TB from the two jump lists */
791 tb_jmp_remove(tb, 0);
792 tb_jmp_remove(tb, 1);
793
794 /* suppress any remaining jumps to this TB */
795 tb1 = tb->jmp_first;
796 for(;;) {
797 n1 = (long)tb1 & 3;
798 if (n1 == 2)
799 break;
800 tb1 = (TranslationBlock *)((long)tb1 & ~3);
801 tb2 = tb1->jmp_next[n1];
802 tb_reset_jump(tb1, n1);
803 tb1->jmp_next[n1] = NULL;
804 tb1 = tb2;
805 }
806 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
807
808 tb_phys_invalidate_count++;
809 }
810
811 static inline void set_bits(uint8_t *tab, int start, int len)
812 {
813 int end, mask, end1;
814
815 end = start + len;
816 tab += start >> 3;
817 mask = 0xff << (start & 7);
818 if ((start & ~7) == (end & ~7)) {
819 if (start < end) {
820 mask &= ~(0xff << (end & 7));
821 *tab |= mask;
822 }
823 } else {
824 *tab++ |= mask;
825 start = (start + 8) & ~7;
826 end1 = end & ~7;
827 while (start < end1) {
828 *tab++ = 0xff;
829 start += 8;
830 }
831 if (start < end) {
832 mask = ~(0xff << (end & 7));
833 *tab |= mask;
834 }
835 }
836 }
837
838 static void build_page_bitmap(PageDesc *p)
839 {
840 int n, tb_start, tb_end;
841 TranslationBlock *tb;
842
843 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
844 if (!p->code_bitmap)
845 return;
846
847 tb = p->first_tb;
848 while (tb != NULL) {
849 n = (long)tb & 3;
850 tb = (TranslationBlock *)((long)tb & ~3);
851 /* NOTE: this is subtle as a TB may span two physical pages */
852 if (n == 0) {
853 /* NOTE: tb_end may be after the end of the page, but
854 it is not a problem */
855 tb_start = tb->pc & ~TARGET_PAGE_MASK;
856 tb_end = tb_start + tb->size;
857 if (tb_end > TARGET_PAGE_SIZE)
858 tb_end = TARGET_PAGE_SIZE;
859 } else {
860 tb_start = 0;
861 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
862 }
863 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
864 tb = tb->page_next[n];
865 }
866 }
867
868 TranslationBlock *tb_gen_code(CPUState *env,
869 target_ulong pc, target_ulong cs_base,
870 int flags, int cflags)
871 {
872 TranslationBlock *tb;
873 uint8_t *tc_ptr;
874 target_ulong phys_pc, phys_page2, virt_page2;
875 int code_gen_size;
876
877 phys_pc = get_phys_addr_code(env, pc);
878 tb = tb_alloc(pc);
879 if (!tb) {
880 /* flush must be done */
881 tb_flush(env);
882 /* cannot fail at this point */
883 tb = tb_alloc(pc);
884 /* Don't forget to invalidate previous TB info. */
885 tb_invalidated_flag = 1;
886 }
887 tc_ptr = code_gen_ptr;
888 tb->tc_ptr = tc_ptr;
889 tb->cs_base = cs_base;
890 tb->flags = flags;
891 tb->cflags = cflags;
892 cpu_gen_code(env, tb, &code_gen_size);
893 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
894
895 /* check next page if needed */
896 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
897 phys_page2 = -1;
898 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
899 phys_page2 = get_phys_addr_code(env, virt_page2);
900 }
901 tb_link_phys(tb, phys_pc, phys_page2);
902 return tb;
903 }
904
905 /* invalidate all TBs which intersect with the target physical page
906 starting in range [start;end[. NOTE: start and end must refer to
907 the same physical page. 'is_cpu_write_access' should be true if called
908 from a real cpu write access: the virtual CPU will exit the current
909 TB if code is modified inside this TB. */
910 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
911 int is_cpu_write_access)
912 {
913 TranslationBlock *tb, *tb_next, *saved_tb;
914 CPUState *env = cpu_single_env;
915 target_ulong tb_start, tb_end;
916 PageDesc *p;
917 int n;
918 #ifdef TARGET_HAS_PRECISE_SMC
919 int current_tb_not_found = is_cpu_write_access;
920 TranslationBlock *current_tb = NULL;
921 int current_tb_modified = 0;
922 target_ulong current_pc = 0;
923 target_ulong current_cs_base = 0;
924 int current_flags = 0;
925 #endif /* TARGET_HAS_PRECISE_SMC */
926
927 p = page_find(start >> TARGET_PAGE_BITS);
928 if (!p)
929 return;
930 if (!p->code_bitmap &&
931 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
932 is_cpu_write_access) {
933 /* build code bitmap */
934 build_page_bitmap(p);
935 }
936
937 /* we remove all the TBs in the range [start, end[ */
938 /* XXX: see if in some cases it could be faster to invalidate all the code */
939 tb = p->first_tb;
940 while (tb != NULL) {
941 n = (long)tb & 3;
942 tb = (TranslationBlock *)((long)tb & ~3);
943 tb_next = tb->page_next[n];
944 /* NOTE: this is subtle as a TB may span two physical pages */
945 if (n == 0) {
946 /* NOTE: tb_end may be after the end of the page, but
947 it is not a problem */
948 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
949 tb_end = tb_start + tb->size;
950 } else {
951 tb_start = tb->page_addr[1];
952 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
953 }
954 if (!(tb_end <= start || tb_start >= end)) {
955 #ifdef TARGET_HAS_PRECISE_SMC
956 if (current_tb_not_found) {
957 current_tb_not_found = 0;
958 current_tb = NULL;
959 if (env->mem_io_pc) {
960 /* now we have a real cpu fault */
961 current_tb = tb_find_pc(env->mem_io_pc);
962 }
963 }
964 if (current_tb == tb &&
965 (current_tb->cflags & CF_COUNT_MASK) != 1) {
966 /* If we are modifying the current TB, we must stop
967 its execution. We could be more precise by checking
968 that the modification is after the current PC, but it
969 would require a specialized function to partially
970 restore the CPU state */
971
972 current_tb_modified = 1;
973 cpu_restore_state(current_tb, env,
974 env->mem_io_pc, NULL);
975 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
976 &current_flags);
977 }
978 #endif /* TARGET_HAS_PRECISE_SMC */
979 /* we need to do that to handle the case where a signal
980 occurs while doing tb_phys_invalidate() */
981 saved_tb = NULL;
982 if (env) {
983 saved_tb = env->current_tb;
984 env->current_tb = NULL;
985 }
986 tb_phys_invalidate(tb, -1);
987 if (env) {
988 env->current_tb = saved_tb;
989 if (env->interrupt_request && env->current_tb)
990 cpu_interrupt(env, env->interrupt_request);
991 }
992 }
993 tb = tb_next;
994 }
995 #if !defined(CONFIG_USER_ONLY)
996 /* if no code remaining, no need to continue to use slow writes */
997 if (!p->first_tb) {
998 invalidate_page_bitmap(p);
999 if (is_cpu_write_access) {
1000 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1001 }
1002 }
1003 #endif
1004 #ifdef TARGET_HAS_PRECISE_SMC
1005 if (current_tb_modified) {
1006 /* we generate a block containing just the instruction
1007 modifying the memory. It will ensure that it cannot modify
1008 itself */
1009 env->current_tb = NULL;
1010 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1011 cpu_resume_from_signal(env, NULL);
1012 }
1013 #endif
1014 }
1015
1016 /* len must be <= 8 and start must be a multiple of len */
1017 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1018 {
1019 PageDesc *p;
1020 int offset, b;
1021 #if 0
1022 if (1) {
1023 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1024 cpu_single_env->mem_io_vaddr, len,
1025 cpu_single_env->eip,
1026 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1027 }
1028 #endif
1029 p = page_find(start >> TARGET_PAGE_BITS);
1030 if (!p)
1031 return;
1032 if (p->code_bitmap) {
1033 offset = start & ~TARGET_PAGE_MASK;
1034 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1035 if (b & ((1 << len) - 1))
1036 goto do_invalidate;
1037 } else {
1038 do_invalidate:
1039 tb_invalidate_phys_page_range(start, start + len, 1);
1040 }
1041 }
1042
1043 #if !defined(CONFIG_SOFTMMU)
1044 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1045 unsigned long pc, void *puc)
1046 {
1047 TranslationBlock *tb;
1048 PageDesc *p;
1049 int n;
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 TranslationBlock *current_tb = NULL;
1052 CPUState *env = cpu_single_env;
1053 int current_tb_modified = 0;
1054 target_ulong current_pc = 0;
1055 target_ulong current_cs_base = 0;
1056 int current_flags = 0;
1057 #endif
1058
1059 addr &= TARGET_PAGE_MASK;
1060 p = page_find(addr >> TARGET_PAGE_BITS);
1061 if (!p)
1062 return;
1063 tb = p->first_tb;
1064 #ifdef TARGET_HAS_PRECISE_SMC
1065 if (tb && pc != 0) {
1066 current_tb = tb_find_pc(pc);
1067 }
1068 #endif
1069 while (tb != NULL) {
1070 n = (long)tb & 3;
1071 tb = (TranslationBlock *)((long)tb & ~3);
1072 #ifdef TARGET_HAS_PRECISE_SMC
1073 if (current_tb == tb &&
1074 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1075 /* If we are modifying the current TB, we must stop
1076 its execution. We could be more precise by checking
1077 that the modification is after the current PC, but it
1078 would require a specialized function to partially
1079 restore the CPU state */
1080
1081 current_tb_modified = 1;
1082 cpu_restore_state(current_tb, env, pc, puc);
1083 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1084 &current_flags);
1085 }
1086 #endif /* TARGET_HAS_PRECISE_SMC */
1087 tb_phys_invalidate(tb, addr);
1088 tb = tb->page_next[n];
1089 }
1090 p->first_tb = NULL;
1091 #ifdef TARGET_HAS_PRECISE_SMC
1092 if (current_tb_modified) {
1093 /* we generate a block containing just the instruction
1094 modifying the memory. It will ensure that it cannot modify
1095 itself */
1096 env->current_tb = NULL;
1097 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1098 cpu_resume_from_signal(env, puc);
1099 }
1100 #endif
1101 }
1102 #endif
1103
1104 /* add the tb in the target page and protect it if necessary */
1105 static inline void tb_alloc_page(TranslationBlock *tb,
1106 unsigned int n, target_ulong page_addr)
1107 {
1108 PageDesc *p;
1109 TranslationBlock *last_first_tb;
1110
1111 tb->page_addr[n] = page_addr;
1112 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1113 tb->page_next[n] = p->first_tb;
1114 last_first_tb = p->first_tb;
1115 p->first_tb = (TranslationBlock *)((long)tb | n);
1116 invalidate_page_bitmap(p);
1117
1118 #if defined(TARGET_HAS_SMC) || 1
1119
1120 #if defined(CONFIG_USER_ONLY)
1121 if (p->flags & PAGE_WRITE) {
1122 target_ulong addr;
1123 PageDesc *p2;
1124 int prot;
1125
1126 /* force the host page as non writable (writes will have a
1127 page fault + mprotect overhead) */
1128 page_addr &= qemu_host_page_mask;
1129 prot = 0;
1130 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1131 addr += TARGET_PAGE_SIZE) {
1132
1133 p2 = page_find (addr >> TARGET_PAGE_BITS);
1134 if (!p2)
1135 continue;
1136 prot |= p2->flags;
1137 p2->flags &= ~PAGE_WRITE;
1138 page_get_flags(addr);
1139 }
1140 mprotect(g2h(page_addr), qemu_host_page_size,
1141 (prot & PAGE_BITS) & ~PAGE_WRITE);
1142 #ifdef DEBUG_TB_INVALIDATE
1143 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1144 page_addr);
1145 #endif
1146 }
1147 #else
1148 /* if some code is already present, then the pages are already
1149 protected. So we handle the case where only the first TB is
1150 allocated in a physical page */
1151 if (!last_first_tb) {
1152 tlb_protect_code(page_addr);
1153 }
1154 #endif
1155
1156 #endif /* TARGET_HAS_SMC */
1157 }
1158
1159 /* Allocate a new translation block. Flush the translation buffer if
1160 too many translation blocks or too much generated code. */
1161 TranslationBlock *tb_alloc(target_ulong pc)
1162 {
1163 TranslationBlock *tb;
1164
1165 if (nb_tbs >= code_gen_max_blocks ||
1166 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1167 return NULL;
1168 tb = &tbs[nb_tbs++];
1169 tb->pc = pc;
1170 tb->cflags = 0;
1171 return tb;
1172 }
1173
1174 void tb_free(TranslationBlock *tb)
1175 {
1176 /* In practice this is mostly used for single use temporary TB
1177 Ignore the hard cases and just back up if this TB happens to
1178 be the last one generated. */
1179 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1180 code_gen_ptr = tb->tc_ptr;
1181 nb_tbs--;
1182 }
1183 }
1184
1185 /* add a new TB and link it to the physical page tables. phys_page2 is
1186 (-1) to indicate that only one page contains the TB. */
1187 void tb_link_phys(TranslationBlock *tb,
1188 target_ulong phys_pc, target_ulong phys_page2)
1189 {
1190 unsigned int h;
1191 TranslationBlock **ptb;
1192
1193 /* Grab the mmap lock to stop another thread invalidating this TB
1194 before we are done. */
1195 mmap_lock();
1196 /* add in the physical hash table */
1197 h = tb_phys_hash_func(phys_pc);
1198 ptb = &tb_phys_hash[h];
1199 tb->phys_hash_next = *ptb;
1200 *ptb = tb;
1201
1202 /* add in the page list */
1203 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1204 if (phys_page2 != -1)
1205 tb_alloc_page(tb, 1, phys_page2);
1206 else
1207 tb->page_addr[1] = -1;
1208
1209 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1210 tb->jmp_next[0] = NULL;
1211 tb->jmp_next[1] = NULL;
1212
1213 /* init original jump addresses */
1214 if (tb->tb_next_offset[0] != 0xffff)
1215 tb_reset_jump(tb, 0);
1216 if (tb->tb_next_offset[1] != 0xffff)
1217 tb_reset_jump(tb, 1);
1218
1219 #ifdef DEBUG_TB_CHECK
1220 tb_page_check();
1221 #endif
1222 mmap_unlock();
1223 }
1224
1225 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1226 tb[1].tc_ptr. Return NULL if not found */
1227 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1228 {
1229 int m_min, m_max, m;
1230 unsigned long v;
1231 TranslationBlock *tb;
1232
1233 if (nb_tbs <= 0)
1234 return NULL;
1235 if (tc_ptr < (unsigned long)code_gen_buffer ||
1236 tc_ptr >= (unsigned long)code_gen_ptr)
1237 return NULL;
1238 /* binary search (cf Knuth) */
1239 m_min = 0;
1240 m_max = nb_tbs - 1;
1241 while (m_min <= m_max) {
1242 m = (m_min + m_max) >> 1;
1243 tb = &tbs[m];
1244 v = (unsigned long)tb->tc_ptr;
1245 if (v == tc_ptr)
1246 return tb;
1247 else if (tc_ptr < v) {
1248 m_max = m - 1;
1249 } else {
1250 m_min = m + 1;
1251 }
1252 }
1253 return &tbs[m_max];
1254 }
1255
1256 static void tb_reset_jump_recursive(TranslationBlock *tb);
1257
1258 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1259 {
1260 TranslationBlock *tb1, *tb_next, **ptb;
1261 unsigned int n1;
1262
1263 tb1 = tb->jmp_next[n];
1264 if (tb1 != NULL) {
1265 /* find head of list */
1266 for(;;) {
1267 n1 = (long)tb1 & 3;
1268 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1269 if (n1 == 2)
1270 break;
1271 tb1 = tb1->jmp_next[n1];
1272 }
1273 /* we are now sure now that tb jumps to tb1 */
1274 tb_next = tb1;
1275
1276 /* remove tb from the jmp_first list */
1277 ptb = &tb_next->jmp_first;
1278 for(;;) {
1279 tb1 = *ptb;
1280 n1 = (long)tb1 & 3;
1281 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1282 if (n1 == n && tb1 == tb)
1283 break;
1284 ptb = &tb1->jmp_next[n1];
1285 }
1286 *ptb = tb->jmp_next[n];
1287 tb->jmp_next[n] = NULL;
1288
1289 /* suppress the jump to next tb in generated code */
1290 tb_reset_jump(tb, n);
1291
1292 /* suppress jumps in the tb on which we could have jumped */
1293 tb_reset_jump_recursive(tb_next);
1294 }
1295 }
1296
1297 static void tb_reset_jump_recursive(TranslationBlock *tb)
1298 {
1299 tb_reset_jump_recursive2(tb, 0);
1300 tb_reset_jump_recursive2(tb, 1);
1301 }
1302
1303 #if defined(TARGET_HAS_ICE)
1304 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1305 {
1306 target_phys_addr_t addr;
1307 target_ulong pd;
1308 ram_addr_t ram_addr;
1309 PhysPageDesc *p;
1310
1311 addr = cpu_get_phys_page_debug(env, pc);
1312 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1313 if (!p) {
1314 pd = IO_MEM_UNASSIGNED;
1315 } else {
1316 pd = p->phys_offset;
1317 }
1318 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1319 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1320 }
1321 #endif
1322
1323 /* Add a watchpoint. */
1324 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1325 int flags, CPUWatchpoint **watchpoint)
1326 {
1327 target_ulong len_mask = ~(len - 1);
1328 CPUWatchpoint *wp;
1329
1330 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1331 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1332 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1333 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1334 return -EINVAL;
1335 }
1336 wp = qemu_malloc(sizeof(*wp));
1337 if (!wp)
1338 return -ENOMEM;
1339
1340 wp->vaddr = addr;
1341 wp->len_mask = len_mask;
1342 wp->flags = flags;
1343
1344 /* keep all GDB-injected watchpoints in front */
1345 if (flags & BP_GDB)
1346 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1347 else
1348 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1349
1350 tlb_flush_page(env, addr);
1351
1352 if (watchpoint)
1353 *watchpoint = wp;
1354 return 0;
1355 }
1356
1357 /* Remove a specific watchpoint. */
1358 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1359 int flags)
1360 {
1361 target_ulong len_mask = ~(len - 1);
1362 CPUWatchpoint *wp;
1363
1364 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1365 if (addr == wp->vaddr && len_mask == wp->len_mask
1366 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1367 cpu_watchpoint_remove_by_ref(env, wp);
1368 return 0;
1369 }
1370 }
1371 return -ENOENT;
1372 }
1373
1374 /* Remove a specific watchpoint by reference. */
1375 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1376 {
1377 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1378
1379 tlb_flush_page(env, watchpoint->vaddr);
1380
1381 qemu_free(watchpoint);
1382 }
1383
1384 /* Remove all matching watchpoints. */
1385 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1386 {
1387 CPUWatchpoint *wp, *next;
1388
1389 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1390 if (wp->flags & mask)
1391 cpu_watchpoint_remove_by_ref(env, wp);
1392 }
1393 }
1394
1395 /* Add a breakpoint. */
1396 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1397 CPUBreakpoint **breakpoint)
1398 {
1399 #if defined(TARGET_HAS_ICE)
1400 CPUBreakpoint *bp;
1401
1402 bp = qemu_malloc(sizeof(*bp));
1403 if (!bp)
1404 return -ENOMEM;
1405
1406 bp->pc = pc;
1407 bp->flags = flags;
1408
1409 /* keep all GDB-injected breakpoints in front */
1410 if (flags & BP_GDB)
1411 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1412 else
1413 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1414
1415 breakpoint_invalidate(env, pc);
1416
1417 if (breakpoint)
1418 *breakpoint = bp;
1419 return 0;
1420 #else
1421 return -ENOSYS;
1422 #endif
1423 }
1424
1425 /* Remove a specific breakpoint. */
1426 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1427 {
1428 #if defined(TARGET_HAS_ICE)
1429 CPUBreakpoint *bp;
1430
1431 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1432 if (bp->pc == pc && bp->flags == flags) {
1433 cpu_breakpoint_remove_by_ref(env, bp);
1434 return 0;
1435 }
1436 }
1437 return -ENOENT;
1438 #else
1439 return -ENOSYS;
1440 #endif
1441 }
1442
1443 /* Remove a specific breakpoint by reference. */
1444 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1445 {
1446 #if defined(TARGET_HAS_ICE)
1447 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1448
1449 breakpoint_invalidate(env, breakpoint->pc);
1450
1451 qemu_free(breakpoint);
1452 #endif
1453 }
1454
1455 /* Remove all matching breakpoints. */
1456 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1457 {
1458 #if defined(TARGET_HAS_ICE)
1459 CPUBreakpoint *bp, *next;
1460
1461 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1462 if (bp->flags & mask)
1463 cpu_breakpoint_remove_by_ref(env, bp);
1464 }
1465 #endif
1466 }
1467
1468 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1469 CPU loop after each instruction */
1470 void cpu_single_step(CPUState *env, int enabled)
1471 {
1472 #if defined(TARGET_HAS_ICE)
1473 if (env->singlestep_enabled != enabled) {
1474 env->singlestep_enabled = enabled;
1475 if (kvm_enabled())
1476 kvm_update_guest_debug(env, 0);
1477 else {
1478 /* must flush all the translated code to avoid inconsistancies */
1479 /* XXX: only flush what is necessary */
1480 tb_flush(env);
1481 }
1482 }
1483 #endif
1484 }
1485
1486 /* enable or disable low levels log */
1487 void cpu_set_log(int log_flags)
1488 {
1489 loglevel = log_flags;
1490 if (loglevel && !logfile) {
1491 logfile = fopen(logfilename, log_append ? "a" : "w");
1492 if (!logfile) {
1493 perror(logfilename);
1494 _exit(1);
1495 }
1496 #if !defined(CONFIG_SOFTMMU)
1497 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1498 {
1499 static char logfile_buf[4096];
1500 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1501 }
1502 #else
1503 setvbuf(logfile, NULL, _IOLBF, 0);
1504 #endif
1505 log_append = 1;
1506 }
1507 if (!loglevel && logfile) {
1508 fclose(logfile);
1509 logfile = NULL;
1510 }
1511 }
1512
1513 void cpu_set_log_filename(const char *filename)
1514 {
1515 logfilename = strdup(filename);
1516 if (logfile) {
1517 fclose(logfile);
1518 logfile = NULL;
1519 }
1520 cpu_set_log(loglevel);
1521 }
1522
1523 /* mask must never be zero, except for A20 change call */
1524 void cpu_interrupt(CPUState *env, int mask)
1525 {
1526 #if !defined(USE_NPTL)
1527 TranslationBlock *tb;
1528 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1529 #endif
1530 int old_mask;
1531
1532 old_mask = env->interrupt_request;
1533 /* FIXME: This is probably not threadsafe. A different thread could
1534 be in the middle of a read-modify-write operation. */
1535 env->interrupt_request |= mask;
1536 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1537 kvm_update_interrupt_request(env);
1538 #if defined(USE_NPTL)
1539 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1540 problem and hope the cpu will stop of its own accord. For userspace
1541 emulation this often isn't actually as bad as it sounds. Often
1542 signals are used primarily to interrupt blocking syscalls. */
1543 #else
1544 if (use_icount) {
1545 env->icount_decr.u16.high = 0xffff;
1546 #ifndef CONFIG_USER_ONLY
1547 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1548 an async event happened and we need to process it. */
1549 if (!can_do_io(env)
1550 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1551 cpu_abort(env, "Raised interrupt while not in I/O function");
1552 }
1553 #endif
1554 } else {
1555 tb = env->current_tb;
1556 /* if the cpu is currently executing code, we must unlink it and
1557 all the potentially executing TB */
1558 if (tb && !testandset(&interrupt_lock)) {
1559 env->current_tb = NULL;
1560 tb_reset_jump_recursive(tb);
1561 resetlock(&interrupt_lock);
1562 }
1563 }
1564 #endif
1565 }
1566
1567 void cpu_reset_interrupt(CPUState *env, int mask)
1568 {
1569 env->interrupt_request &= ~mask;
1570 }
1571
1572 const CPULogItem cpu_log_items[] = {
1573 { CPU_LOG_TB_OUT_ASM, "out_asm",
1574 "show generated host assembly code for each compiled TB" },
1575 { CPU_LOG_TB_IN_ASM, "in_asm",
1576 "show target assembly code for each compiled TB" },
1577 { CPU_LOG_TB_OP, "op",
1578 "show micro ops for each compiled TB" },
1579 { CPU_LOG_TB_OP_OPT, "op_opt",
1580 "show micro ops "
1581 #ifdef TARGET_I386
1582 "before eflags optimization and "
1583 #endif
1584 "after liveness analysis" },
1585 { CPU_LOG_INT, "int",
1586 "show interrupts/exceptions in short format" },
1587 { CPU_LOG_EXEC, "exec",
1588 "show trace before each executed TB (lots of logs)" },
1589 { CPU_LOG_TB_CPU, "cpu",
1590 "show CPU state before block translation" },
1591 #ifdef TARGET_I386
1592 { CPU_LOG_PCALL, "pcall",
1593 "show protected mode far calls/returns/exceptions" },
1594 #endif
1595 #ifdef DEBUG_IOPORT
1596 { CPU_LOG_IOPORT, "ioport",
1597 "show all i/o ports accesses" },
1598 #endif
1599 { 0, NULL, NULL },
1600 };
1601
1602 static int cmp1(const char *s1, int n, const char *s2)
1603 {
1604 if (strlen(s2) != n)
1605 return 0;
1606 return memcmp(s1, s2, n) == 0;
1607 }
1608
1609 /* takes a comma separated list of log masks. Return 0 if error. */
1610 int cpu_str_to_log_mask(const char *str)
1611 {
1612 const CPULogItem *item;
1613 int mask;
1614 const char *p, *p1;
1615
1616 p = str;
1617 mask = 0;
1618 for(;;) {
1619 p1 = strchr(p, ',');
1620 if (!p1)
1621 p1 = p + strlen(p);
1622 if(cmp1(p,p1-p,"all")) {
1623 for(item = cpu_log_items; item->mask != 0; item++) {
1624 mask |= item->mask;
1625 }
1626 } else {
1627 for(item = cpu_log_items; item->mask != 0; item++) {
1628 if (cmp1(p, p1 - p, item->name))
1629 goto found;
1630 }
1631 return 0;
1632 }
1633 found:
1634 mask |= item->mask;
1635 if (*p1 != ',')
1636 break;
1637 p = p1 + 1;
1638 }
1639 return mask;
1640 }
1641
1642 void cpu_abort(CPUState *env, const char *fmt, ...)
1643 {
1644 va_list ap;
1645 va_list ap2;
1646
1647 va_start(ap, fmt);
1648 va_copy(ap2, ap);
1649 fprintf(stderr, "qemu: fatal: ");
1650 vfprintf(stderr, fmt, ap);
1651 fprintf(stderr, "\n");
1652 #ifdef TARGET_I386
1653 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1654 #else
1655 cpu_dump_state(env, stderr, fprintf, 0);
1656 #endif
1657 if (qemu_log_enabled()) {
1658 qemu_log("qemu: fatal: ");
1659 qemu_log_vprintf(fmt, ap2);
1660 qemu_log("\n");
1661 #ifdef TARGET_I386
1662 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1663 #else
1664 log_cpu_state(env, 0);
1665 #endif
1666 qemu_log_flush();
1667 qemu_log_close();
1668 }
1669 va_end(ap2);
1670 va_end(ap);
1671 abort();
1672 }
1673
1674 CPUState *cpu_copy(CPUState *env)
1675 {
1676 CPUState *new_env = cpu_init(env->cpu_model_str);
1677 CPUState *next_cpu = new_env->next_cpu;
1678 int cpu_index = new_env->cpu_index;
1679 #if defined(TARGET_HAS_ICE)
1680 CPUBreakpoint *bp;
1681 CPUWatchpoint *wp;
1682 #endif
1683
1684 memcpy(new_env, env, sizeof(CPUState));
1685
1686 /* Preserve chaining and index. */
1687 new_env->next_cpu = next_cpu;
1688 new_env->cpu_index = cpu_index;
1689
1690 /* Clone all break/watchpoints.
1691 Note: Once we support ptrace with hw-debug register access, make sure
1692 BP_CPU break/watchpoints are handled correctly on clone. */
1693 TAILQ_INIT(&env->breakpoints);
1694 TAILQ_INIT(&env->watchpoints);
1695 #if defined(TARGET_HAS_ICE)
1696 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1697 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1698 }
1699 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1700 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1701 wp->flags, NULL);
1702 }
1703 #endif
1704
1705 return new_env;
1706 }
1707
1708 #if !defined(CONFIG_USER_ONLY)
1709
1710 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1711 {
1712 unsigned int i;
1713
1714 /* Discard jump cache entries for any tb which might potentially
1715 overlap the flushed page. */
1716 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1717 memset (&env->tb_jmp_cache[i], 0,
1718 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1719
1720 i = tb_jmp_cache_hash_page(addr);
1721 memset (&env->tb_jmp_cache[i], 0,
1722 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1723 }
1724
1725 /* NOTE: if flush_global is true, also flush global entries (not
1726 implemented yet) */
1727 void tlb_flush(CPUState *env, int flush_global)
1728 {
1729 int i;
1730
1731 #if defined(DEBUG_TLB)
1732 printf("tlb_flush:\n");
1733 #endif
1734 /* must reset current TB so that interrupts cannot modify the
1735 links while we are modifying them */
1736 env->current_tb = NULL;
1737
1738 for(i = 0; i < CPU_TLB_SIZE; i++) {
1739 env->tlb_table[0][i].addr_read = -1;
1740 env->tlb_table[0][i].addr_write = -1;
1741 env->tlb_table[0][i].addr_code = -1;
1742 env->tlb_table[1][i].addr_read = -1;
1743 env->tlb_table[1][i].addr_write = -1;
1744 env->tlb_table[1][i].addr_code = -1;
1745 #if (NB_MMU_MODES >= 3)
1746 env->tlb_table[2][i].addr_read = -1;
1747 env->tlb_table[2][i].addr_write = -1;
1748 env->tlb_table[2][i].addr_code = -1;
1749 #if (NB_MMU_MODES == 4)
1750 env->tlb_table[3][i].addr_read = -1;
1751 env->tlb_table[3][i].addr_write = -1;
1752 env->tlb_table[3][i].addr_code = -1;
1753 #endif
1754 #endif
1755 }
1756
1757 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1758
1759 #ifdef USE_KQEMU
1760 if (env->kqemu_enabled) {
1761 kqemu_flush(env, flush_global);
1762 }
1763 #endif
1764 tlb_flush_count++;
1765 }
1766
1767 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1768 {
1769 if (addr == (tlb_entry->addr_read &
1770 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1771 addr == (tlb_entry->addr_write &
1772 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1773 addr == (tlb_entry->addr_code &
1774 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1775 tlb_entry->addr_read = -1;
1776 tlb_entry->addr_write = -1;
1777 tlb_entry->addr_code = -1;
1778 }
1779 }
1780
1781 void tlb_flush_page(CPUState *env, target_ulong addr)
1782 {
1783 int i;
1784
1785 #if defined(DEBUG_TLB)
1786 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1787 #endif
1788 /* must reset current TB so that interrupts cannot modify the
1789 links while we are modifying them */
1790 env->current_tb = NULL;
1791
1792 addr &= TARGET_PAGE_MASK;
1793 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1794 tlb_flush_entry(&env->tlb_table[0][i], addr);
1795 tlb_flush_entry(&env->tlb_table[1][i], addr);
1796 #if (NB_MMU_MODES >= 3)
1797 tlb_flush_entry(&env->tlb_table[2][i], addr);
1798 #if (NB_MMU_MODES == 4)
1799 tlb_flush_entry(&env->tlb_table[3][i], addr);
1800 #endif
1801 #endif
1802
1803 tlb_flush_jmp_cache(env, addr);
1804
1805 #ifdef USE_KQEMU
1806 if (env->kqemu_enabled) {
1807 kqemu_flush_page(env, addr);
1808 }
1809 #endif
1810 }
1811
1812 /* update the TLBs so that writes to code in the virtual page 'addr'
1813 can be detected */
1814 static void tlb_protect_code(ram_addr_t ram_addr)
1815 {
1816 cpu_physical_memory_reset_dirty(ram_addr,
1817 ram_addr + TARGET_PAGE_SIZE,
1818 CODE_DIRTY_FLAG);
1819 }
1820
1821 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1822 tested for self modifying code */
1823 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1824 target_ulong vaddr)
1825 {
1826 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1827 }
1828
1829 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1830 unsigned long start, unsigned long length)
1831 {
1832 unsigned long addr;
1833 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1834 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1835 if ((addr - start) < length) {
1836 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1837 }
1838 }
1839 }
1840
1841 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1842 int dirty_flags)
1843 {
1844 CPUState *env;
1845 unsigned long length, start1;
1846 int i, mask, len;
1847 uint8_t *p;
1848
1849 start &= TARGET_PAGE_MASK;
1850 end = TARGET_PAGE_ALIGN(end);
1851
1852 length = end - start;
1853 if (length == 0)
1854 return;
1855 len = length >> TARGET_PAGE_BITS;
1856 #ifdef USE_KQEMU
1857 /* XXX: should not depend on cpu context */
1858 env = first_cpu;
1859 if (env->kqemu_enabled) {
1860 ram_addr_t addr;
1861 addr = start;
1862 for(i = 0; i < len; i++) {
1863 kqemu_set_notdirty(env, addr);
1864 addr += TARGET_PAGE_SIZE;
1865 }
1866 }
1867 #endif
1868 mask = ~dirty_flags;
1869 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1870 for(i = 0; i < len; i++)
1871 p[i] &= mask;
1872
1873 /* we modify the TLB cache so that the dirty bit will be set again
1874 when accessing the range */
1875 start1 = start + (unsigned long)phys_ram_base;
1876 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1877 for(i = 0; i < CPU_TLB_SIZE; i++)
1878 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
1879 for(i = 0; i < CPU_TLB_SIZE; i++)
1880 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
1881 #if (NB_MMU_MODES >= 3)
1882 for(i = 0; i < CPU_TLB_SIZE; i++)
1883 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1884 #if (NB_MMU_MODES == 4)
1885 for(i = 0; i < CPU_TLB_SIZE; i++)
1886 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1887 #endif
1888 #endif
1889 }
1890 }
1891
1892 int cpu_physical_memory_set_dirty_tracking(int enable)
1893 {
1894 int r=0;
1895
1896 if (kvm_enabled())
1897 r = kvm_physical_memory_set_dirty_tracking(enable);
1898 in_migration = enable;
1899 return r;
1900 }
1901
1902 int cpu_physical_memory_get_dirty_tracking(void)
1903 {
1904 return in_migration;
1905 }
1906
1907 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1908 {
1909 if (kvm_enabled())
1910 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1911 }
1912
1913 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1914 {
1915 ram_addr_t ram_addr;
1916
1917 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1918 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
1919 tlb_entry->addend - (unsigned long)phys_ram_base;
1920 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1921 tlb_entry->addr_write |= TLB_NOTDIRTY;
1922 }
1923 }
1924 }
1925
1926 /* update the TLB according to the current state of the dirty bits */
1927 void cpu_tlb_update_dirty(CPUState *env)
1928 {
1929 int i;
1930 for(i = 0; i < CPU_TLB_SIZE; i++)
1931 tlb_update_dirty(&env->tlb_table[0][i]);
1932 for(i = 0; i < CPU_TLB_SIZE; i++)
1933 tlb_update_dirty(&env->tlb_table[1][i]);
1934 #if (NB_MMU_MODES >= 3)
1935 for(i = 0; i < CPU_TLB_SIZE; i++)
1936 tlb_update_dirty(&env->tlb_table[2][i]);
1937 #if (NB_MMU_MODES == 4)
1938 for(i = 0; i < CPU_TLB_SIZE; i++)
1939 tlb_update_dirty(&env->tlb_table[3][i]);
1940 #endif
1941 #endif
1942 }
1943
1944 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1945 {
1946 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1947 tlb_entry->addr_write = vaddr;
1948 }
1949
1950 /* update the TLB corresponding to virtual page vaddr
1951 so that it is no longer dirty */
1952 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1953 {
1954 int i;
1955
1956 vaddr &= TARGET_PAGE_MASK;
1957 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1958 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1959 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
1960 #if (NB_MMU_MODES >= 3)
1961 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
1962 #if (NB_MMU_MODES == 4)
1963 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
1964 #endif
1965 #endif
1966 }
1967
1968 /* add a new TLB entry. At most one entry for a given virtual address
1969 is permitted. Return 0 if OK or 2 if the page could not be mapped
1970 (can only happen in non SOFTMMU mode for I/O pages or pages
1971 conflicting with the host address space). */
1972 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1973 target_phys_addr_t paddr, int prot,
1974 int mmu_idx, int is_softmmu)
1975 {
1976 PhysPageDesc *p;
1977 unsigned long pd;
1978 unsigned int index;
1979 target_ulong address;
1980 target_ulong code_address;
1981 target_phys_addr_t addend;
1982 int ret;
1983 CPUTLBEntry *te;
1984 CPUWatchpoint *wp;
1985 target_phys_addr_t iotlb;
1986
1987 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
1988 if (!p) {
1989 pd = IO_MEM_UNASSIGNED;
1990 } else {
1991 pd = p->phys_offset;
1992 }
1993 #if defined(DEBUG_TLB)
1994 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1995 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
1996 #endif
1997
1998 ret = 0;
1999 address = vaddr;
2000 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2001 /* IO memory case (romd handled later) */
2002 address |= TLB_MMIO;
2003 }
2004 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
2005 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2006 /* Normal RAM. */
2007 iotlb = pd & TARGET_PAGE_MASK;
2008 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2009 iotlb |= IO_MEM_NOTDIRTY;
2010 else
2011 iotlb |= IO_MEM_ROM;
2012 } else {
2013 /* IO handlers are currently passed a phsical address.
2014 It would be nice to pass an offset from the base address
2015 of that region. This would avoid having to special case RAM,
2016 and avoid full address decoding in every device.
2017 We can't use the high bits of pd for this because
2018 IO_MEM_ROMD uses these as a ram address. */
2019 iotlb = (pd & ~TARGET_PAGE_MASK);
2020 if (p) {
2021 iotlb += p->region_offset;
2022 } else {
2023 iotlb += paddr;
2024 }
2025 }
2026
2027 code_address = address;
2028 /* Make accesses to pages with watchpoints go via the
2029 watchpoint trap routines. */
2030 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2031 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2032 iotlb = io_mem_watch + paddr;
2033 /* TODO: The memory case can be optimized by not trapping
2034 reads of pages with a write breakpoint. */
2035 address |= TLB_MMIO;
2036 }
2037 }
2038
2039 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2040 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2041 te = &env->tlb_table[mmu_idx][index];
2042 te->addend = addend - vaddr;
2043 if (prot & PAGE_READ) {
2044 te->addr_read = address;
2045 } else {
2046 te->addr_read = -1;
2047 }
2048
2049 if (prot & PAGE_EXEC) {
2050 te->addr_code = code_address;
2051 } else {
2052 te->addr_code = -1;
2053 }
2054 if (prot & PAGE_WRITE) {
2055 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2056 (pd & IO_MEM_ROMD)) {
2057 /* Write access calls the I/O callback. */
2058 te->addr_write = address | TLB_MMIO;
2059 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2060 !cpu_physical_memory_is_dirty(pd)) {
2061 te->addr_write = address | TLB_NOTDIRTY;
2062 } else {
2063 te->addr_write = address;
2064 }
2065 } else {
2066 te->addr_write = -1;
2067 }
2068 return ret;
2069 }
2070
2071 #else
2072
2073 void tlb_flush(CPUState *env, int flush_global)
2074 {
2075 }
2076
2077 void tlb_flush_page(CPUState *env, target_ulong addr)
2078 {
2079 }
2080
2081 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2082 target_phys_addr_t paddr, int prot,
2083 int mmu_idx, int is_softmmu)
2084 {
2085 return 0;
2086 }
2087
2088 /* dump memory mappings */
2089 void page_dump(FILE *f)
2090 {
2091 unsigned long start, end;
2092 int i, j, prot, prot1;
2093 PageDesc *p;
2094
2095 fprintf(f, "%-8s %-8s %-8s %s\n",
2096 "start", "end", "size", "prot");
2097 start = -1;
2098 end = -1;
2099 prot = 0;
2100 for(i = 0; i <= L1_SIZE; i++) {
2101 if (i < L1_SIZE)
2102 p = l1_map[i];
2103 else
2104 p = NULL;
2105 for(j = 0;j < L2_SIZE; j++) {
2106 if (!p)
2107 prot1 = 0;
2108 else
2109 prot1 = p[j].flags;
2110 if (prot1 != prot) {
2111 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2112 if (start != -1) {
2113 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2114 start, end, end - start,
2115 prot & PAGE_READ ? 'r' : '-',
2116 prot & PAGE_WRITE ? 'w' : '-',
2117 prot & PAGE_EXEC ? 'x' : '-');
2118 }
2119 if (prot1 != 0)
2120 start = end;
2121 else
2122 start = -1;
2123 prot = prot1;
2124 }
2125 if (!p)
2126 break;
2127 }
2128 }
2129 }
2130
2131 int page_get_flags(target_ulong address)
2132 {
2133 PageDesc *p;
2134
2135 p = page_find(address >> TARGET_PAGE_BITS);
2136 if (!p)
2137 return 0;
2138 return p->flags;
2139 }
2140
2141 /* modify the flags of a page and invalidate the code if
2142 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2143 depending on PAGE_WRITE */
2144 void page_set_flags(target_ulong start, target_ulong end, int flags)
2145 {
2146 PageDesc *p;
2147 target_ulong addr;
2148
2149 /* mmap_lock should already be held. */
2150 start = start & TARGET_PAGE_MASK;
2151 end = TARGET_PAGE_ALIGN(end);
2152 if (flags & PAGE_WRITE)
2153 flags |= PAGE_WRITE_ORG;
2154 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2155 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2156 /* We may be called for host regions that are outside guest
2157 address space. */
2158 if (!p)
2159 return;
2160 /* if the write protection is set, then we invalidate the code
2161 inside */
2162 if (!(p->flags & PAGE_WRITE) &&
2163 (flags & PAGE_WRITE) &&
2164 p->first_tb) {
2165 tb_invalidate_phys_page(addr, 0, NULL);
2166 }
2167 p->flags = flags;
2168 }
2169 }
2170
2171 int page_check_range(target_ulong start, target_ulong len, int flags)
2172 {
2173 PageDesc *p;
2174 target_ulong end;
2175 target_ulong addr;
2176
2177 if (start + len < start)
2178 /* we've wrapped around */
2179 return -1;
2180
2181 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2182 start = start & TARGET_PAGE_MASK;
2183
2184 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2185 p = page_find(addr >> TARGET_PAGE_BITS);
2186 if( !p )
2187 return -1;
2188 if( !(p->flags & PAGE_VALID) )
2189 return -1;
2190
2191 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2192 return -1;
2193 if (flags & PAGE_WRITE) {
2194 if (!(p->flags & PAGE_WRITE_ORG))
2195 return -1;
2196 /* unprotect the page if it was put read-only because it
2197 contains translated code */
2198 if (!(p->flags & PAGE_WRITE)) {
2199 if (!page_unprotect(addr, 0, NULL))
2200 return -1;
2201 }
2202 return 0;
2203 }
2204 }
2205 return 0;
2206 }
2207
2208 /* called from signal handler: invalidate the code and unprotect the
2209 page. Return TRUE if the fault was succesfully handled. */
2210 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2211 {
2212 unsigned int page_index, prot, pindex;
2213 PageDesc *p, *p1;
2214 target_ulong host_start, host_end, addr;
2215
2216 /* Technically this isn't safe inside a signal handler. However we
2217 know this only ever happens in a synchronous SEGV handler, so in
2218 practice it seems to be ok. */
2219 mmap_lock();
2220
2221 host_start = address & qemu_host_page_mask;
2222 page_index = host_start >> TARGET_PAGE_BITS;
2223 p1 = page_find(page_index);
2224 if (!p1) {
2225 mmap_unlock();
2226 return 0;
2227 }
2228 host_end = host_start + qemu_host_page_size;
2229 p = p1;
2230 prot = 0;
2231 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2232 prot |= p->flags;
2233 p++;
2234 }
2235 /* if the page was really writable, then we change its
2236 protection back to writable */
2237 if (prot & PAGE_WRITE_ORG) {
2238 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2239 if (!(p1[pindex].flags & PAGE_WRITE)) {
2240 mprotect((void *)g2h(host_start), qemu_host_page_size,
2241 (prot & PAGE_BITS) | PAGE_WRITE);
2242 p1[pindex].flags |= PAGE_WRITE;
2243 /* and since the content will be modified, we must invalidate
2244 the corresponding translated code. */
2245 tb_invalidate_phys_page(address, pc, puc);
2246 #ifdef DEBUG_TB_CHECK
2247 tb_invalidate_check(address);
2248 #endif
2249 mmap_unlock();
2250 return 1;
2251 }
2252 }
2253 mmap_unlock();
2254 return 0;
2255 }
2256
2257 static inline void tlb_set_dirty(CPUState *env,
2258 unsigned long addr, target_ulong vaddr)
2259 {
2260 }
2261 #endif /* defined(CONFIG_USER_ONLY) */
2262
2263 #if !defined(CONFIG_USER_ONLY)
2264
2265 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2266 ram_addr_t memory, ram_addr_t region_offset);
2267 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2268 ram_addr_t orig_memory, ram_addr_t region_offset);
2269 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2270 need_subpage) \
2271 do { \
2272 if (addr > start_addr) \
2273 start_addr2 = 0; \
2274 else { \
2275 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2276 if (start_addr2 > 0) \
2277 need_subpage = 1; \
2278 } \
2279 \
2280 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2281 end_addr2 = TARGET_PAGE_SIZE - 1; \
2282 else { \
2283 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2284 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2285 need_subpage = 1; \
2286 } \
2287 } while (0)
2288
2289 /* register physical memory. 'size' must be a multiple of the target
2290 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2291 io memory page. The address used when calling the IO function is
2292 the offset from the start of the region, plus region_offset. Both
2293 start_region and regon_offset are rounded down to a page boundary
2294 before calculating this offset. This should not be a problem unless
2295 the low bits of start_addr and region_offset differ. */
2296 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2297 ram_addr_t size,
2298 ram_addr_t phys_offset,
2299 ram_addr_t region_offset)
2300 {
2301 target_phys_addr_t addr, end_addr;
2302 PhysPageDesc *p;
2303 CPUState *env;
2304 ram_addr_t orig_size = size;
2305 void *subpage;
2306
2307 #ifdef USE_KQEMU
2308 /* XXX: should not depend on cpu context */
2309 env = first_cpu;
2310 if (env->kqemu_enabled) {
2311 kqemu_set_phys_mem(start_addr, size, phys_offset);
2312 }
2313 #endif
2314 if (kvm_enabled())
2315 kvm_set_phys_mem(start_addr, size, phys_offset);
2316
2317 region_offset &= TARGET_PAGE_MASK;
2318 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2319 end_addr = start_addr + (target_phys_addr_t)size;
2320 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2321 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2322 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2323 ram_addr_t orig_memory = p->phys_offset;
2324 target_phys_addr_t start_addr2, end_addr2;
2325 int need_subpage = 0;
2326
2327 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2328 need_subpage);
2329 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2330 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2331 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2332 &p->phys_offset, orig_memory,
2333 p->region_offset);
2334 } else {
2335 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2336 >> IO_MEM_SHIFT];
2337 }
2338 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2339 region_offset);
2340 p->region_offset = 0;
2341 } else {
2342 p->phys_offset = phys_offset;
2343 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2344 (phys_offset & IO_MEM_ROMD))
2345 phys_offset += TARGET_PAGE_SIZE;
2346 }
2347 } else {
2348 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2349 p->phys_offset = phys_offset;
2350 p->region_offset = region_offset;
2351 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2352 (phys_offset & IO_MEM_ROMD)) {
2353 phys_offset += TARGET_PAGE_SIZE;
2354 } else {
2355 target_phys_addr_t start_addr2, end_addr2;
2356 int need_subpage = 0;
2357
2358 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2359 end_addr2, need_subpage);
2360
2361 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2362 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2363 &p->phys_offset, IO_MEM_UNASSIGNED,
2364 0);
2365 subpage_register(subpage, start_addr2, end_addr2,
2366 phys_offset, region_offset);
2367 p->region_offset = 0;
2368 }
2369 }
2370 }
2371 region_offset += TARGET_PAGE_SIZE;
2372 }
2373
2374 /* since each CPU stores ram addresses in its TLB cache, we must
2375 reset the modified entries */
2376 /* XXX: slow ! */
2377 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2378 tlb_flush(env, 1);
2379 }
2380 }
2381
2382 /* XXX: temporary until new memory mapping API */
2383 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2384 {
2385 PhysPageDesc *p;
2386
2387 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2388 if (!p)
2389 return IO_MEM_UNASSIGNED;
2390 return p->phys_offset;
2391 }
2392
2393 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2394 {
2395 if (kvm_enabled())
2396 kvm_coalesce_mmio_region(addr, size);
2397 }
2398
2399 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2400 {
2401 if (kvm_enabled())
2402 kvm_uncoalesce_mmio_region(addr, size);
2403 }
2404
2405 /* XXX: better than nothing */
2406 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2407 {
2408 ram_addr_t addr;
2409 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
2410 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2411 (uint64_t)size, (uint64_t)phys_ram_size);
2412 abort();
2413 }
2414 addr = phys_ram_alloc_offset;
2415 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2416 return addr;
2417 }
2418
2419 void qemu_ram_free(ram_addr_t addr)
2420 {
2421 }
2422
2423 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2424 {
2425 #ifdef DEBUG_UNASSIGNED
2426 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2427 #endif
2428 #if defined(TARGET_SPARC)
2429 do_unassigned_access(addr, 0, 0, 0, 1);
2430 #endif
2431 return 0;
2432 }
2433
2434 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2435 {
2436 #ifdef DEBUG_UNASSIGNED
2437 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2438 #endif
2439 #if defined(TARGET_SPARC)
2440 do_unassigned_access(addr, 0, 0, 0, 2);
2441 #endif
2442 return 0;
2443 }
2444
2445 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2446 {
2447 #ifdef DEBUG_UNASSIGNED
2448 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2449 #endif
2450 #if defined(TARGET_SPARC)
2451 do_unassigned_access(addr, 0, 0, 0, 4);
2452 #endif
2453 return 0;
2454 }
2455
2456 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2457 {
2458 #ifdef DEBUG_UNASSIGNED
2459 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2460 #endif
2461 #if defined(TARGET_SPARC)
2462 do_unassigned_access(addr, 1, 0, 0, 1);
2463 #endif
2464 }
2465
2466 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2467 {
2468 #ifdef DEBUG_UNASSIGNED
2469 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2470 #endif
2471 #if defined(TARGET_SPARC)
2472 do_unassigned_access(addr, 1, 0, 0, 2);
2473 #endif
2474 }
2475
2476 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2477 {
2478 #ifdef DEBUG_UNASSIGNED
2479 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2480 #endif
2481 #if defined(TARGET_SPARC)
2482 do_unassigned_access(addr, 1, 0, 0, 4);
2483 #endif
2484 }
2485
2486 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2487 unassigned_mem_readb,
2488 unassigned_mem_readw,
2489 unassigned_mem_readl,
2490 };
2491
2492 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2493 unassigned_mem_writeb,
2494 unassigned_mem_writew,
2495 unassigned_mem_writel,
2496 };
2497
2498 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2499 uint32_t val)
2500 {
2501 int dirty_flags;
2502 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2503 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2504 #if !defined(CONFIG_USER_ONLY)
2505 tb_invalidate_phys_page_fast(ram_addr, 1);
2506 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2507 #endif
2508 }
2509 stb_p(phys_ram_base + ram_addr, val);
2510 #ifdef USE_KQEMU
2511 if (cpu_single_env->kqemu_enabled &&
2512 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2513 kqemu_modify_page(cpu_single_env, ram_addr);
2514 #endif
2515 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2516 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2517 /* we remove the notdirty callback only if the code has been
2518 flushed */
2519 if (dirty_flags == 0xff)
2520 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2521 }
2522
2523 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2524 uint32_t val)
2525 {
2526 int dirty_flags;
2527 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2528 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2529 #if !defined(CONFIG_USER_ONLY)
2530 tb_invalidate_phys_page_fast(ram_addr, 2);
2531 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2532 #endif
2533 }
2534 stw_p(phys_ram_base + ram_addr, val);
2535 #ifdef USE_KQEMU
2536 if (cpu_single_env->kqemu_enabled &&
2537 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2538 kqemu_modify_page(cpu_single_env, ram_addr);
2539 #endif
2540 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2541 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2542 /* we remove the notdirty callback only if the code has been
2543 flushed */
2544 if (dirty_flags == 0xff)
2545 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2546 }
2547
2548 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2549 uint32_t val)
2550 {
2551 int dirty_flags;
2552 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2553 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2554 #if !defined(CONFIG_USER_ONLY)
2555 tb_invalidate_phys_page_fast(ram_addr, 4);
2556 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2557 #endif
2558 }
2559 stl_p(phys_ram_base + ram_addr, val);
2560 #ifdef USE_KQEMU
2561 if (cpu_single_env->kqemu_enabled &&
2562 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2563 kqemu_modify_page(cpu_single_env, ram_addr);
2564 #endif
2565 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2566 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2567 /* we remove the notdirty callback only if the code has been
2568 flushed */
2569 if (dirty_flags == 0xff)
2570 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2571 }
2572
2573 static CPUReadMemoryFunc *error_mem_read[3] = {
2574 NULL, /* never used */
2575 NULL, /* never used */
2576 NULL, /* never used */
2577 };
2578
2579 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2580 notdirty_mem_writeb,
2581 notdirty_mem_writew,
2582 notdirty_mem_writel,
2583 };
2584
2585 /* Generate a debug exception if a watchpoint has been hit. */
2586 static void check_watchpoint(int offset, int len_mask, int flags)
2587 {
2588 CPUState *env = cpu_single_env;
2589 target_ulong pc, cs_base;
2590 TranslationBlock *tb;
2591 target_ulong vaddr;
2592 CPUWatchpoint *wp;
2593 int cpu_flags;
2594
2595 if (env->watchpoint_hit) {
2596 /* We re-entered the check after replacing the TB. Now raise
2597 * the debug interrupt so that is will trigger after the
2598 * current instruction. */
2599 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2600 return;
2601 }
2602 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2603 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2604 if ((vaddr == (wp->vaddr & len_mask) ||
2605 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2606 wp->flags |= BP_WATCHPOINT_HIT;
2607 if (!env->watchpoint_hit) {
2608 env->watchpoint_hit = wp;
2609 tb = tb_find_pc(env->mem_io_pc);
2610 if (!tb) {
2611 cpu_abort(env, "check_watchpoint: could not find TB for "
2612 "pc=%p", (void *)env->mem_io_pc);
2613 }
2614 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2615 tb_phys_invalidate(tb, -1);
2616 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2617 env->exception_index = EXCP_DEBUG;
2618 } else {
2619 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2620 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2621 }
2622 cpu_resume_from_signal(env, NULL);
2623 }
2624 } else {
2625 wp->flags &= ~BP_WATCHPOINT_HIT;
2626 }
2627 }
2628 }
2629
2630 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2631 so these check for a hit then pass through to the normal out-of-line
2632 phys routines. */
2633 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2634 {
2635 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2636 return ldub_phys(addr);
2637 }
2638
2639 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2640 {
2641 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2642 return lduw_phys(addr);
2643 }
2644
2645 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2646 {
2647 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2648 return ldl_phys(addr);
2649 }
2650
2651 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2652 uint32_t val)
2653 {
2654 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2655 stb_phys(addr, val);
2656 }
2657
2658 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2659 uint32_t val)
2660 {
2661 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2662 stw_phys(addr, val);
2663 }
2664
2665 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2666 uint32_t val)
2667 {
2668 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2669 stl_phys(addr, val);
2670 }
2671
2672 static CPUReadMemoryFunc *watch_mem_read[3] = {
2673 watch_mem_readb,
2674 watch_mem_readw,
2675 watch_mem_readl,
2676 };
2677
2678 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2679 watch_mem_writeb,
2680 watch_mem_writew,
2681 watch_mem_writel,
2682 };
2683
2684 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2685 unsigned int len)
2686 {
2687 uint32_t ret;
2688 unsigned int idx;
2689
2690 idx = SUBPAGE_IDX(addr);
2691 #if defined(DEBUG_SUBPAGE)
2692 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2693 mmio, len, addr, idx);
2694 #endif
2695 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2696 addr + mmio->region_offset[idx][0][len]);
2697
2698 return ret;
2699 }
2700
2701 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2702 uint32_t value, unsigned int len)
2703 {
2704 unsigned int idx;
2705
2706 idx = SUBPAGE_IDX(addr);
2707 #if defined(DEBUG_SUBPAGE)
2708 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2709 mmio, len, addr, idx, value);
2710 #endif
2711 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2712 addr + mmio->region_offset[idx][1][len],
2713 value);
2714 }
2715
2716 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2717 {
2718 #if defined(DEBUG_SUBPAGE)
2719 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2720 #endif
2721
2722 return subpage_readlen(opaque, addr, 0);
2723 }
2724
2725 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2726 uint32_t value)
2727 {
2728 #if defined(DEBUG_SUBPAGE)
2729 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2730 #endif
2731 subpage_writelen(opaque, addr, value, 0);
2732 }
2733
2734 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2735 {
2736 #if defined(DEBUG_SUBPAGE)
2737 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2738 #endif
2739
2740 return subpage_readlen(opaque, addr, 1);
2741 }
2742
2743 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2744 uint32_t value)
2745 {
2746 #if defined(DEBUG_SUBPAGE)
2747 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2748 #endif
2749 subpage_writelen(opaque, addr, value, 1);
2750 }
2751
2752 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2753 {
2754 #if defined(DEBUG_SUBPAGE)
2755 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2756 #endif
2757
2758 return subpage_readlen(opaque, addr, 2);
2759 }
2760
2761 static void subpage_writel (void *opaque,
2762 target_phys_addr_t addr, uint32_t value)
2763 {
2764 #if defined(DEBUG_SUBPAGE)
2765 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2766 #endif
2767 subpage_writelen(opaque, addr, value, 2);
2768 }
2769
2770 static CPUReadMemoryFunc *subpage_read[] = {
2771 &subpage_readb,
2772 &subpage_readw,
2773 &subpage_readl,
2774 };
2775
2776 static CPUWriteMemoryFunc *subpage_write[] = {
2777 &subpage_writeb,
2778 &subpage_writew,
2779 &subpage_writel,
2780 };
2781
2782 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2783 ram_addr_t memory, ram_addr_t region_offset)
2784 {
2785 int idx, eidx;
2786 unsigned int i;
2787
2788 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2789 return -1;
2790 idx = SUBPAGE_IDX(start);
2791 eidx = SUBPAGE_IDX(end);
2792 #if defined(DEBUG_SUBPAGE)
2793 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2794 mmio, start, end, idx, eidx, memory);
2795 #endif
2796 memory >>= IO_MEM_SHIFT;
2797 for (; idx <= eidx; idx++) {
2798 for (i = 0; i < 4; i++) {
2799 if (io_mem_read[memory][i]) {
2800 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2801 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2802 mmio->region_offset[idx][0][i] = region_offset;
2803 }
2804 if (io_mem_write[memory][i]) {
2805 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2806 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
2807 mmio->region_offset[idx][1][i] = region_offset;
2808 }
2809 }
2810 }
2811
2812 return 0;
2813 }
2814
2815 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2816 ram_addr_t orig_memory, ram_addr_t region_offset)
2817 {
2818 subpage_t *mmio;
2819 int subpage_memory;
2820
2821 mmio = qemu_mallocz(sizeof(subpage_t));
2822 if (mmio != NULL) {
2823 mmio->base = base;
2824 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2825 #if defined(DEBUG_SUBPAGE)
2826 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2827 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2828 #endif
2829 *phys = subpage_memory | IO_MEM_SUBPAGE;
2830 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2831 region_offset);
2832 }
2833
2834 return mmio;
2835 }
2836
2837 static int get_free_io_mem_idx(void)
2838 {
2839 int i;
2840
2841 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
2842 if (!io_mem_used[i]) {
2843 io_mem_used[i] = 1;
2844 return i;
2845 }
2846
2847 return -1;
2848 }
2849
2850 static void io_mem_init(void)
2851 {
2852 int i;
2853
2854 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
2855 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
2856 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
2857 for (i=0; i<5; i++)
2858 io_mem_used[i] = 1;
2859
2860 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
2861 watch_mem_write, NULL);
2862 /* alloc dirty bits array */
2863 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
2864 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
2865 }
2866
2867 /* mem_read and mem_write are arrays of functions containing the
2868 function to access byte (index 0), word (index 1) and dword (index
2869 2). Functions can be omitted with a NULL function pointer. The
2870 registered functions may be modified dynamically later.
2871 If io_index is non zero, the corresponding io zone is
2872 modified. If it is zero, a new io zone is allocated. The return
2873 value can be used with cpu_register_physical_memory(). (-1) is
2874 returned if error. */
2875 int cpu_register_io_memory(int io_index,
2876 CPUReadMemoryFunc **mem_read,
2877 CPUWriteMemoryFunc **mem_write,
2878 void *opaque)
2879 {
2880 int i, subwidth = 0;
2881
2882 if (io_index <= 0) {
2883 io_index = get_free_io_mem_idx();
2884 if (io_index == -1)
2885 return io_index;
2886 } else {
2887 if (io_index >= IO_MEM_NB_ENTRIES)
2888 return -1;
2889 }
2890
2891 for(i = 0;i < 3; i++) {
2892 if (!mem_read[i] || !mem_write[i])
2893 subwidth = IO_MEM_SUBWIDTH;
2894 io_mem_read[io_index][i] = mem_read[i];
2895 io_mem_write[io_index][i] = mem_write[i];
2896 }
2897 io_mem_opaque[io_index] = opaque;
2898 return (io_index << IO_MEM_SHIFT) | subwidth;
2899 }
2900
2901 void cpu_unregister_io_memory(int io_table_address)
2902 {
2903 int i;
2904 int io_index = io_table_address >> IO_MEM_SHIFT;
2905
2906 for (i=0;i < 3; i++) {
2907 io_mem_read[io_index][i] = unassigned_mem_read[i];
2908 io_mem_write[io_index][i] = unassigned_mem_write[i];
2909 }
2910 io_mem_opaque[io_index] = NULL;
2911 io_mem_used[io_index] = 0;
2912 }
2913
2914 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2915 {
2916 return io_mem_write[io_index >> IO_MEM_SHIFT];
2917 }
2918
2919 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2920 {
2921 return io_mem_read[io_index >> IO_MEM_SHIFT];
2922 }
2923
2924 #endif /* !defined(CONFIG_USER_ONLY) */
2925
2926 /* physical memory access (slow version, mainly for debug) */
2927 #if defined(CONFIG_USER_ONLY)
2928 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2929 int len, int is_write)
2930 {
2931 int l, flags;
2932 target_ulong page;
2933 void * p;
2934
2935 while (len > 0) {
2936 page = addr & TARGET_PAGE_MASK;
2937 l = (page + TARGET_PAGE_SIZE) - addr;
2938 if (l > len)
2939 l = len;
2940 flags = page_get_flags(page);
2941 if (!(flags & PAGE_VALID))
2942 return;
2943 if (is_write) {
2944 if (!(flags & PAGE_WRITE))
2945 return;
2946 /* XXX: this code should not depend on lock_user */
2947 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
2948 /* FIXME - should this return an error rather than just fail? */
2949 return;
2950 memcpy(p, buf, l);
2951 unlock_user(p, addr, l);
2952 } else {
2953 if (!(flags & PAGE_READ))
2954 return;
2955 /* XXX: this code should not depend on lock_user */
2956 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
2957 /* FIXME - should this return an error rather than just fail? */
2958 return;
2959 memcpy(buf, p, l);
2960 unlock_user(p, addr, 0);
2961 }
2962 len -= l;
2963 buf += l;
2964 addr += l;
2965 }
2966 }
2967
2968 #else
2969 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
2970 int len, int is_write)
2971 {
2972 int l, io_index;
2973 uint8_t *ptr;
2974 uint32_t val;
2975 target_phys_addr_t page;
2976 unsigned long pd;
2977 PhysPageDesc *p;
2978
2979 while (len > 0) {
2980 page = addr & TARGET_PAGE_MASK;
2981 l = (page + TARGET_PAGE_SIZE) - addr;
2982 if (l > len)
2983 l = len;
2984 p = phys_page_find(page >> TARGET_PAGE_BITS);
2985 if (!p) {
2986 pd = IO_MEM_UNASSIGNED;
2987 } else {
2988 pd = p->phys_offset;
2989 }
2990
2991 if (is_write) {
2992 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
2993 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
2994 if (p)
2995 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
2996 /* XXX: could force cpu_single_env to NULL to avoid
2997 potential bugs */
2998 if (l >= 4 && ((addr & 3) == 0)) {
2999 /* 32 bit write access */
3000 val = ldl_p(buf);
3001 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3002 l = 4;
3003 } else if (l >= 2 && ((addr & 1) == 0)) {
3004 /* 16 bit write access */
3005 val = lduw_p(buf);
3006 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
3007 l = 2;
3008 } else {
3009 /* 8 bit write access */
3010 val = ldub_p(buf);
3011 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
3012 l = 1;
3013 }
3014 } else {
3015 unsigned long addr1;
3016 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3017 /* RAM case */
3018 ptr = phys_ram_base + addr1;
3019 memcpy(ptr, buf, l);
3020 if (!cpu_physical_memory_is_dirty(addr1)) {
3021 /* invalidate code */
3022 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3023 /* set dirty bit */
3024 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3025 (0xff & ~CODE_DIRTY_FLAG);
3026 }
3027 /* qemu doesn't execute guest code directly, but kvm does
3028 therefore fluch instruction caches */
3029 if (kvm_enabled())
3030 flush_icache_range((unsigned long)ptr,
3031 ((unsigned long)ptr)+l);
3032 }
3033 } else {
3034 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3035 !(pd & IO_MEM_ROMD)) {
3036 /* I/O case */
3037 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3038 if (p)
3039 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3040 if (l >= 4 && ((addr & 3) == 0)) {
3041 /* 32 bit read access */
3042 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3043 stl_p(buf, val);
3044 l = 4;
3045 } else if (l >= 2 && ((addr & 1) == 0)) {
3046 /* 16 bit read access */
3047 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3048 stw_p(buf, val);
3049 l = 2;
3050 } else {
3051 /* 8 bit read access */
3052 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
3053 stb_p(buf, val);
3054 l = 1;
3055 }
3056 } else {
3057 /* RAM case */
3058 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3059 (addr & ~TARGET_PAGE_MASK);
3060 memcpy(buf, ptr, l);
3061 }
3062 }
3063 len -= l;
3064 buf += l;
3065 addr += l;
3066 }
3067 }
3068
3069 /* used for ROM loading : can write in RAM and ROM */
3070 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3071 const uint8_t *buf, int len)
3072 {
3073 int l;
3074 uint8_t *ptr;
3075 target_phys_addr_t page;
3076 unsigned long pd;
3077 PhysPageDesc *p;
3078
3079 while (len > 0) {
3080 page = addr & TARGET_PAGE_MASK;
3081 l = (page + TARGET_PAGE_SIZE) - addr;
3082 if (l > len)
3083 l = len;
3084 p = phys_page_find(page >> TARGET_PAGE_BITS);
3085 if (!p) {
3086 pd = IO_MEM_UNASSIGNED;
3087 } else {
3088 pd = p->phys_offset;
3089 }
3090
3091 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3092 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3093 !(pd & IO_MEM_ROMD)) {
3094 /* do nothing */
3095 } else {
3096 unsigned long addr1;
3097 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3098 /* ROM/RAM case */
3099 ptr = phys_ram_base + addr1;
3100 memcpy(ptr, buf, l);
3101 }
3102 len -= l;
3103 buf += l;
3104 addr += l;
3105 }
3106 }
3107
3108
3109 /* warning: addr must be aligned */
3110 uint32_t ldl_phys(target_phys_addr_t addr)
3111 {
3112 int io_index;
3113 uint8_t *ptr;
3114 uint32_t val;
3115 unsigned long pd;
3116 PhysPageDesc *p;
3117
3118 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3119 if (!p) {
3120 pd = IO_MEM_UNASSIGNED;
3121 } else {
3122 pd = p->phys_offset;
3123 }
3124
3125 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3126 !(pd & IO_MEM_ROMD)) {
3127 /* I/O case */
3128 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3129 if (p)
3130 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3131 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3132 } else {
3133 /* RAM case */
3134 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3135 (addr & ~TARGET_PAGE_MASK);
3136 val = ldl_p(ptr);
3137 }
3138 return val;
3139 }
3140
3141 /* warning: addr must be aligned */
3142 uint64_t ldq_phys(target_phys_addr_t addr)
3143 {
3144 int io_index;
3145 uint8_t *ptr;
3146 uint64_t val;
3147 unsigned long pd;
3148 PhysPageDesc *p;
3149
3150 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3151 if (!p) {
3152 pd = IO_MEM_UNASSIGNED;
3153 } else {
3154 pd = p->phys_offset;
3155 }
3156
3157 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3158 !(pd & IO_MEM_ROMD)) {
3159 /* I/O case */
3160 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3161 if (p)
3162 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3163 #ifdef TARGET_WORDS_BIGENDIAN
3164 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3165 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3166 #else
3167 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3168 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3169 #endif
3170 } else {
3171 /* RAM case */
3172 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3173 (addr & ~TARGET_PAGE_MASK);
3174 val = ldq_p(ptr);
3175 }
3176 return val;
3177 }
3178
3179 /* XXX: optimize */
3180 uint32_t ldub_phys(target_phys_addr_t addr)
3181 {
3182 uint8_t val;
3183 cpu_physical_memory_read(addr, &val, 1);
3184 return val;
3185 }
3186
3187 /* XXX: optimize */
3188 uint32_t lduw_phys(target_phys_addr_t addr)
3189 {
3190 uint16_t val;
3191 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3192 return tswap16(val);
3193 }
3194
3195 #ifdef __GNUC__
3196 #define likely(x) __builtin_expect(!!(x), 1)
3197 #define unlikely(x) __builtin_expect(!!(x), 0)
3198 #else
3199 #define likely(x) x
3200 #define unlikely(x) x
3201 #endif
3202
3203 /* warning: addr must be aligned. The ram page is not masked as dirty
3204 and the code inside is not invalidated. It is useful if the dirty
3205 bits are used to track modified PTEs */
3206 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3207 {
3208 int io_index;
3209 uint8_t *ptr;
3210 unsigned long pd;
3211 PhysPageDesc *p;
3212
3213 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3214 if (!p) {
3215 pd = IO_MEM_UNASSIGNED;
3216 } else {
3217 pd = p->phys_offset;
3218 }
3219
3220 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3221 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3222 if (p)
3223 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3224 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3225 } else {
3226 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3227 ptr = phys_ram_base + addr1;
3228 stl_p(ptr, val);
3229
3230 if (unlikely(in_migration)) {
3231 if (!cpu_physical_memory_is_dirty(addr1)) {
3232 /* invalidate code */
3233 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3234 /* set dirty bit */
3235 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3236 (0xff & ~CODE_DIRTY_FLAG);
3237 }
3238 }
3239 }
3240 }
3241
3242 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3243 {
3244 int io_index;
3245 uint8_t *ptr;
3246 unsigned long pd;
3247 PhysPageDesc *p;
3248
3249 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3250 if (!p) {
3251 pd = IO_MEM_UNASSIGNED;
3252 } else {
3253 pd = p->phys_offset;
3254 }
3255
3256 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3257 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3258 if (p)
3259 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3260 #ifdef TARGET_WORDS_BIGENDIAN
3261 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3262 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3263 #else
3264 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3265 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3266 #endif
3267 } else {
3268 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
3269 (addr & ~TARGET_PAGE_MASK);
3270 stq_p(ptr, val);
3271 }
3272 }
3273
3274 /* warning: addr must be aligned */
3275 void stl_phys(target_phys_addr_t addr, uint32_t val)
3276 {
3277 int io_index;
3278 uint8_t *ptr;
3279 unsigned long pd;
3280 PhysPageDesc *p;
3281
3282 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3283 if (!p) {
3284 pd = IO_MEM_UNASSIGNED;
3285 } else {
3286 pd = p->phys_offset;
3287 }
3288
3289 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3290 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3291 if (p)
3292 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3293 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3294 } else {
3295 unsigned long addr1;
3296 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3297 /* RAM case */
3298 ptr = phys_ram_base + addr1;
3299 stl_p(ptr, val);
3300 if (!cpu_physical_memory_is_dirty(addr1)) {
3301 /* invalidate code */
3302 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3303 /* set dirty bit */
3304 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3305 (0xff & ~CODE_DIRTY_FLAG);
3306 }
3307 }
3308 }
3309
3310 /* XXX: optimize */
3311 void stb_phys(target_phys_addr_t addr, uint32_t val)
3312 {
3313 uint8_t v = val;
3314 cpu_physical_memory_write(addr, &v, 1);
3315 }
3316
3317 /* XXX: optimize */
3318 void stw_phys(target_phys_addr_t addr, uint32_t val)
3319 {
3320 uint16_t v = tswap16(val);
3321 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3322 }
3323
3324 /* XXX: optimize */
3325 void stq_phys(target_phys_addr_t addr, uint64_t val)
3326 {
3327 val = tswap64(val);
3328 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3329 }
3330
3331 #endif
3332
3333 /* virtual memory access for debug */
3334 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3335 uint8_t *buf, int len, int is_write)
3336 {
3337 int l;
3338 target_phys_addr_t phys_addr;
3339 target_ulong page;
3340
3341 while (len > 0) {
3342 page = addr & TARGET_PAGE_MASK;
3343 phys_addr = cpu_get_phys_page_debug(env, page);
3344 /* if no physical page mapped, return an error */
3345 if (phys_addr == -1)
3346 return -1;
3347 l = (page + TARGET_PAGE_SIZE) - addr;
3348 if (l > len)
3349 l = len;
3350 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
3351 buf, l, is_write);
3352 len -= l;
3353 buf += l;
3354 addr += l;
3355 }
3356 return 0;
3357 }
3358
3359 /* in deterministic execution mode, instructions doing device I/Os
3360 must be at the end of the TB */
3361 void cpu_io_recompile(CPUState *env, void *retaddr)
3362 {
3363 TranslationBlock *tb;
3364 uint32_t n, cflags;
3365 target_ulong pc, cs_base;
3366 uint64_t flags;
3367
3368 tb = tb_find_pc((unsigned long)retaddr);
3369 if (!tb) {
3370 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3371 retaddr);
3372 }
3373 n = env->icount_decr.u16.low + tb->icount;
3374 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3375 /* Calculate how many instructions had been executed before the fault
3376 occurred. */
3377 n = n - env->icount_decr.u16.low;
3378 /* Generate a new TB ending on the I/O insn. */
3379 n++;
3380 /* On MIPS and SH, delay slot instructions can only be restarted if
3381 they were already the first instruction in the TB. If this is not
3382 the first instruction in a TB then re-execute the preceding
3383 branch. */
3384 #if defined(TARGET_MIPS)
3385 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3386 env->active_tc.PC -= 4;
3387 env->icount_decr.u16.low++;
3388 env->hflags &= ~MIPS_HFLAG_BMASK;
3389 }
3390 #elif defined(TARGET_SH4)
3391 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3392 && n > 1) {
3393 env->pc -= 2;
3394 env->icount_decr.u16.low++;
3395 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3396 }
3397 #endif
3398 /* This should never happen. */
3399 if (n > CF_COUNT_MASK)
3400 cpu_abort(env, "TB too big during recompile");
3401
3402 cflags = n | CF_LAST_IO;
3403 pc = tb->pc;
3404 cs_base = tb->cs_base;
3405 flags = tb->flags;
3406 tb_phys_invalidate(tb, -1);
3407 /* FIXME: In theory this could raise an exception. In practice
3408 we have already translated the block once so it's probably ok. */
3409 tb_gen_code(env, pc, cs_base, flags, cflags);
3410 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3411 the first in the TB) then we end up generating a whole new TB and
3412 repeating the fault, which is horribly inefficient.
3413 Better would be to execute just this insn uncached, or generate a
3414 second new TB. */
3415 cpu_resume_from_signal(env, NULL);
3416 }
3417
3418 void dump_exec_info(FILE *f,
3419 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3420 {
3421 int i, target_code_size, max_target_code_size;
3422 int direct_jmp_count, direct_jmp2_count, cross_page;
3423 TranslationBlock *tb;
3424
3425 target_code_size = 0;
3426 max_target_code_size = 0;
3427 cross_page = 0;
3428 direct_jmp_count = 0;
3429 direct_jmp2_count = 0;
3430 for(i = 0; i < nb_tbs; i++) {
3431 tb = &tbs[i];
3432 target_code_size += tb->size;
3433 if (tb->size > max_target_code_size)
3434 max_target_code_size = tb->size;
3435 if (tb->page_addr[1] != -1)
3436 cross_page++;
3437 if (tb->tb_next_offset[0] != 0xffff) {
3438 direct_jmp_count++;
3439 if (tb->tb_next_offset[1] != 0xffff) {
3440 direct_jmp2_count++;
3441 }
3442 }
3443 }
3444 /* XXX: avoid using doubles ? */
3445 cpu_fprintf(f, "Translation buffer state:\n");
3446 cpu_fprintf(f, "gen code size %ld/%ld\n",
3447 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3448 cpu_fprintf(f, "TB count %d/%d\n",
3449 nb_tbs, code_gen_max_blocks);
3450 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3451 nb_tbs ? target_code_size / nb_tbs : 0,
3452 max_target_code_size);
3453 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3454 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3455 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3456 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3457 cross_page,
3458 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3459 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3460 direct_jmp_count,
3461 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3462 direct_jmp2_count,
3463 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3464 cpu_fprintf(f, "\nStatistics:\n");
3465 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3466 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3467 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3468 tcg_dump_info(f, cpu_fprintf);
3469 }
3470
3471 #if !defined(CONFIG_USER_ONLY)
3472
3473 #define MMUSUFFIX _cmmu
3474 #define GETPC() NULL
3475 #define env cpu_single_env
3476 #define SOFTMMU_CODE_ACCESS
3477
3478 #define SHIFT 0
3479 #include "softmmu_template.h"
3480
3481 #define SHIFT 1
3482 #include "softmmu_template.h"
3483
3484 #define SHIFT 2
3485 #include "softmmu_template.h"
3486
3487 #define SHIFT 3
3488 #include "softmmu_template.h"
3489
3490 #undef env
3491
3492 #endif
Patches shipped by Fedora