?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
target-s390: Convert STFL
[qemu/agraf.git] / exec.c
bloba6923addd48c6d0252c3296f3b560a5fa41d52b4
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_UNASSIGNED
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 int phys_ram_fd;
58 static int in_migration;
59
60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61
62 static MemoryRegion *system_memory;
63 static MemoryRegion *system_io;
64
65 AddressSpace address_space_io;
66 AddressSpace address_space_memory;
67 DMAContext dma_context_memory;
68
69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
70 static MemoryRegion io_mem_subpage_ram;
71
72 #endif
73
74 CPUArchState *first_cpu;
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUArchState *,cpu_single_env);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount = 0;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 static MemoryRegionSection *phys_sections;
86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
87 static uint16_t phys_section_unassigned;
88 static uint16_t phys_section_notdirty;
89 static uint16_t phys_section_rom;
90 static uint16_t phys_section_watch;
91
92 /* Simple allocator for PhysPageEntry nodes */
93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
95
96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
97
98 static void io_mem_init(void);
99 static void memory_map_init(void);
100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
101
102 static MemoryRegion io_mem_watch;
103 #endif
104
105 #if !defined(CONFIG_USER_ONLY)
106
107 static void phys_map_node_reserve(unsigned nodes)
108 {
109 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
110 typedef PhysPageEntry Node[L2_SIZE];
111 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
112 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
113 phys_map_nodes_nb + nodes);
114 phys_map_nodes = g_renew(Node, phys_map_nodes,
115 phys_map_nodes_nb_alloc);
116 }
117 }
118
119 static uint16_t phys_map_node_alloc(void)
120 {
121 unsigned i;
122 uint16_t ret;
123
124 ret = phys_map_nodes_nb++;
125 assert(ret != PHYS_MAP_NODE_NIL);
126 assert(ret != phys_map_nodes_nb_alloc);
127 for (i = 0; i < L2_SIZE; ++i) {
128 phys_map_nodes[ret][i].is_leaf = 0;
129 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
130 }
131 return ret;
132 }
133
134 static void phys_map_nodes_reset(void)
135 {
136 phys_map_nodes_nb = 0;
137 }
138
139
140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
141 hwaddr *nb, uint16_t leaf,
142 int level)
143 {
144 PhysPageEntry *p;
145 int i;
146 hwaddr step = (hwaddr)1 << (level * L2_BITS);
147
148 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
149 lp->ptr = phys_map_node_alloc();
150 p = phys_map_nodes[lp->ptr];
151 if (level == 0) {
152 for (i = 0; i < L2_SIZE; i++) {
153 p[i].is_leaf = 1;
154 p[i].ptr = phys_section_unassigned;
155 }
156 }
157 } else {
158 p = phys_map_nodes[lp->ptr];
159 }
160 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
161
162 while (*nb && lp < &p[L2_SIZE]) {
163 if ((*index & (step - 1)) == 0 && *nb >= step) {
164 lp->is_leaf = true;
165 lp->ptr = leaf;
166 *index += step;
167 *nb -= step;
168 } else {
169 phys_page_set_level(lp, index, nb, leaf, level - 1);
170 }
171 ++lp;
172 }
173 }
174
175 static void phys_page_set(AddressSpaceDispatch *d,
176 hwaddr index, hwaddr nb,
177 uint16_t leaf)
178 {
179 /* Wildly overreserve - it doesn't matter much. */
180 phys_map_node_reserve(3 * P_L2_LEVELS);
181
182 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
183 }
184
185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
186 {
187 PhysPageEntry lp = d->phys_map;
188 PhysPageEntry *p;
189 int i;
190 uint16_t s_index = phys_section_unassigned;
191
192 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
193 if (lp.ptr == PHYS_MAP_NODE_NIL) {
194 goto not_found;
195 }
196 p = phys_map_nodes[lp.ptr];
197 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
198 }
199
200 s_index = lp.ptr;
201 not_found:
202 return &phys_sections[s_index];
203 }
204
205 bool memory_region_is_unassigned(MemoryRegion *mr)
206 {
207 return mr != &io_mem_ram && mr != &io_mem_rom
208 && mr != &io_mem_notdirty && !mr->rom_device
209 && mr != &io_mem_watch;
210 }
211 #endif
212
213 void cpu_exec_init_all(void)
214 {
215 #if !defined(CONFIG_USER_ONLY)
216 qemu_mutex_init(&ram_list.mutex);
217 memory_map_init();
218 io_mem_init();
219 #endif
220 }
221
222 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
223
224 static int cpu_common_post_load(void *opaque, int version_id)
225 {
226 CPUArchState *env = opaque;
227
228 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
229 version_id is increased. */
230 env->interrupt_request &= ~0x01;
231 tlb_flush(env, 1);
232
233 return 0;
234 }
235
236 static const VMStateDescription vmstate_cpu_common = {
237 .name = "cpu_common",
238 .version_id = 1,
239 .minimum_version_id = 1,
240 .minimum_version_id_old = 1,
241 .post_load = cpu_common_post_load,
242 .fields = (VMStateField []) {
243 VMSTATE_UINT32(halted, CPUArchState),
244 VMSTATE_UINT32(interrupt_request, CPUArchState),
245 VMSTATE_END_OF_LIST()
246 }
247 };
248 #endif
249
250 CPUArchState *qemu_get_cpu(int cpu)
251 {
252 CPUArchState *env = first_cpu;
253
254 while (env) {
255 if (env->cpu_index == cpu)
256 break;
257 env = env->next_cpu;
258 }
259
260 return env;
261 }
262
263 void cpu_exec_init(CPUArchState *env)
264 {
265 #ifndef CONFIG_USER_ONLY
266 CPUState *cpu = ENV_GET_CPU(env);
267 #endif
268 CPUArchState **penv;
269 int cpu_index;
270
271 #if defined(CONFIG_USER_ONLY)
272 cpu_list_lock();
273 #endif
274 env->next_cpu = NULL;
275 penv = &first_cpu;
276 cpu_index = 0;
277 while (*penv != NULL) {
278 penv = &(*penv)->next_cpu;
279 cpu_index++;
280 }
281 env->cpu_index = cpu_index;
282 env->numa_node = 0;
283 QTAILQ_INIT(&env->breakpoints);
284 QTAILQ_INIT(&env->watchpoints);
285 #ifndef CONFIG_USER_ONLY
286 cpu->thread_id = qemu_get_thread_id();
287 #endif
288 *penv = env;
289 #if defined(CONFIG_USER_ONLY)
290 cpu_list_unlock();
291 #endif
292 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
293 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
294 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
295 cpu_save, cpu_load, env);
296 #endif
297 }
298
299 #if defined(TARGET_HAS_ICE)
300 #if defined(CONFIG_USER_ONLY)
301 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
302 {
303 tb_invalidate_phys_page_range(pc, pc + 1, 0);
304 }
305 #else
306 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
307 {
308 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
309 (pc & ~TARGET_PAGE_MASK));
310 }
311 #endif
312 #endif /* TARGET_HAS_ICE */
313
314 #if defined(CONFIG_USER_ONLY)
315 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
316
317 {
318 }
319
320 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
321 int flags, CPUWatchpoint **watchpoint)
322 {
323 return -ENOSYS;
324 }
325 #else
326 /* Add a watchpoint. */
327 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
328 int flags, CPUWatchpoint **watchpoint)
329 {
330 target_ulong len_mask = ~(len - 1);
331 CPUWatchpoint *wp;
332
333 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
334 if ((len & (len - 1)) || (addr & ~len_mask) ||
335 len == 0 || len > TARGET_PAGE_SIZE) {
336 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
337 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
338 return -EINVAL;
339 }
340 wp = g_malloc(sizeof(*wp));
341
342 wp->vaddr = addr;
343 wp->len_mask = len_mask;
344 wp->flags = flags;
345
346 /* keep all GDB-injected watchpoints in front */
347 if (flags & BP_GDB)
348 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
349 else
350 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
351
352 tlb_flush_page(env, addr);
353
354 if (watchpoint)
355 *watchpoint = wp;
356 return 0;
357 }
358
359 /* Remove a specific watchpoint. */
360 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
361 int flags)
362 {
363 target_ulong len_mask = ~(len - 1);
364 CPUWatchpoint *wp;
365
366 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
367 if (addr == wp->vaddr && len_mask == wp->len_mask
368 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
369 cpu_watchpoint_remove_by_ref(env, wp);
370 return 0;
371 }
372 }
373 return -ENOENT;
374 }
375
376 /* Remove a specific watchpoint by reference. */
377 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
378 {
379 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
380
381 tlb_flush_page(env, watchpoint->vaddr);
382
383 g_free(watchpoint);
384 }
385
386 /* Remove all matching watchpoints. */
387 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
388 {
389 CPUWatchpoint *wp, *next;
390
391 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
392 if (wp->flags & mask)
393 cpu_watchpoint_remove_by_ref(env, wp);
394 }
395 }
396 #endif
397
398 /* Add a breakpoint. */
399 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
400 CPUBreakpoint **breakpoint)
401 {
402 #if defined(TARGET_HAS_ICE)
403 CPUBreakpoint *bp;
404
405 bp = g_malloc(sizeof(*bp));
406
407 bp->pc = pc;
408 bp->flags = flags;
409
410 /* keep all GDB-injected breakpoints in front */
411 if (flags & BP_GDB)
412 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
413 else
414 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
415
416 breakpoint_invalidate(env, pc);
417
418 if (breakpoint)
419 *breakpoint = bp;
420 return 0;
421 #else
422 return -ENOSYS;
423 #endif
424 }
425
426 /* Remove a specific breakpoint. */
427 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
428 {
429 #if defined(TARGET_HAS_ICE)
430 CPUBreakpoint *bp;
431
432 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
433 if (bp->pc == pc && bp->flags == flags) {
434 cpu_breakpoint_remove_by_ref(env, bp);
435 return 0;
436 }
437 }
438 return -ENOENT;
439 #else
440 return -ENOSYS;
441 #endif
442 }
443
444 /* Remove a specific breakpoint by reference. */
445 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
446 {
447 #if defined(TARGET_HAS_ICE)
448 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
449
450 breakpoint_invalidate(env, breakpoint->pc);
451
452 g_free(breakpoint);
453 #endif
454 }
455
456 /* Remove all matching breakpoints. */
457 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
458 {
459 #if defined(TARGET_HAS_ICE)
460 CPUBreakpoint *bp, *next;
461
462 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
463 if (bp->flags & mask)
464 cpu_breakpoint_remove_by_ref(env, bp);
465 }
466 #endif
467 }
468
469 /* enable or disable single step mode. EXCP_DEBUG is returned by the
470 CPU loop after each instruction */
471 void cpu_single_step(CPUArchState *env, int enabled)
472 {
473 #if defined(TARGET_HAS_ICE)
474 if (env->singlestep_enabled != enabled) {
475 env->singlestep_enabled = enabled;
476 if (kvm_enabled())
477 kvm_update_guest_debug(env, 0);
478 else {
479 /* must flush all the translated code to avoid inconsistencies */
480 /* XXX: only flush what is necessary */
481 tb_flush(env);
482 }
483 }
484 #endif
485 }
486
487 void cpu_reset_interrupt(CPUArchState *env, int mask)
488 {
489 env->interrupt_request &= ~mask;
490 }
491
492 void cpu_exit(CPUArchState *env)
493 {
494 env->exit_request = 1;
495 cpu_unlink_tb(env);
496 }
497
498 void cpu_abort(CPUArchState *env, const char *fmt, ...)
499 {
500 va_list ap;
501 va_list ap2;
502
503 va_start(ap, fmt);
504 va_copy(ap2, ap);
505 fprintf(stderr, "qemu: fatal: ");
506 vfprintf(stderr, fmt, ap);
507 fprintf(stderr, "\n");
508 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
509 if (qemu_log_enabled()) {
510 qemu_log("qemu: fatal: ");
511 qemu_log_vprintf(fmt, ap2);
512 qemu_log("\n");
513 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
514 qemu_log_flush();
515 qemu_log_close();
516 }
517 va_end(ap2);
518 va_end(ap);
519 #if defined(CONFIG_USER_ONLY)
520 {
521 struct sigaction act;
522 sigfillset(&act.sa_mask);
523 act.sa_handler = SIG_DFL;
524 sigaction(SIGABRT, &act, NULL);
525 }
526 #endif
527 abort();
528 }
529
530 CPUArchState *cpu_copy(CPUArchState *env)
531 {
532 CPUArchState *new_env = cpu_init(env->cpu_model_str);
533 CPUArchState *next_cpu = new_env->next_cpu;
534 int cpu_index = new_env->cpu_index;
535 #if defined(TARGET_HAS_ICE)
536 CPUBreakpoint *bp;
537 CPUWatchpoint *wp;
538 #endif
539
540 memcpy(new_env, env, sizeof(CPUArchState));
541
542 /* Preserve chaining and index. */
543 new_env->next_cpu = next_cpu;
544 new_env->cpu_index = cpu_index;
545
546 /* Clone all break/watchpoints.
547 Note: Once we support ptrace with hw-debug register access, make sure
548 BP_CPU break/watchpoints are handled correctly on clone. */
549 QTAILQ_INIT(&env->breakpoints);
550 QTAILQ_INIT(&env->watchpoints);
551 #if defined(TARGET_HAS_ICE)
552 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
553 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
554 }
555 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
556 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
557 wp->flags, NULL);
558 }
559 #endif
560
561 return new_env;
562 }
563
564 #if !defined(CONFIG_USER_ONLY)
565 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
566 uintptr_t length)
567 {
568 uintptr_t start1;
569
570 /* we modify the TLB cache so that the dirty bit will be set again
571 when accessing the range */
572 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
573 /* Check that we don't span multiple blocks - this breaks the
574 address comparisons below. */
575 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
576 != (end - 1) - start) {
577 abort();
578 }
579 cpu_tlb_reset_dirty_all(start1, length);
580
581 }
582
583 /* Note: start and end must be within the same ram block. */
584 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
585 int dirty_flags)
586 {
587 uintptr_t length;
588
589 start &= TARGET_PAGE_MASK;
590 end = TARGET_PAGE_ALIGN(end);
591
592 length = end - start;
593 if (length == 0)
594 return;
595 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
596
597 if (tcg_enabled()) {
598 tlb_reset_dirty_range_all(start, end, length);
599 }
600 }
601
602 static int cpu_physical_memory_set_dirty_tracking(int enable)
603 {
604 int ret = 0;
605 in_migration = enable;
606 return ret;
607 }
608
609 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
610 MemoryRegionSection *section,
611 target_ulong vaddr,
612 hwaddr paddr,
613 int prot,
614 target_ulong *address)
615 {
616 hwaddr iotlb;
617 CPUWatchpoint *wp;
618
619 if (memory_region_is_ram(section->mr)) {
620 /* Normal RAM. */
621 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
622 + memory_region_section_addr(section, paddr);
623 if (!section->readonly) {
624 iotlb |= phys_section_notdirty;
625 } else {
626 iotlb |= phys_section_rom;
627 }
628 } else {
629 /* IO handlers are currently passed a physical address.
630 It would be nice to pass an offset from the base address
631 of that region. This would avoid having to special case RAM,
632 and avoid full address decoding in every device.
633 We can't use the high bits of pd for this because
634 IO_MEM_ROMD uses these as a ram address. */
635 iotlb = section - phys_sections;
636 iotlb += memory_region_section_addr(section, paddr);
637 }
638
639 /* Make accesses to pages with watchpoints go via the
640 watchpoint trap routines. */
641 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
642 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
643 /* Avoid trapping reads of pages with a write breakpoint. */
644 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
645 iotlb = phys_section_watch + paddr;
646 *address |= TLB_MMIO;
647 break;
648 }
649 }
650 }
651
652 return iotlb;
653 }
654 #endif /* defined(CONFIG_USER_ONLY) */
655
656 #if !defined(CONFIG_USER_ONLY)
657
658 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
659 typedef struct subpage_t {
660 MemoryRegion iomem;
661 hwaddr base;
662 uint16_t sub_section[TARGET_PAGE_SIZE];
663 } subpage_t;
664
665 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
666 uint16_t section);
667 static subpage_t *subpage_init(hwaddr base);
668 static void destroy_page_desc(uint16_t section_index)
669 {
670 MemoryRegionSection *section = &phys_sections[section_index];
671 MemoryRegion *mr = section->mr;
672
673 if (mr->subpage) {
674 subpage_t *subpage = container_of(mr, subpage_t, iomem);
675 memory_region_destroy(&subpage->iomem);
676 g_free(subpage);
677 }
678 }
679
680 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
681 {
682 unsigned i;
683 PhysPageEntry *p;
684
685 if (lp->ptr == PHYS_MAP_NODE_NIL) {
686 return;
687 }
688
689 p = phys_map_nodes[lp->ptr];
690 for (i = 0; i < L2_SIZE; ++i) {
691 if (!p[i].is_leaf) {
692 destroy_l2_mapping(&p[i], level - 1);
693 } else {
694 destroy_page_desc(p[i].ptr);
695 }
696 }
697 lp->is_leaf = 0;
698 lp->ptr = PHYS_MAP_NODE_NIL;
699 }
700
701 static void destroy_all_mappings(AddressSpaceDispatch *d)
702 {
703 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
704 phys_map_nodes_reset();
705 }
706
707 static uint16_t phys_section_add(MemoryRegionSection *section)
708 {
709 if (phys_sections_nb == phys_sections_nb_alloc) {
710 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
711 phys_sections = g_renew(MemoryRegionSection, phys_sections,
712 phys_sections_nb_alloc);
713 }
714 phys_sections[phys_sections_nb] = *section;
715 return phys_sections_nb++;
716 }
717
718 static void phys_sections_clear(void)
719 {
720 phys_sections_nb = 0;
721 }
722
723 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
724 {
725 subpage_t *subpage;
726 hwaddr base = section->offset_within_address_space
727 & TARGET_PAGE_MASK;
728 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
729 MemoryRegionSection subsection = {
730 .offset_within_address_space = base,
731 .size = TARGET_PAGE_SIZE,
732 };
733 hwaddr start, end;
734
735 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
736
737 if (!(existing->mr->subpage)) {
738 subpage = subpage_init(base);
739 subsection.mr = &subpage->iomem;
740 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
741 phys_section_add(&subsection));
742 } else {
743 subpage = container_of(existing->mr, subpage_t, iomem);
744 }
745 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
746 end = start + section->size - 1;
747 subpage_register(subpage, start, end, phys_section_add(section));
748 }
749
750
751 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
752 {
753 hwaddr start_addr = section->offset_within_address_space;
754 ram_addr_t size = section->size;
755 hwaddr addr;
756 uint16_t section_index = phys_section_add(section);
757
758 assert(size);
759
760 addr = start_addr;
761 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
762 section_index);
763 }
764
765 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
766 {
767 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
768 MemoryRegionSection now = *section, remain = *section;
769
770 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
771 || (now.size < TARGET_PAGE_SIZE)) {
772 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
773 - now.offset_within_address_space,
774 now.size);
775 register_subpage(d, &now);
776 remain.size -= now.size;
777 remain.offset_within_address_space += now.size;
778 remain.offset_within_region += now.size;
779 }
780 while (remain.size >= TARGET_PAGE_SIZE) {
781 now = remain;
782 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
783 now.size = TARGET_PAGE_SIZE;
784 register_subpage(d, &now);
785 } else {
786 now.size &= TARGET_PAGE_MASK;
787 register_multipage(d, &now);
788 }
789 remain.size -= now.size;
790 remain.offset_within_address_space += now.size;
791 remain.offset_within_region += now.size;
792 }
793 now = remain;
794 if (now.size) {
795 register_subpage(d, &now);
796 }
797 }
798
799 void qemu_flush_coalesced_mmio_buffer(void)
800 {
801 if (kvm_enabled())
802 kvm_flush_coalesced_mmio_buffer();
803 }
804
805 void qemu_mutex_lock_ramlist(void)
806 {
807 qemu_mutex_lock(&ram_list.mutex);
808 }
809
810 void qemu_mutex_unlock_ramlist(void)
811 {
812 qemu_mutex_unlock(&ram_list.mutex);
813 }
814
815 #if defined(__linux__) && !defined(TARGET_S390X)
816
817 #include <sys/vfs.h>
818
819 #define HUGETLBFS_MAGIC 0x958458f6
820
821 static long gethugepagesize(const char *path)
822 {
823 struct statfs fs;
824 int ret;
825
826 do {
827 ret = statfs(path, &fs);
828 } while (ret != 0 && errno == EINTR);
829
830 if (ret != 0) {
831 perror(path);
832 return 0;
833 }
834
835 if (fs.f_type != HUGETLBFS_MAGIC)
836 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
837
838 return fs.f_bsize;
839 }
840
841 static void *file_ram_alloc(RAMBlock *block,
842 ram_addr_t memory,
843 const char *path)
844 {
845 char *filename;
846 void *area;
847 int fd;
848 #ifdef MAP_POPULATE
849 int flags;
850 #endif
851 unsigned long hpagesize;
852
853 hpagesize = gethugepagesize(path);
854 if (!hpagesize) {
855 return NULL;
856 }
857
858 if (memory < hpagesize) {
859 return NULL;
860 }
861
862 if (kvm_enabled() && !kvm_has_sync_mmu()) {
863 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
864 return NULL;
865 }
866
867 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
868 return NULL;
869 }
870
871 fd = mkstemp(filename);
872 if (fd < 0) {
873 perror("unable to create backing store for hugepages");
874 free(filename);
875 return NULL;
876 }
877 unlink(filename);
878 free(filename);
879
880 memory = (memory+hpagesize-1) & ~(hpagesize-1);
881
882 /*
883 * ftruncate is not supported by hugetlbfs in older
884 * hosts, so don't bother bailing out on errors.
885 * If anything goes wrong with it under other filesystems,
886 * mmap will fail.
887 */
888 if (ftruncate(fd, memory))
889 perror("ftruncate");
890
891 #ifdef MAP_POPULATE
892 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
893 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
894 * to sidestep this quirk.
895 */
896 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
897 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
898 #else
899 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
900 #endif
901 if (area == MAP_FAILED) {
902 perror("file_ram_alloc: can't mmap RAM pages");
903 close(fd);
904 return (NULL);
905 }
906 block->fd = fd;
907 return area;
908 }
909 #endif
910
911 static ram_addr_t find_ram_offset(ram_addr_t size)
912 {
913 RAMBlock *block, *next_block;
914 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
915
916 if (QTAILQ_EMPTY(&ram_list.blocks))
917 return 0;
918
919 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
920 ram_addr_t end, next = RAM_ADDR_MAX;
921
922 end = block->offset + block->length;
923
924 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
925 if (next_block->offset >= end) {
926 next = MIN(next, next_block->offset);
927 }
928 }
929 if (next - end >= size && next - end < mingap) {
930 offset = end;
931 mingap = next - end;
932 }
933 }
934
935 if (offset == RAM_ADDR_MAX) {
936 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
937 (uint64_t)size);
938 abort();
939 }
940
941 return offset;
942 }
943
944 ram_addr_t last_ram_offset(void)
945 {
946 RAMBlock *block;
947 ram_addr_t last = 0;
948
949 QTAILQ_FOREACH(block, &ram_list.blocks, next)
950 last = MAX(last, block->offset + block->length);
951
952 return last;
953 }
954
955 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
956 {
957 int ret;
958 QemuOpts *machine_opts;
959
960 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
961 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
962 if (machine_opts &&
963 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
964 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
965 if (ret) {
966 perror("qemu_madvise");
967 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
968 "but dump_guest_core=off specified\n");
969 }
970 }
971 }
972
973 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
974 {
975 RAMBlock *new_block, *block;
976
977 new_block = NULL;
978 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
979 if (block->offset == addr) {
980 new_block = block;
981 break;
982 }
983 }
984 assert(new_block);
985 assert(!new_block->idstr[0]);
986
987 if (dev) {
988 char *id = qdev_get_dev_path(dev);
989 if (id) {
990 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
991 g_free(id);
992 }
993 }
994 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
995
996 /* This assumes the iothread lock is taken here too. */
997 qemu_mutex_lock_ramlist();
998 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
999 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1000 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1001 new_block->idstr);
1002 abort();
1003 }
1004 }
1005 qemu_mutex_unlock_ramlist();
1006 }
1007
1008 static int memory_try_enable_merging(void *addr, size_t len)
1009 {
1010 QemuOpts *opts;
1011
1012 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1013 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1014 /* disabled by the user */
1015 return 0;
1016 }
1017
1018 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1019 }
1020
1021 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1022 MemoryRegion *mr)
1023 {
1024 RAMBlock *block, *new_block;
1025
1026 size = TARGET_PAGE_ALIGN(size);
1027 new_block = g_malloc0(sizeof(*new_block));
1028
1029 /* This assumes the iothread lock is taken here too. */
1030 qemu_mutex_lock_ramlist();
1031 new_block->mr = mr;
1032 new_block->offset = find_ram_offset(size);
1033 if (host) {
1034 new_block->host = host;
1035 new_block->flags |= RAM_PREALLOC_MASK;
1036 } else {
1037 if (mem_path) {
1038 #if defined (__linux__) && !defined(TARGET_S390X)
1039 new_block->host = file_ram_alloc(new_block, size, mem_path);
1040 if (!new_block->host) {
1041 new_block->host = qemu_vmalloc(size);
1042 memory_try_enable_merging(new_block->host, size);
1043 }
1044 #else
1045 fprintf(stderr, "-mem-path option unsupported\n");
1046 exit(1);
1047 #endif
1048 } else {
1049 if (xen_enabled()) {
1050 xen_ram_alloc(new_block->offset, size, mr);
1051 } else if (kvm_enabled()) {
1052 /* some s390/kvm configurations have special constraints */
1053 new_block->host = kvm_vmalloc(size);
1054 } else {
1055 new_block->host = qemu_vmalloc(size);
1056 }
1057 memory_try_enable_merging(new_block->host, size);
1058 }
1059 }
1060 new_block->length = size;
1061
1062 /* Keep the list sorted from biggest to smallest block. */
1063 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1064 if (block->length < new_block->length) {
1065 break;
1066 }
1067 }
1068 if (block) {
1069 QTAILQ_INSERT_BEFORE(block, new_block, next);
1070 } else {
1071 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1072 }
1073 ram_list.mru_block = NULL;
1074
1075 ram_list.version++;
1076 qemu_mutex_unlock_ramlist();
1077
1078 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1079 last_ram_offset() >> TARGET_PAGE_BITS);
1080 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1081 0, size >> TARGET_PAGE_BITS);
1082 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1083
1084 qemu_ram_setup_dump(new_block->host, size);
1085 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1086
1087 if (kvm_enabled())
1088 kvm_setup_guest_memory(new_block->host, size);
1089
1090 return new_block->offset;
1091 }
1092
1093 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1094 {
1095 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1096 }
1097
1098 void qemu_ram_free_from_ptr(ram_addr_t addr)
1099 {
1100 RAMBlock *block;
1101
1102 /* This assumes the iothread lock is taken here too. */
1103 qemu_mutex_lock_ramlist();
1104 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1105 if (addr == block->offset) {
1106 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1107 ram_list.mru_block = NULL;
1108 ram_list.version++;
1109 g_free(block);
1110 break;
1111 }
1112 }
1113 qemu_mutex_unlock_ramlist();
1114 }
1115
1116 void qemu_ram_free(ram_addr_t addr)
1117 {
1118 RAMBlock *block;
1119
1120 /* This assumes the iothread lock is taken here too. */
1121 qemu_mutex_lock_ramlist();
1122 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1123 if (addr == block->offset) {
1124 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1125 ram_list.mru_block = NULL;
1126 ram_list.version++;
1127 if (block->flags & RAM_PREALLOC_MASK) {
1128 ;
1129 } else if (mem_path) {
1130 #if defined (__linux__) && !defined(TARGET_S390X)
1131 if (block->fd) {
1132 munmap(block->host, block->length);
1133 close(block->fd);
1134 } else {
1135 qemu_vfree(block->host);
1136 }
1137 #else
1138 abort();
1139 #endif
1140 } else {
1141 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1142 munmap(block->host, block->length);
1143 #else
1144 if (xen_enabled()) {
1145 xen_invalidate_map_cache_entry(block->host);
1146 } else {
1147 qemu_vfree(block->host);
1148 }
1149 #endif
1150 }
1151 g_free(block);
1152 break;
1153 }
1154 }
1155 qemu_mutex_unlock_ramlist();
1156
1157 }
1158
1159 #ifndef _WIN32
1160 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1161 {
1162 RAMBlock *block;
1163 ram_addr_t offset;
1164 int flags;
1165 void *area, *vaddr;
1166
1167 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1168 offset = addr - block->offset;
1169 if (offset < block->length) {
1170 vaddr = block->host + offset;
1171 if (block->flags & RAM_PREALLOC_MASK) {
1172 ;
1173 } else {
1174 flags = MAP_FIXED;
1175 munmap(vaddr, length);
1176 if (mem_path) {
1177 #if defined(__linux__) && !defined(TARGET_S390X)
1178 if (block->fd) {
1179 #ifdef MAP_POPULATE
1180 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1181 MAP_PRIVATE;
1182 #else
1183 flags |= MAP_PRIVATE;
1184 #endif
1185 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1186 flags, block->fd, offset);
1187 } else {
1188 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1189 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1190 flags, -1, 0);
1191 }
1192 #else
1193 abort();
1194 #endif
1195 } else {
1196 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1197 flags |= MAP_SHARED | MAP_ANONYMOUS;
1198 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1199 flags, -1, 0);
1200 #else
1201 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1202 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1203 flags, -1, 0);
1204 #endif
1205 }
1206 if (area != vaddr) {
1207 fprintf(stderr, "Could not remap addr: "
1208 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1209 length, addr);
1210 exit(1);
1211 }
1212 memory_try_enable_merging(vaddr, length);
1213 qemu_ram_setup_dump(vaddr, length);
1214 }
1215 return;
1216 }
1217 }
1218 }
1219 #endif /* !_WIN32 */
1220
1221 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1222 With the exception of the softmmu code in this file, this should
1223 only be used for local memory (e.g. video ram) that the device owns,
1224 and knows it isn't going to access beyond the end of the block.
1225
1226 It should not be used for general purpose DMA.
1227 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1228 */
1229 void *qemu_get_ram_ptr(ram_addr_t addr)
1230 {
1231 RAMBlock *block;
1232
1233 /* The list is protected by the iothread lock here. */
1234 block = ram_list.mru_block;
1235 if (block && addr - block->offset < block->length) {
1236 goto found;
1237 }
1238 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1239 if (addr - block->offset < block->length) {
1240 goto found;
1241 }
1242 }
1243
1244 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1245 abort();
1246
1247 found:
1248 ram_list.mru_block = block;
1249 if (xen_enabled()) {
1250 /* We need to check if the requested address is in the RAM
1251 * because we don't want to map the entire memory in QEMU.
1252 * In that case just map until the end of the page.
1253 */
1254 if (block->offset == 0) {
1255 return xen_map_cache(addr, 0, 0);
1256 } else if (block->host == NULL) {
1257 block->host =
1258 xen_map_cache(block->offset, block->length, 1);
1259 }
1260 }
1261 return block->host + (addr - block->offset);
1262 }
1263
1264 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1265 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1266 *
1267 * ??? Is this still necessary?
1268 */
1269 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1270 {
1271 RAMBlock *block;
1272
1273 /* The list is protected by the iothread lock here. */
1274 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1275 if (addr - block->offset < block->length) {
1276 if (xen_enabled()) {
1277 /* We need to check if the requested address is in the RAM
1278 * because we don't want to map the entire memory in QEMU.
1279 * In that case just map until the end of the page.
1280 */
1281 if (block->offset == 0) {
1282 return xen_map_cache(addr, 0, 0);
1283 } else if (block->host == NULL) {
1284 block->host =
1285 xen_map_cache(block->offset, block->length, 1);
1286 }
1287 }
1288 return block->host + (addr - block->offset);
1289 }
1290 }
1291
1292 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1293 abort();
1294
1295 return NULL;
1296 }
1297
1298 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1299 * but takes a size argument */
1300 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1301 {
1302 if (*size == 0) {
1303 return NULL;
1304 }
1305 if (xen_enabled()) {
1306 return xen_map_cache(addr, *size, 1);
1307 } else {
1308 RAMBlock *block;
1309
1310 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1311 if (addr - block->offset < block->length) {
1312 if (addr - block->offset + *size > block->length)
1313 *size = block->length - addr + block->offset;
1314 return block->host + (addr - block->offset);
1315 }
1316 }
1317
1318 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1319 abort();
1320 }
1321 }
1322
1323 void qemu_put_ram_ptr(void *addr)
1324 {
1325 trace_qemu_put_ram_ptr(addr);
1326 }
1327
1328 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1329 {
1330 RAMBlock *block;
1331 uint8_t *host = ptr;
1332
1333 if (xen_enabled()) {
1334 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1335 return 0;
1336 }
1337
1338 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1339 /* This case append when the block is not mapped. */
1340 if (block->host == NULL) {
1341 continue;
1342 }
1343 if (host - block->host < block->length) {
1344 *ram_addr = block->offset + (host - block->host);
1345 return 0;
1346 }
1347 }
1348
1349 return -1;
1350 }
1351
1352 /* Some of the softmmu routines need to translate from a host pointer
1353 (typically a TLB entry) back to a ram offset. */
1354 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1355 {
1356 ram_addr_t ram_addr;
1357
1358 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1359 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1360 abort();
1361 }
1362 return ram_addr;
1363 }
1364
1365 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1366 unsigned size)
1367 {
1368 #ifdef DEBUG_UNASSIGNED
1369 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1370 #endif
1371 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1372 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1373 #endif
1374 return 0;
1375 }
1376
1377 static void unassigned_mem_write(void *opaque, hwaddr addr,
1378 uint64_t val, unsigned size)
1379 {
1380 #ifdef DEBUG_UNASSIGNED
1381 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1382 #endif
1383 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1384 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1385 #endif
1386 }
1387
1388 static const MemoryRegionOps unassigned_mem_ops = {
1389 .read = unassigned_mem_read,
1390 .write = unassigned_mem_write,
1391 .endianness = DEVICE_NATIVE_ENDIAN,
1392 };
1393
1394 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1395 unsigned size)
1396 {
1397 abort();
1398 }
1399
1400 static void error_mem_write(void *opaque, hwaddr addr,
1401 uint64_t value, unsigned size)
1402 {
1403 abort();
1404 }
1405
1406 static const MemoryRegionOps error_mem_ops = {
1407 .read = error_mem_read,
1408 .write = error_mem_write,
1409 .endianness = DEVICE_NATIVE_ENDIAN,
1410 };
1411
1412 static const MemoryRegionOps rom_mem_ops = {
1413 .read = error_mem_read,
1414 .write = unassigned_mem_write,
1415 .endianness = DEVICE_NATIVE_ENDIAN,
1416 };
1417
1418 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1419 uint64_t val, unsigned size)
1420 {
1421 int dirty_flags;
1422 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1423 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1424 #if !defined(CONFIG_USER_ONLY)
1425 tb_invalidate_phys_page_fast(ram_addr, size);
1426 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1427 #endif
1428 }
1429 switch (size) {
1430 case 1:
1431 stb_p(qemu_get_ram_ptr(ram_addr), val);
1432 break;
1433 case 2:
1434 stw_p(qemu_get_ram_ptr(ram_addr), val);
1435 break;
1436 case 4:
1437 stl_p(qemu_get_ram_ptr(ram_addr), val);
1438 break;
1439 default:
1440 abort();
1441 }
1442 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1443 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1444 /* we remove the notdirty callback only if the code has been
1445 flushed */
1446 if (dirty_flags == 0xff)
1447 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1448 }
1449
1450 static const MemoryRegionOps notdirty_mem_ops = {
1451 .read = error_mem_read,
1452 .write = notdirty_mem_write,
1453 .endianness = DEVICE_NATIVE_ENDIAN,
1454 };
1455
1456 /* Generate a debug exception if a watchpoint has been hit. */
1457 static void check_watchpoint(int offset, int len_mask, int flags)
1458 {
1459 CPUArchState *env = cpu_single_env;
1460 target_ulong pc, cs_base;
1461 target_ulong vaddr;
1462 CPUWatchpoint *wp;
1463 int cpu_flags;
1464
1465 if (env->watchpoint_hit) {
1466 /* We re-entered the check after replacing the TB. Now raise
1467 * the debug interrupt so that is will trigger after the
1468 * current instruction. */
1469 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1470 return;
1471 }
1472 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1473 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1474 if ((vaddr == (wp->vaddr & len_mask) ||
1475 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1476 wp->flags |= BP_WATCHPOINT_HIT;
1477 if (!env->watchpoint_hit) {
1478 env->watchpoint_hit = wp;
1479 tb_check_watchpoint(env);
1480 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1481 env->exception_index = EXCP_DEBUG;
1482 cpu_loop_exit(env);
1483 } else {
1484 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1485 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1486 cpu_resume_from_signal(env, NULL);
1487 }
1488 }
1489 } else {
1490 wp->flags &= ~BP_WATCHPOINT_HIT;
1491 }
1492 }
1493 }
1494
1495 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1496 so these check for a hit then pass through to the normal out-of-line
1497 phys routines. */
1498 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1499 unsigned size)
1500 {
1501 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1502 switch (size) {
1503 case 1: return ldub_phys(addr);
1504 case 2: return lduw_phys(addr);
1505 case 4: return ldl_phys(addr);
1506 default: abort();
1507 }
1508 }
1509
1510 static void watch_mem_write(void *opaque, hwaddr addr,
1511 uint64_t val, unsigned size)
1512 {
1513 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1514 switch (size) {
1515 case 1:
1516 stb_phys(addr, val);
1517 break;
1518 case 2:
1519 stw_phys(addr, val);
1520 break;
1521 case 4:
1522 stl_phys(addr, val);
1523 break;
1524 default: abort();
1525 }
1526 }
1527
1528 static const MemoryRegionOps watch_mem_ops = {
1529 .read = watch_mem_read,
1530 .write = watch_mem_write,
1531 .endianness = DEVICE_NATIVE_ENDIAN,
1532 };
1533
1534 static uint64_t subpage_read(void *opaque, hwaddr addr,
1535 unsigned len)
1536 {
1537 subpage_t *mmio = opaque;
1538 unsigned int idx = SUBPAGE_IDX(addr);
1539 MemoryRegionSection *section;
1540 #if defined(DEBUG_SUBPAGE)
1541 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1542 mmio, len, addr, idx);
1543 #endif
1544
1545 section = &phys_sections[mmio->sub_section[idx]];
1546 addr += mmio->base;
1547 addr -= section->offset_within_address_space;
1548 addr += section->offset_within_region;
1549 return io_mem_read(section->mr, addr, len);
1550 }
1551
1552 static void subpage_write(void *opaque, hwaddr addr,
1553 uint64_t value, unsigned len)
1554 {
1555 subpage_t *mmio = opaque;
1556 unsigned int idx = SUBPAGE_IDX(addr);
1557 MemoryRegionSection *section;
1558 #if defined(DEBUG_SUBPAGE)
1559 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1560 " idx %d value %"PRIx64"\n",
1561 __func__, mmio, len, addr, idx, value);
1562 #endif
1563
1564 section = &phys_sections[mmio->sub_section[idx]];
1565 addr += mmio->base;
1566 addr -= section->offset_within_address_space;
1567 addr += section->offset_within_region;
1568 io_mem_write(section->mr, addr, value, len);
1569 }
1570
1571 static const MemoryRegionOps subpage_ops = {
1572 .read = subpage_read,
1573 .write = subpage_write,
1574 .endianness = DEVICE_NATIVE_ENDIAN,
1575 };
1576
1577 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1578 unsigned size)
1579 {
1580 ram_addr_t raddr = addr;
1581 void *ptr = qemu_get_ram_ptr(raddr);
1582 switch (size) {
1583 case 1: return ldub_p(ptr);
1584 case 2: return lduw_p(ptr);
1585 case 4: return ldl_p(ptr);
1586 default: abort();
1587 }
1588 }
1589
1590 static void subpage_ram_write(void *opaque, hwaddr addr,
1591 uint64_t value, unsigned size)
1592 {
1593 ram_addr_t raddr = addr;
1594 void *ptr = qemu_get_ram_ptr(raddr);
1595 switch (size) {
1596 case 1: return stb_p(ptr, value);
1597 case 2: return stw_p(ptr, value);
1598 case 4: return stl_p(ptr, value);
1599 default: abort();
1600 }
1601 }
1602
1603 static const MemoryRegionOps subpage_ram_ops = {
1604 .read = subpage_ram_read,
1605 .write = subpage_ram_write,
1606 .endianness = DEVICE_NATIVE_ENDIAN,
1607 };
1608
1609 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1610 uint16_t section)
1611 {
1612 int idx, eidx;
1613
1614 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1615 return -1;
1616 idx = SUBPAGE_IDX(start);
1617 eidx = SUBPAGE_IDX(end);
1618 #if defined(DEBUG_SUBPAGE)
1619 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1620 mmio, start, end, idx, eidx, memory);
1621 #endif
1622 if (memory_region_is_ram(phys_sections[section].mr)) {
1623 MemoryRegionSection new_section = phys_sections[section];
1624 new_section.mr = &io_mem_subpage_ram;
1625 section = phys_section_add(&new_section);
1626 }
1627 for (; idx <= eidx; idx++) {
1628 mmio->sub_section[idx] = section;
1629 }
1630
1631 return 0;
1632 }
1633
1634 static subpage_t *subpage_init(hwaddr base)
1635 {
1636 subpage_t *mmio;
1637
1638 mmio = g_malloc0(sizeof(subpage_t));
1639
1640 mmio->base = base;
1641 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1642 "subpage", TARGET_PAGE_SIZE);
1643 mmio->iomem.subpage = true;
1644 #if defined(DEBUG_SUBPAGE)
1645 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1646 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1647 #endif
1648 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1649
1650 return mmio;
1651 }
1652
1653 static uint16_t dummy_section(MemoryRegion *mr)
1654 {
1655 MemoryRegionSection section = {
1656 .mr = mr,
1657 .offset_within_address_space = 0,
1658 .offset_within_region = 0,
1659 .size = UINT64_MAX,
1660 };
1661
1662 return phys_section_add(&section);
1663 }
1664
1665 MemoryRegion *iotlb_to_region(hwaddr index)
1666 {
1667 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1668 }
1669
1670 static void io_mem_init(void)
1671 {
1672 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1673 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1674 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1675 "unassigned", UINT64_MAX);
1676 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1677 "notdirty", UINT64_MAX);
1678 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1679 "subpage-ram", UINT64_MAX);
1680 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1681 "watch", UINT64_MAX);
1682 }
1683
1684 static void mem_begin(MemoryListener *listener)
1685 {
1686 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1687
1688 destroy_all_mappings(d);
1689 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1690 }
1691
1692 static void core_begin(MemoryListener *listener)
1693 {
1694 phys_sections_clear();
1695 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1696 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1697 phys_section_rom = dummy_section(&io_mem_rom);
1698 phys_section_watch = dummy_section(&io_mem_watch);
1699 }
1700
1701 static void tcg_commit(MemoryListener *listener)
1702 {
1703 CPUArchState *env;
1704
1705 /* since each CPU stores ram addresses in its TLB cache, we must
1706 reset the modified entries */
1707 /* XXX: slow ! */
1708 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1709 tlb_flush(env, 1);
1710 }
1711 }
1712
1713 static void core_log_global_start(MemoryListener *listener)
1714 {
1715 cpu_physical_memory_set_dirty_tracking(1);
1716 }
1717
1718 static void core_log_global_stop(MemoryListener *listener)
1719 {
1720 cpu_physical_memory_set_dirty_tracking(0);
1721 }
1722
1723 static void io_region_add(MemoryListener *listener,
1724 MemoryRegionSection *section)
1725 {
1726 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1727
1728 mrio->mr = section->mr;
1729 mrio->offset = section->offset_within_region;
1730 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1731 section->offset_within_address_space, section->size);
1732 ioport_register(&mrio->iorange);
1733 }
1734
1735 static void io_region_del(MemoryListener *listener,
1736 MemoryRegionSection *section)
1737 {
1738 isa_unassign_ioport(section->offset_within_address_space, section->size);
1739 }
1740
1741 static MemoryListener core_memory_listener = {
1742 .begin = core_begin,
1743 .log_global_start = core_log_global_start,
1744 .log_global_stop = core_log_global_stop,
1745 .priority = 1,
1746 };
1747
1748 static MemoryListener io_memory_listener = {
1749 .region_add = io_region_add,
1750 .region_del = io_region_del,
1751 .priority = 0,
1752 };
1753
1754 static MemoryListener tcg_memory_listener = {
1755 .commit = tcg_commit,
1756 };
1757
1758 void address_space_init_dispatch(AddressSpace *as)
1759 {
1760 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1761
1762 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1763 d->listener = (MemoryListener) {
1764 .begin = mem_begin,
1765 .region_add = mem_add,
1766 .region_nop = mem_add,
1767 .priority = 0,
1768 };
1769 as->dispatch = d;
1770 memory_listener_register(&d->listener, as);
1771 }
1772
1773 void address_space_destroy_dispatch(AddressSpace *as)
1774 {
1775 AddressSpaceDispatch *d = as->dispatch;
1776
1777 memory_listener_unregister(&d->listener);
1778 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1779 g_free(d);
1780 as->dispatch = NULL;
1781 }
1782
1783 static void memory_map_init(void)
1784 {
1785 system_memory = g_malloc(sizeof(*system_memory));
1786 memory_region_init(system_memory, "system", INT64_MAX);
1787 address_space_init(&address_space_memory, system_memory);
1788 address_space_memory.name = "memory";
1789
1790 system_io = g_malloc(sizeof(*system_io));
1791 memory_region_init(system_io, "io", 65536);
1792 address_space_init(&address_space_io, system_io);
1793 address_space_io.name = "I/O";
1794
1795 memory_listener_register(&core_memory_listener, &address_space_memory);
1796 memory_listener_register(&io_memory_listener, &address_space_io);
1797 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1798
1799 dma_context_init(&dma_context_memory, &address_space_memory,
1800 NULL, NULL, NULL);
1801 }
1802
1803 MemoryRegion *get_system_memory(void)
1804 {
1805 return system_memory;
1806 }
1807
1808 MemoryRegion *get_system_io(void)
1809 {
1810 return system_io;
1811 }
1812
1813 #endif /* !defined(CONFIG_USER_ONLY) */
1814
1815 /* physical memory access (slow version, mainly for debug) */
1816 #if defined(CONFIG_USER_ONLY)
1817 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1818 uint8_t *buf, int len, int is_write)
1819 {
1820 int l, flags;
1821 target_ulong page;
1822 void * p;
1823
1824 while (len > 0) {
1825 page = addr & TARGET_PAGE_MASK;
1826 l = (page + TARGET_PAGE_SIZE) - addr;
1827 if (l > len)
1828 l = len;
1829 flags = page_get_flags(page);
1830 if (!(flags & PAGE_VALID))
1831 return -1;
1832 if (is_write) {
1833 if (!(flags & PAGE_WRITE))
1834 return -1;
1835 /* XXX: this code should not depend on lock_user */
1836 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1837 return -1;
1838 memcpy(p, buf, l);
1839 unlock_user(p, addr, l);
1840 } else {
1841 if (!(flags & PAGE_READ))
1842 return -1;
1843 /* XXX: this code should not depend on lock_user */
1844 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1845 return -1;
1846 memcpy(buf, p, l);
1847 unlock_user(p, addr, 0);
1848 }
1849 len -= l;
1850 buf += l;
1851 addr += l;
1852 }
1853 return 0;
1854 }
1855
1856 #else
1857
1858 static void invalidate_and_set_dirty(hwaddr addr,
1859 hwaddr length)
1860 {
1861 if (!cpu_physical_memory_is_dirty(addr)) {
1862 /* invalidate code */
1863 tb_invalidate_phys_page_range(addr, addr + length, 0);
1864 /* set dirty bit */
1865 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1866 }
1867 xen_modified_memory(addr, length);
1868 }
1869
1870 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1871 int len, bool is_write)
1872 {
1873 AddressSpaceDispatch *d = as->dispatch;
1874 int l;
1875 uint8_t *ptr;
1876 uint32_t val;
1877 hwaddr page;
1878 MemoryRegionSection *section;
1879
1880 while (len > 0) {
1881 page = addr & TARGET_PAGE_MASK;
1882 l = (page + TARGET_PAGE_SIZE) - addr;
1883 if (l > len)
1884 l = len;
1885 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1886
1887 if (is_write) {
1888 if (!memory_region_is_ram(section->mr)) {
1889 hwaddr addr1;
1890 addr1 = memory_region_section_addr(section, addr);
1891 /* XXX: could force cpu_single_env to NULL to avoid
1892 potential bugs */
1893 if (l >= 4 && ((addr1 & 3) == 0)) {
1894 /* 32 bit write access */
1895 val = ldl_p(buf);
1896 io_mem_write(section->mr, addr1, val, 4);
1897 l = 4;
1898 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1899 /* 16 bit write access */
1900 val = lduw_p(buf);
1901 io_mem_write(section->mr, addr1, val, 2);
1902 l = 2;
1903 } else {
1904 /* 8 bit write access */
1905 val = ldub_p(buf);
1906 io_mem_write(section->mr, addr1, val, 1);
1907 l = 1;
1908 }
1909 } else if (!section->readonly) {
1910 ram_addr_t addr1;
1911 addr1 = memory_region_get_ram_addr(section->mr)
1912 + memory_region_section_addr(section, addr);
1913 /* RAM case */
1914 ptr = qemu_get_ram_ptr(addr1);
1915 memcpy(ptr, buf, l);
1916 invalidate_and_set_dirty(addr1, l);
1917 qemu_put_ram_ptr(ptr);
1918 }
1919 } else {
1920 if (!(memory_region_is_ram(section->mr) ||
1921 memory_region_is_romd(section->mr))) {
1922 hwaddr addr1;
1923 /* I/O case */
1924 addr1 = memory_region_section_addr(section, addr);
1925 if (l >= 4 && ((addr1 & 3) == 0)) {
1926 /* 32 bit read access */
1927 val = io_mem_read(section->mr, addr1, 4);
1928 stl_p(buf, val);
1929 l = 4;
1930 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1931 /* 16 bit read access */
1932 val = io_mem_read(section->mr, addr1, 2);
1933 stw_p(buf, val);
1934 l = 2;
1935 } else {
1936 /* 8 bit read access */
1937 val = io_mem_read(section->mr, addr1, 1);
1938 stb_p(buf, val);
1939 l = 1;
1940 }
1941 } else {
1942 /* RAM case */
1943 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1944 + memory_region_section_addr(section,
1945 addr));
1946 memcpy(buf, ptr, l);
1947 qemu_put_ram_ptr(ptr);
1948 }
1949 }
1950 len -= l;
1951 buf += l;
1952 addr += l;
1953 }
1954 }
1955
1956 void address_space_write(AddressSpace *as, hwaddr addr,
1957 const uint8_t *buf, int len)
1958 {
1959 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1960 }
1961
1962 /**
1963 * address_space_read: read from an address space.
1964 *
1965 * @as: #AddressSpace to be accessed
1966 * @addr: address within that address space
1967 * @buf: buffer with the data transferred
1968 */
1969 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1970 {
1971 address_space_rw(as, addr, buf, len, false);
1972 }
1973
1974
1975 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1976 int len, int is_write)
1977 {
1978 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1979 }
1980
1981 /* used for ROM loading : can write in RAM and ROM */
1982 void cpu_physical_memory_write_rom(hwaddr addr,
1983 const uint8_t *buf, int len)
1984 {
1985 AddressSpaceDispatch *d = address_space_memory.dispatch;
1986 int l;
1987 uint8_t *ptr;
1988 hwaddr page;
1989 MemoryRegionSection *section;
1990
1991 while (len > 0) {
1992 page = addr & TARGET_PAGE_MASK;
1993 l = (page + TARGET_PAGE_SIZE) - addr;
1994 if (l > len)
1995 l = len;
1996 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1997
1998 if (!(memory_region_is_ram(section->mr) ||
1999 memory_region_is_romd(section->mr))) {
2000 /* do nothing */
2001 } else {
2002 unsigned long addr1;
2003 addr1 = memory_region_get_ram_addr(section->mr)
2004 + memory_region_section_addr(section, addr);
2005 /* ROM/RAM case */
2006 ptr = qemu_get_ram_ptr(addr1);
2007 memcpy(ptr, buf, l);
2008 invalidate_and_set_dirty(addr1, l);
2009 qemu_put_ram_ptr(ptr);
2010 }
2011 len -= l;
2012 buf += l;
2013 addr += l;
2014 }
2015 }
2016
2017 typedef struct {
2018 void *buffer;
2019 hwaddr addr;
2020 hwaddr len;
2021 } BounceBuffer;
2022
2023 static BounceBuffer bounce;
2024
2025 typedef struct MapClient {
2026 void *opaque;
2027 void (*callback)(void *opaque);
2028 QLIST_ENTRY(MapClient) link;
2029 } MapClient;
2030
2031 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2032 = QLIST_HEAD_INITIALIZER(map_client_list);
2033
2034 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2035 {
2036 MapClient *client = g_malloc(sizeof(*client));
2037
2038 client->opaque = opaque;
2039 client->callback = callback;
2040 QLIST_INSERT_HEAD(&map_client_list, client, link);
2041 return client;
2042 }
2043
2044 static void cpu_unregister_map_client(void *_client)
2045 {
2046 MapClient *client = (MapClient *)_client;
2047
2048 QLIST_REMOVE(client, link);
2049 g_free(client);
2050 }
2051
2052 static void cpu_notify_map_clients(void)
2053 {
2054 MapClient *client;
2055
2056 while (!QLIST_EMPTY(&map_client_list)) {
2057 client = QLIST_FIRST(&map_client_list);
2058 client->callback(client->opaque);
2059 cpu_unregister_map_client(client);
2060 }
2061 }
2062
2063 /* Map a physical memory region into a host virtual address.
2064 * May map a subset of the requested range, given by and returned in *plen.
2065 * May return NULL if resources needed to perform the mapping are exhausted.
2066 * Use only for reads OR writes - not for read-modify-write operations.
2067 * Use cpu_register_map_client() to know when retrying the map operation is
2068 * likely to succeed.
2069 */
2070 void *address_space_map(AddressSpace *as,
2071 hwaddr addr,
2072 hwaddr *plen,
2073 bool is_write)
2074 {
2075 AddressSpaceDispatch *d = as->dispatch;
2076 hwaddr len = *plen;
2077 hwaddr todo = 0;
2078 int l;
2079 hwaddr page;
2080 MemoryRegionSection *section;
2081 ram_addr_t raddr = RAM_ADDR_MAX;
2082 ram_addr_t rlen;
2083 void *ret;
2084
2085 while (len > 0) {
2086 page = addr & TARGET_PAGE_MASK;
2087 l = (page + TARGET_PAGE_SIZE) - addr;
2088 if (l > len)
2089 l = len;
2090 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2091
2092 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2093 if (todo || bounce.buffer) {
2094 break;
2095 }
2096 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2097 bounce.addr = addr;
2098 bounce.len = l;
2099 if (!is_write) {
2100 address_space_read(as, addr, bounce.buffer, l);
2101 }
2102
2103 *plen = l;
2104 return bounce.buffer;
2105 }
2106 if (!todo) {
2107 raddr = memory_region_get_ram_addr(section->mr)
2108 + memory_region_section_addr(section, addr);
2109 }
2110
2111 len -= l;
2112 addr += l;
2113 todo += l;
2114 }
2115 rlen = todo;
2116 ret = qemu_ram_ptr_length(raddr, &rlen);
2117 *plen = rlen;
2118 return ret;
2119 }
2120
2121 /* Unmaps a memory region previously mapped by address_space_map().
2122 * Will also mark the memory as dirty if is_write == 1. access_len gives
2123 * the amount of memory that was actually read or written by the caller.
2124 */
2125 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2126 int is_write, hwaddr access_len)
2127 {
2128 if (buffer != bounce.buffer) {
2129 if (is_write) {
2130 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2131 while (access_len) {
2132 unsigned l;
2133 l = TARGET_PAGE_SIZE;
2134 if (l > access_len)
2135 l = access_len;
2136 invalidate_and_set_dirty(addr1, l);
2137 addr1 += l;
2138 access_len -= l;
2139 }
2140 }
2141 if (xen_enabled()) {
2142 xen_invalidate_map_cache_entry(buffer);
2143 }
2144 return;
2145 }
2146 if (is_write) {
2147 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2148 }
2149 qemu_vfree(bounce.buffer);
2150 bounce.buffer = NULL;
2151 cpu_notify_map_clients();
2152 }
2153
2154 void *cpu_physical_memory_map(hwaddr addr,
2155 hwaddr *plen,
2156 int is_write)
2157 {
2158 return address_space_map(&address_space_memory, addr, plen, is_write);
2159 }
2160
2161 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2162 int is_write, hwaddr access_len)
2163 {
2164 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2165 }
2166
2167 /* warning: addr must be aligned */
2168 static inline uint32_t ldl_phys_internal(hwaddr addr,
2169 enum device_endian endian)
2170 {
2171 uint8_t *ptr;
2172 uint32_t val;
2173 MemoryRegionSection *section;
2174
2175 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2176
2177 if (!(memory_region_is_ram(section->mr) ||
2178 memory_region_is_romd(section->mr))) {
2179 /* I/O case */
2180 addr = memory_region_section_addr(section, addr);
2181 val = io_mem_read(section->mr, addr, 4);
2182 #if defined(TARGET_WORDS_BIGENDIAN)
2183 if (endian == DEVICE_LITTLE_ENDIAN) {
2184 val = bswap32(val);
2185 }
2186 #else
2187 if (endian == DEVICE_BIG_ENDIAN) {
2188 val = bswap32(val);
2189 }
2190 #endif
2191 } else {
2192 /* RAM case */
2193 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2194 & TARGET_PAGE_MASK)
2195 + memory_region_section_addr(section, addr));
2196 switch (endian) {
2197 case DEVICE_LITTLE_ENDIAN:
2198 val = ldl_le_p(ptr);
2199 break;
2200 case DEVICE_BIG_ENDIAN:
2201 val = ldl_be_p(ptr);
2202 break;
2203 default:
2204 val = ldl_p(ptr);
2205 break;
2206 }
2207 }
2208 return val;
2209 }
2210
2211 uint32_t ldl_phys(hwaddr addr)
2212 {
2213 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2214 }
2215
2216 uint32_t ldl_le_phys(hwaddr addr)
2217 {
2218 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2219 }
2220
2221 uint32_t ldl_be_phys(hwaddr addr)
2222 {
2223 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2224 }
2225
2226 /* warning: addr must be aligned */
2227 static inline uint64_t ldq_phys_internal(hwaddr addr,
2228 enum device_endian endian)
2229 {
2230 uint8_t *ptr;
2231 uint64_t val;
2232 MemoryRegionSection *section;
2233
2234 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2235
2236 if (!(memory_region_is_ram(section->mr) ||
2237 memory_region_is_romd(section->mr))) {
2238 /* I/O case */
2239 addr = memory_region_section_addr(section, addr);
2240
2241 /* XXX This is broken when device endian != cpu endian.
2242 Fix and add "endian" variable check */
2243 #ifdef TARGET_WORDS_BIGENDIAN
2244 val = io_mem_read(section->mr, addr, 4) << 32;
2245 val |= io_mem_read(section->mr, addr + 4, 4);
2246 #else
2247 val = io_mem_read(section->mr, addr, 4);
2248 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2249 #endif
2250 } else {
2251 /* RAM case */
2252 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2253 & TARGET_PAGE_MASK)
2254 + memory_region_section_addr(section, addr));
2255 switch (endian) {
2256 case DEVICE_LITTLE_ENDIAN:
2257 val = ldq_le_p(ptr);
2258 break;
2259 case DEVICE_BIG_ENDIAN:
2260 val = ldq_be_p(ptr);
2261 break;
2262 default:
2263 val = ldq_p(ptr);
2264 break;
2265 }
2266 }
2267 return val;
2268 }
2269
2270 uint64_t ldq_phys(hwaddr addr)
2271 {
2272 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2273 }
2274
2275 uint64_t ldq_le_phys(hwaddr addr)
2276 {
2277 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2278 }
2279
2280 uint64_t ldq_be_phys(hwaddr addr)
2281 {
2282 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2283 }
2284
2285 /* XXX: optimize */
2286 uint32_t ldub_phys(hwaddr addr)
2287 {
2288 uint8_t val;
2289 cpu_physical_memory_read(addr, &val, 1);
2290 return val;
2291 }
2292
2293 /* warning: addr must be aligned */
2294 static inline uint32_t lduw_phys_internal(hwaddr addr,
2295 enum device_endian endian)
2296 {
2297 uint8_t *ptr;
2298 uint64_t val;
2299 MemoryRegionSection *section;
2300
2301 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2302
2303 if (!(memory_region_is_ram(section->mr) ||
2304 memory_region_is_romd(section->mr))) {
2305 /* I/O case */
2306 addr = memory_region_section_addr(section, addr);
2307 val = io_mem_read(section->mr, addr, 2);
2308 #if defined(TARGET_WORDS_BIGENDIAN)
2309 if (endian == DEVICE_LITTLE_ENDIAN) {
2310 val = bswap16(val);
2311 }
2312 #else
2313 if (endian == DEVICE_BIG_ENDIAN) {
2314 val = bswap16(val);
2315 }
2316 #endif
2317 } else {
2318 /* RAM case */
2319 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2320 & TARGET_PAGE_MASK)
2321 + memory_region_section_addr(section, addr));
2322 switch (endian) {
2323 case DEVICE_LITTLE_ENDIAN:
2324 val = lduw_le_p(ptr);
2325 break;
2326 case DEVICE_BIG_ENDIAN:
2327 val = lduw_be_p(ptr);
2328 break;
2329 default:
2330 val = lduw_p(ptr);
2331 break;
2332 }
2333 }
2334 return val;
2335 }
2336
2337 uint32_t lduw_phys(hwaddr addr)
2338 {
2339 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2340 }
2341
2342 uint32_t lduw_le_phys(hwaddr addr)
2343 {
2344 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2345 }
2346
2347 uint32_t lduw_be_phys(hwaddr addr)
2348 {
2349 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2350 }
2351
2352 /* warning: addr must be aligned. The ram page is not masked as dirty
2353 and the code inside is not invalidated. It is useful if the dirty
2354 bits are used to track modified PTEs */
2355 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2356 {
2357 uint8_t *ptr;
2358 MemoryRegionSection *section;
2359
2360 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2361
2362 if (!memory_region_is_ram(section->mr) || section->readonly) {
2363 addr = memory_region_section_addr(section, addr);
2364 if (memory_region_is_ram(section->mr)) {
2365 section = &phys_sections[phys_section_rom];
2366 }
2367 io_mem_write(section->mr, addr, val, 4);
2368 } else {
2369 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2370 & TARGET_PAGE_MASK)
2371 + memory_region_section_addr(section, addr);
2372 ptr = qemu_get_ram_ptr(addr1);
2373 stl_p(ptr, val);
2374
2375 if (unlikely(in_migration)) {
2376 if (!cpu_physical_memory_is_dirty(addr1)) {
2377 /* invalidate code */
2378 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2379 /* set dirty bit */
2380 cpu_physical_memory_set_dirty_flags(
2381 addr1, (0xff & ~CODE_DIRTY_FLAG));
2382 }
2383 }
2384 }
2385 }
2386
2387 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2388 {
2389 uint8_t *ptr;
2390 MemoryRegionSection *section;
2391
2392 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2393
2394 if (!memory_region_is_ram(section->mr) || section->readonly) {
2395 addr = memory_region_section_addr(section, addr);
2396 if (memory_region_is_ram(section->mr)) {
2397 section = &phys_sections[phys_section_rom];
2398 }
2399 #ifdef TARGET_WORDS_BIGENDIAN
2400 io_mem_write(section->mr, addr, val >> 32, 4);
2401 io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2402 #else
2403 io_mem_write(section->mr, addr, (uint32_t)val, 4);
2404 io_mem_write(section->mr, addr + 4, val >> 32, 4);
2405 #endif
2406 } else {
2407 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2408 & TARGET_PAGE_MASK)
2409 + memory_region_section_addr(section, addr));
2410 stq_p(ptr, val);
2411 }
2412 }
2413
2414 /* warning: addr must be aligned */
2415 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2416 enum device_endian endian)
2417 {
2418 uint8_t *ptr;
2419 MemoryRegionSection *section;
2420
2421 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2422
2423 if (!memory_region_is_ram(section->mr) || section->readonly) {
2424 addr = memory_region_section_addr(section, addr);
2425 if (memory_region_is_ram(section->mr)) {
2426 section = &phys_sections[phys_section_rom];
2427 }
2428 #if defined(TARGET_WORDS_BIGENDIAN)
2429 if (endian == DEVICE_LITTLE_ENDIAN) {
2430 val = bswap32(val);
2431 }
2432 #else
2433 if (endian == DEVICE_BIG_ENDIAN) {
2434 val = bswap32(val);
2435 }
2436 #endif
2437 io_mem_write(section->mr, addr, val, 4);
2438 } else {
2439 unsigned long addr1;
2440 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2441 + memory_region_section_addr(section, addr);
2442 /* RAM case */
2443 ptr = qemu_get_ram_ptr(addr1);
2444 switch (endian) {
2445 case DEVICE_LITTLE_ENDIAN:
2446 stl_le_p(ptr, val);
2447 break;
2448 case DEVICE_BIG_ENDIAN:
2449 stl_be_p(ptr, val);
2450 break;
2451 default:
2452 stl_p(ptr, val);
2453 break;
2454 }
2455 invalidate_and_set_dirty(addr1, 4);
2456 }
2457 }
2458
2459 void stl_phys(hwaddr addr, uint32_t val)
2460 {
2461 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2462 }
2463
2464 void stl_le_phys(hwaddr addr, uint32_t val)
2465 {
2466 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2467 }
2468
2469 void stl_be_phys(hwaddr addr, uint32_t val)
2470 {
2471 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2472 }
2473
2474 /* XXX: optimize */
2475 void stb_phys(hwaddr addr, uint32_t val)
2476 {
2477 uint8_t v = val;
2478 cpu_physical_memory_write(addr, &v, 1);
2479 }
2480
2481 /* warning: addr must be aligned */
2482 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2483 enum device_endian endian)
2484 {
2485 uint8_t *ptr;
2486 MemoryRegionSection *section;
2487
2488 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2489
2490 if (!memory_region_is_ram(section->mr) || section->readonly) {
2491 addr = memory_region_section_addr(section, addr);
2492 if (memory_region_is_ram(section->mr)) {
2493 section = &phys_sections[phys_section_rom];
2494 }
2495 #if defined(TARGET_WORDS_BIGENDIAN)
2496 if (endian == DEVICE_LITTLE_ENDIAN) {
2497 val = bswap16(val);
2498 }
2499 #else
2500 if (endian == DEVICE_BIG_ENDIAN) {
2501 val = bswap16(val);
2502 }
2503 #endif
2504 io_mem_write(section->mr, addr, val, 2);
2505 } else {
2506 unsigned long addr1;
2507 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2508 + memory_region_section_addr(section, addr);
2509 /* RAM case */
2510 ptr = qemu_get_ram_ptr(addr1);
2511 switch (endian) {
2512 case DEVICE_LITTLE_ENDIAN:
2513 stw_le_p(ptr, val);
2514 break;
2515 case DEVICE_BIG_ENDIAN:
2516 stw_be_p(ptr, val);
2517 break;
2518 default:
2519 stw_p(ptr, val);
2520 break;
2521 }
2522 invalidate_and_set_dirty(addr1, 2);
2523 }
2524 }
2525
2526 void stw_phys(hwaddr addr, uint32_t val)
2527 {
2528 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2529 }
2530
2531 void stw_le_phys(hwaddr addr, uint32_t val)
2532 {
2533 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2534 }
2535
2536 void stw_be_phys(hwaddr addr, uint32_t val)
2537 {
2538 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2539 }
2540
2541 /* XXX: optimize */
2542 void stq_phys(hwaddr addr, uint64_t val)
2543 {
2544 val = tswap64(val);
2545 cpu_physical_memory_write(addr, &val, 8);
2546 }
2547
2548 void stq_le_phys(hwaddr addr, uint64_t val)
2549 {
2550 val = cpu_to_le64(val);
2551 cpu_physical_memory_write(addr, &val, 8);
2552 }
2553
2554 void stq_be_phys(hwaddr addr, uint64_t val)
2555 {
2556 val = cpu_to_be64(val);
2557 cpu_physical_memory_write(addr, &val, 8);
2558 }
2559
2560 /* virtual memory access for debug (includes writing to ROM) */
2561 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2562 uint8_t *buf, int len, int is_write)
2563 {
2564 int l;
2565 hwaddr phys_addr;
2566 target_ulong page;
2567
2568 while (len > 0) {
2569 page = addr & TARGET_PAGE_MASK;
2570 phys_addr = cpu_get_phys_page_debug(env, page);
2571 /* if no physical page mapped, return an error */
2572 if (phys_addr == -1)
2573 return -1;
2574 l = (page + TARGET_PAGE_SIZE) - addr;
2575 if (l > len)
2576 l = len;
2577 phys_addr += (addr & ~TARGET_PAGE_MASK);
2578 if (is_write)
2579 cpu_physical_memory_write_rom(phys_addr, buf, l);
2580 else
2581 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2582 len -= l;
2583 buf += l;
2584 addr += l;
2585 }
2586 return 0;
2587 }
2588 #endif
2589
2590 #if !defined(CONFIG_USER_ONLY)
2591
2592 /*
2593 * A helper function for the _utterly broken_ virtio device model to find out if
2594 * it's running on a big endian machine. Don't do this at home kids!
2595 */
2596 bool virtio_is_big_endian(void);
2597 bool virtio_is_big_endian(void)
2598 {
2599 #if defined(TARGET_WORDS_BIGENDIAN)
2600 return true;
2601 #else
2602 return false;
2603 #endif
2604 }
2605
2606 #endif
2607
2608 #ifndef CONFIG_USER_ONLY
2609 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2610 {
2611 MemoryRegionSection *section;
2612
2613 section = phys_page_find(address_space_memory.dispatch,
2614 phys_addr >> TARGET_PAGE_BITS);
2615
2616 return !(memory_region_is_ram(section->mr) ||
2617 memory_region_is_romd(section->mr));
2618 }
2619 #endif
Qemu with PowerPC KVM improvements