qapi: Improve specificity of type/member descriptions
[qemu/armbru.git] / accel / tcg / cputlb.c
blobe984a98dc466a136ccfb93c6fd15fdd7cba24c11
1 /*
2 * Common CPU TLB handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/main-loop.h"
22 #include "hw/core/tcg-cpu-ops.h"
23 #include "exec/exec-all.h"
24 #include "exec/memory.h"
25 #include "exec/cpu_ldst.h"
26 #include "exec/cputlb.h"
27 #include "exec/memory-internal.h"
28 #include "exec/ram_addr.h"
29 #include "tcg/tcg.h"
30 #include "qemu/error-report.h"
31 #include "exec/log.h"
32 #include "exec/helper-proto.h"
33 #include "qemu/atomic.h"
34 #include "qemu/atomic128.h"
35 #include "exec/translate-all.h"
36 #include "trace.h"
37 #include "tb-hash.h"
38 #include "internal.h"
39 #ifdef CONFIG_PLUGIN
40 #include "qemu/plugin-memory.h"
41 #endif
42 #include "tcg/tcg-ldst.h"
44 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
45 /* #define DEBUG_TLB */
46 /* #define DEBUG_TLB_LOG */
48 #ifdef DEBUG_TLB
49 # define DEBUG_TLB_GATE 1
50 # ifdef DEBUG_TLB_LOG
51 # define DEBUG_TLB_LOG_GATE 1
52 # else
53 # define DEBUG_TLB_LOG_GATE 0
54 # endif
55 #else
56 # define DEBUG_TLB_GATE 0
57 # define DEBUG_TLB_LOG_GATE 0
58 #endif
60 #define tlb_debug(fmt, ...) do { \
61 if (DEBUG_TLB_LOG_GATE) { \
62 qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
63 ## __VA_ARGS__); \
64 } else if (DEBUG_TLB_GATE) { \
65 fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
66 } \
67 } while (0)
69 #define assert_cpu_is_self(cpu) do { \
70 if (DEBUG_TLB_GATE) { \
71 g_assert(!(cpu)->created || qemu_cpu_is_self(cpu)); \
72 } \
73 } while (0)
75 /* run_on_cpu_data.target_ptr should always be big enough for a
76 * target_ulong even on 32 bit builds */
77 QEMU_BUILD_BUG_ON(sizeof(target_ulong) > sizeof(run_on_cpu_data));
79 /* We currently can't handle more than 16 bits in the MMUIDX bitmask.
81 QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
82 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
84 static inline size_t tlb_n_entries(CPUTLBDescFast *fast)
86 return (fast->mask >> CPU_TLB_ENTRY_BITS) + 1;
89 static inline size_t sizeof_tlb(CPUTLBDescFast *fast)
91 return fast->mask + (1 << CPU_TLB_ENTRY_BITS);
94 static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns,
95 size_t max_entries)
97 desc->window_begin_ns = ns;
98 desc->window_max_entries = max_entries;
101 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
103 CPUJumpCache *jc = cpu->tb_jmp_cache;
104 int i, i0;
106 if (unlikely(!jc)) {
107 return;
110 i0 = tb_jmp_cache_hash_page(page_addr);
111 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
112 qatomic_set(&jc->array[i0 + i].tb, NULL);
117 * tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
118 * @desc: The CPUTLBDesc portion of the TLB
119 * @fast: The CPUTLBDescFast portion of the same TLB
121 * Called with tlb_lock_held.
123 * We have two main constraints when resizing a TLB: (1) we only resize it
124 * on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
125 * the array or unnecessarily flushing it), which means we do not control how
126 * frequently the resizing can occur; (2) we don't have access to the guest's
127 * future scheduling decisions, and therefore have to decide the magnitude of
128 * the resize based on past observations.
130 * In general, a memory-hungry process can benefit greatly from an appropriately
131 * sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
132 * we just have to make the TLB as large as possible; while an oversized TLB
133 * results in minimal TLB miss rates, it also takes longer to be flushed
134 * (flushes can be _very_ frequent), and the reduced locality can also hurt
135 * performance.
137 * To achieve near-optimal performance for all kinds of workloads, we:
139 * 1. Aggressively increase the size of the TLB when the use rate of the
140 * TLB being flushed is high, since it is likely that in the near future this
141 * memory-hungry process will execute again, and its memory hungriness will
142 * probably be similar.
144 * 2. Slowly reduce the size of the TLB as the use rate declines over a
145 * reasonably large time window. The rationale is that if in such a time window
146 * we have not observed a high TLB use rate, it is likely that we won't observe
147 * it in the near future. In that case, once a time window expires we downsize
148 * the TLB to match the maximum use rate observed in the window.
150 * 3. Try to keep the maximum use rate in a time window in the 30-70% range,
151 * since in that range performance is likely near-optimal. Recall that the TLB
152 * is direct mapped, so we want the use rate to be low (or at least not too
153 * high), since otherwise we are likely to have a significant amount of
154 * conflict misses.
156 static void tlb_mmu_resize_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast,
157 int64_t now)
159 size_t old_size = tlb_n_entries(fast);
160 size_t rate;
161 size_t new_size = old_size;
162 int64_t window_len_ms = 100;
163 int64_t window_len_ns = window_len_ms * 1000 * 1000;
164 bool window_expired = now > desc->window_begin_ns + window_len_ns;
166 if (desc->n_used_entries > desc->window_max_entries) {
167 desc->window_max_entries = desc->n_used_entries;
169 rate = desc->window_max_entries * 100 / old_size;
171 if (rate > 70) {
172 new_size = MIN(old_size << 1, 1 << CPU_TLB_DYN_MAX_BITS);
173 } else if (rate < 30 && window_expired) {
174 size_t ceil = pow2ceil(desc->window_max_entries);
175 size_t expected_rate = desc->window_max_entries * 100 / ceil;
178 * Avoid undersizing when the max number of entries seen is just below
179 * a pow2. For instance, if max_entries == 1025, the expected use rate
180 * would be 1025/2048==50%. However, if max_entries == 1023, we'd get
181 * 1023/1024==99.9% use rate, so we'd likely end up doubling the size
182 * later. Thus, make sure that the expected use rate remains below 70%.
183 * (and since we double the size, that means the lowest rate we'd
184 * expect to get is 35%, which is still in the 30-70% range where
185 * we consider that the size is appropriate.)
187 if (expected_rate > 70) {
188 ceil *= 2;
190 new_size = MAX(ceil, 1 << CPU_TLB_DYN_MIN_BITS);
193 if (new_size == old_size) {
194 if (window_expired) {
195 tlb_window_reset(desc, now, desc->n_used_entries);
197 return;
200 g_free(fast->table);
201 g_free(desc->fulltlb);
203 tlb_window_reset(desc, now, 0);
204 /* desc->n_used_entries is cleared by the caller */
205 fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
206 fast->table = g_try_new(CPUTLBEntry, new_size);
207 desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
210 * If the allocations fail, try smaller sizes. We just freed some
211 * memory, so going back to half of new_size has a good chance of working.
212 * Increased memory pressure elsewhere in the system might cause the
213 * allocations to fail though, so we progressively reduce the allocation
214 * size, aborting if we cannot even allocate the smallest TLB we support.
216 while (fast->table == NULL || desc->fulltlb == NULL) {
217 if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) {
218 error_report("%s: %s", __func__, strerror(errno));
219 abort();
221 new_size = MAX(new_size >> 1, 1 << CPU_TLB_DYN_MIN_BITS);
222 fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
224 g_free(fast->table);
225 g_free(desc->fulltlb);
226 fast->table = g_try_new(CPUTLBEntry, new_size);
227 desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
231 static void tlb_mmu_flush_locked(CPUTLBDesc *desc, CPUTLBDescFast *fast)
233 desc->n_used_entries = 0;
234 desc->large_page_addr = -1;
235 desc->large_page_mask = -1;
236 desc->vindex = 0;
237 memset(fast->table, -1, sizeof_tlb(fast));
238 memset(desc->vtable, -1, sizeof(desc->vtable));
241 static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx,
242 int64_t now)
244 CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx];
245 CPUTLBDescFast *fast = &env_tlb(env)->f[mmu_idx];
247 tlb_mmu_resize_locked(desc, fast, now);
248 tlb_mmu_flush_locked(desc, fast);
251 static void tlb_mmu_init(CPUTLBDesc *desc, CPUTLBDescFast *fast, int64_t now)
253 size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS;
255 tlb_window_reset(desc, now, 0);
256 desc->n_used_entries = 0;
257 fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS;
258 fast->table = g_new(CPUTLBEntry, n_entries);
259 desc->fulltlb = g_new(CPUTLBEntryFull, n_entries);
260 tlb_mmu_flush_locked(desc, fast);
263 static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx)
265 env_tlb(env)->d[mmu_idx].n_used_entries++;
268 static inline void tlb_n_used_entries_dec(CPUArchState *env, uintptr_t mmu_idx)
270 env_tlb(env)->d[mmu_idx].n_used_entries--;
273 void tlb_init(CPUState *cpu)
275 CPUArchState *env = cpu->env_ptr;
276 int64_t now = get_clock_realtime();
277 int i;
279 qemu_spin_init(&env_tlb(env)->c.lock);
281 /* All tlbs are initialized flushed. */
282 env_tlb(env)->c.dirty = 0;
284 for (i = 0; i < NB_MMU_MODES; i++) {
285 tlb_mmu_init(&env_tlb(env)->d[i], &env_tlb(env)->f[i], now);
289 void tlb_destroy(CPUState *cpu)
291 CPUArchState *env = cpu->env_ptr;
292 int i;
294 qemu_spin_destroy(&env_tlb(env)->c.lock);
295 for (i = 0; i < NB_MMU_MODES; i++) {
296 CPUTLBDesc *desc = &env_tlb(env)->d[i];
297 CPUTLBDescFast *fast = &env_tlb(env)->f[i];
299 g_free(fast->table);
300 g_free(desc->fulltlb);
304 /* flush_all_helper: run fn across all cpus
306 * If the wait flag is set then the src cpu's helper will be queued as
307 * "safe" work and the loop exited creating a synchronisation point
308 * where all queued work will be finished before execution starts
309 * again.
311 static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
312 run_on_cpu_data d)
314 CPUState *cpu;
316 CPU_FOREACH(cpu) {
317 if (cpu != src) {
318 async_run_on_cpu(cpu, fn, d);
323 void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide)
325 CPUState *cpu;
326 size_t full = 0, part = 0, elide = 0;
328 CPU_FOREACH(cpu) {
329 CPUArchState *env = cpu->env_ptr;
331 full += qatomic_read(&env_tlb(env)->c.full_flush_count);
332 part += qatomic_read(&env_tlb(env)->c.part_flush_count);
333 elide += qatomic_read(&env_tlb(env)->c.elide_flush_count);
335 *pfull = full;
336 *ppart = part;
337 *pelide = elide;
340 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
342 CPUArchState *env = cpu->env_ptr;
343 uint16_t asked = data.host_int;
344 uint16_t all_dirty, work, to_clean;
345 int64_t now = get_clock_realtime();
347 assert_cpu_is_self(cpu);
349 tlb_debug("mmu_idx:0x%04" PRIx16 "\n", asked);
351 qemu_spin_lock(&env_tlb(env)->c.lock);
353 all_dirty = env_tlb(env)->c.dirty;
354 to_clean = asked & all_dirty;
355 all_dirty &= ~to_clean;
356 env_tlb(env)->c.dirty = all_dirty;
358 for (work = to_clean; work != 0; work &= work - 1) {
359 int mmu_idx = ctz32(work);
360 tlb_flush_one_mmuidx_locked(env, mmu_idx, now);
363 qemu_spin_unlock(&env_tlb(env)->c.lock);
365 tcg_flush_jmp_cache(cpu);
367 if (to_clean == ALL_MMUIDX_BITS) {
368 qatomic_set(&env_tlb(env)->c.full_flush_count,
369 env_tlb(env)->c.full_flush_count + 1);
370 } else {
371 qatomic_set(&env_tlb(env)->c.part_flush_count,
372 env_tlb(env)->c.part_flush_count + ctpop16(to_clean));
373 if (to_clean != asked) {
374 qatomic_set(&env_tlb(env)->c.elide_flush_count,
375 env_tlb(env)->c.elide_flush_count +
376 ctpop16(asked & ~to_clean));
381 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
383 tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
385 if (cpu->created && !qemu_cpu_is_self(cpu)) {
386 async_run_on_cpu(cpu, tlb_flush_by_mmuidx_async_work,
387 RUN_ON_CPU_HOST_INT(idxmap));
388 } else {
389 tlb_flush_by_mmuidx_async_work(cpu, RUN_ON_CPU_HOST_INT(idxmap));
393 void tlb_flush(CPUState *cpu)
395 tlb_flush_by_mmuidx(cpu, ALL_MMUIDX_BITS);
398 void tlb_flush_by_mmuidx_all_cpus(CPUState *src_cpu, uint16_t idxmap)
400 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
402 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
404 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
405 fn(src_cpu, RUN_ON_CPU_HOST_INT(idxmap));
408 void tlb_flush_all_cpus(CPUState *src_cpu)
410 tlb_flush_by_mmuidx_all_cpus(src_cpu, ALL_MMUIDX_BITS);
413 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap)
415 const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
417 tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
419 flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
420 async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
423 void tlb_flush_all_cpus_synced(CPUState *src_cpu)
425 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, ALL_MMUIDX_BITS);
428 static bool tlb_hit_page_mask_anyprot(CPUTLBEntry *tlb_entry,
429 target_ulong page, target_ulong mask)
431 page &= mask;
432 mask &= TARGET_PAGE_MASK | TLB_INVALID_MASK;
434 return (page == (tlb_entry->addr_read & mask) ||
435 page == (tlb_addr_write(tlb_entry) & mask) ||
436 page == (tlb_entry->addr_code & mask));
439 static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry,
440 target_ulong page)
442 return tlb_hit_page_mask_anyprot(tlb_entry, page, -1);
446 * tlb_entry_is_empty - return true if the entry is not in use
447 * @te: pointer to CPUTLBEntry
449 static inline bool tlb_entry_is_empty(const CPUTLBEntry *te)
451 return te->addr_read == -1 && te->addr_write == -1 && te->addr_code == -1;
454 /* Called with tlb_c.lock held */
455 static bool tlb_flush_entry_mask_locked(CPUTLBEntry *tlb_entry,
456 target_ulong page,
457 target_ulong mask)
459 if (tlb_hit_page_mask_anyprot(tlb_entry, page, mask)) {
460 memset(tlb_entry, -1, sizeof(*tlb_entry));
461 return true;
463 return false;
466 static inline bool tlb_flush_entry_locked(CPUTLBEntry *tlb_entry,
467 target_ulong page)
469 return tlb_flush_entry_mask_locked(tlb_entry, page, -1);
472 /* Called with tlb_c.lock held */
473 static void tlb_flush_vtlb_page_mask_locked(CPUArchState *env, int mmu_idx,
474 target_ulong page,
475 target_ulong mask)
477 CPUTLBDesc *d = &env_tlb(env)->d[mmu_idx];
478 int k;
480 assert_cpu_is_self(env_cpu(env));
481 for (k = 0; k < CPU_VTLB_SIZE; k++) {
482 if (tlb_flush_entry_mask_locked(&d->vtable[k], page, mask)) {
483 tlb_n_used_entries_dec(env, mmu_idx);
488 static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
489 target_ulong page)
491 tlb_flush_vtlb_page_mask_locked(env, mmu_idx, page, -1);
494 static void tlb_flush_page_locked(CPUArchState *env, int midx,
495 target_ulong page)
497 target_ulong lp_addr = env_tlb(env)->d[midx].large_page_addr;
498 target_ulong lp_mask = env_tlb(env)->d[midx].large_page_mask;
500 /* Check if we need to flush due to large pages. */
501 if ((page & lp_mask) == lp_addr) {
502 tlb_debug("forcing full flush midx %d ("
503 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
504 midx, lp_addr, lp_mask);
505 tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
506 } else {
507 if (tlb_flush_entry_locked(tlb_entry(env, midx, page), page)) {
508 tlb_n_used_entries_dec(env, midx);
510 tlb_flush_vtlb_page_locked(env, midx, page);
515 * tlb_flush_page_by_mmuidx_async_0:
516 * @cpu: cpu on which to flush
517 * @addr: page of virtual address to flush
518 * @idxmap: set of mmu_idx to flush
520 * Helper for tlb_flush_page_by_mmuidx and friends, flush one page
521 * at @addr from the tlbs indicated by @idxmap from @cpu.
523 static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
524 target_ulong addr,
525 uint16_t idxmap)
527 CPUArchState *env = cpu->env_ptr;
528 int mmu_idx;
530 assert_cpu_is_self(cpu);
532 tlb_debug("page addr:" TARGET_FMT_lx " mmu_map:0x%x\n", addr, idxmap);
534 qemu_spin_lock(&env_tlb(env)->c.lock);
535 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
536 if ((idxmap >> mmu_idx) & 1) {
537 tlb_flush_page_locked(env, mmu_idx, addr);
540 qemu_spin_unlock(&env_tlb(env)->c.lock);
543 * Discard jump cache entries for any tb which might potentially
544 * overlap the flushed page, which includes the previous.
546 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
547 tb_jmp_cache_clear_page(cpu, addr);
551 * tlb_flush_page_by_mmuidx_async_1:
552 * @cpu: cpu on which to flush
553 * @data: encoded addr + idxmap
555 * Helper for tlb_flush_page_by_mmuidx and friends, called through
556 * async_run_on_cpu. The idxmap parameter is encoded in the page
557 * offset of the target_ptr field. This limits the set of mmu_idx
558 * that can be passed via this method.
560 static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu,
561 run_on_cpu_data data)
563 target_ulong addr_and_idxmap = (target_ulong) data.target_ptr;
564 target_ulong addr = addr_and_idxmap & TARGET_PAGE_MASK;
565 uint16_t idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK;
567 tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
570 typedef struct {
571 target_ulong addr;
572 uint16_t idxmap;
573 } TLBFlushPageByMMUIdxData;
576 * tlb_flush_page_by_mmuidx_async_2:
577 * @cpu: cpu on which to flush
578 * @data: allocated addr + idxmap
580 * Helper for tlb_flush_page_by_mmuidx and friends, called through
581 * async_run_on_cpu. The addr+idxmap parameters are stored in a
582 * TLBFlushPageByMMUIdxData structure that has been allocated
583 * specifically for this helper. Free the structure when done.
585 static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu,
586 run_on_cpu_data data)
588 TLBFlushPageByMMUIdxData *d = data.host_ptr;
590 tlb_flush_page_by_mmuidx_async_0(cpu, d->addr, d->idxmap);
591 g_free(d);
594 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, uint16_t idxmap)
596 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%" PRIx16 "\n", addr, idxmap);
598 /* This should already be page aligned */
599 addr &= TARGET_PAGE_MASK;
601 if (qemu_cpu_is_self(cpu)) {
602 tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
603 } else if (idxmap < TARGET_PAGE_SIZE) {
605 * Most targets have only a few mmu_idx. In the case where
606 * we can stuff idxmap into the low TARGET_PAGE_BITS, avoid
607 * allocating memory for this operation.
609 async_run_on_cpu(cpu, tlb_flush_page_by_mmuidx_async_1,
610 RUN_ON_CPU_TARGET_PTR(addr | idxmap));
611 } else {
612 TLBFlushPageByMMUIdxData *d = g_new(TLBFlushPageByMMUIdxData, 1);
614 /* Otherwise allocate a structure, freed by the worker. */
615 d->addr = addr;
616 d->idxmap = idxmap;
617 async_run_on_cpu(cpu, tlb_flush_page_by_mmuidx_async_2,
618 RUN_ON_CPU_HOST_PTR(d));
622 void tlb_flush_page(CPUState *cpu, target_ulong addr)
624 tlb_flush_page_by_mmuidx(cpu, addr, ALL_MMUIDX_BITS);
627 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
628 uint16_t idxmap)
630 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
632 /* This should already be page aligned */
633 addr &= TARGET_PAGE_MASK;
636 * Allocate memory to hold addr+idxmap only when needed.
637 * See tlb_flush_page_by_mmuidx for details.
639 if (idxmap < TARGET_PAGE_SIZE) {
640 flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1,
641 RUN_ON_CPU_TARGET_PTR(addr | idxmap));
642 } else {
643 CPUState *dst_cpu;
645 /* Allocate a separate data block for each destination cpu. */
646 CPU_FOREACH(dst_cpu) {
647 if (dst_cpu != src_cpu) {
648 TLBFlushPageByMMUIdxData *d
649 = g_new(TLBFlushPageByMMUIdxData, 1);
651 d->addr = addr;
652 d->idxmap = idxmap;
653 async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2,
654 RUN_ON_CPU_HOST_PTR(d));
659 tlb_flush_page_by_mmuidx_async_0(src_cpu, addr, idxmap);
662 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
664 tlb_flush_page_by_mmuidx_all_cpus(src, addr, ALL_MMUIDX_BITS);
667 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
668 target_ulong addr,
669 uint16_t idxmap)
671 tlb_debug("addr: "TARGET_FMT_lx" mmu_idx:%"PRIx16"\n", addr, idxmap);
673 /* This should already be page aligned */
674 addr &= TARGET_PAGE_MASK;
677 * Allocate memory to hold addr+idxmap only when needed.
678 * See tlb_flush_page_by_mmuidx for details.
680 if (idxmap < TARGET_PAGE_SIZE) {
681 flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1,
682 RUN_ON_CPU_TARGET_PTR(addr | idxmap));
683 async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_1,
684 RUN_ON_CPU_TARGET_PTR(addr | idxmap));
685 } else {
686 CPUState *dst_cpu;
687 TLBFlushPageByMMUIdxData *d;
689 /* Allocate a separate data block for each destination cpu. */
690 CPU_FOREACH(dst_cpu) {
691 if (dst_cpu != src_cpu) {
692 d = g_new(TLBFlushPageByMMUIdxData, 1);
693 d->addr = addr;
694 d->idxmap = idxmap;
695 async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2,
696 RUN_ON_CPU_HOST_PTR(d));
700 d = g_new(TLBFlushPageByMMUIdxData, 1);
701 d->addr = addr;
702 d->idxmap = idxmap;
703 async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_2,
704 RUN_ON_CPU_HOST_PTR(d));
708 void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr)
710 tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
713 static void tlb_flush_range_locked(CPUArchState *env, int midx,
714 target_ulong addr, target_ulong len,
715 unsigned bits)
717 CPUTLBDesc *d = &env_tlb(env)->d[midx];
718 CPUTLBDescFast *f = &env_tlb(env)->f[midx];
719 target_ulong mask = MAKE_64BIT_MASK(0, bits);
722 * If @bits is smaller than the tlb size, there may be multiple entries
723 * within the TLB; otherwise all addresses that match under @mask hit
724 * the same TLB entry.
725 * TODO: Perhaps allow bits to be a few bits less than the size.
726 * For now, just flush the entire TLB.
728 * If @len is larger than the tlb size, then it will take longer to
729 * test all of the entries in the TLB than it will to flush it all.
731 if (mask < f->mask || len > f->mask) {
732 tlb_debug("forcing full flush midx %d ("
733 TARGET_FMT_lx "/" TARGET_FMT_lx "+" TARGET_FMT_lx ")\n",
734 midx, addr, mask, len);
735 tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
736 return;
740 * Check if we need to flush due to large pages.
741 * Because large_page_mask contains all 1's from the msb,
742 * we only need to test the end of the range.
744 if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
745 tlb_debug("forcing full flush midx %d ("
746 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
747 midx, d->large_page_addr, d->large_page_mask);
748 tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
749 return;
752 for (target_ulong i = 0; i < len; i += TARGET_PAGE_SIZE) {
753 target_ulong page = addr + i;
754 CPUTLBEntry *entry = tlb_entry(env, midx, page);
756 if (tlb_flush_entry_mask_locked(entry, page, mask)) {
757 tlb_n_used_entries_dec(env, midx);
759 tlb_flush_vtlb_page_mask_locked(env, midx, page, mask);
763 typedef struct {
764 target_ulong addr;
765 target_ulong len;
766 uint16_t idxmap;
767 uint16_t bits;
768 } TLBFlushRangeData;
770 static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
771 TLBFlushRangeData d)
773 CPUArchState *env = cpu->env_ptr;
774 int mmu_idx;
776 assert_cpu_is_self(cpu);
778 tlb_debug("range:" TARGET_FMT_lx "/%u+" TARGET_FMT_lx " mmu_map:0x%x\n",
779 d.addr, d.bits, d.len, d.idxmap);
781 qemu_spin_lock(&env_tlb(env)->c.lock);
782 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
783 if ((d.idxmap >> mmu_idx) & 1) {
784 tlb_flush_range_locked(env, mmu_idx, d.addr, d.len, d.bits);
787 qemu_spin_unlock(&env_tlb(env)->c.lock);
790 * If the length is larger than the jump cache size, then it will take
791 * longer to clear each entry individually than it will to clear it all.
793 if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) {
794 tcg_flush_jmp_cache(cpu);
795 return;
799 * Discard jump cache entries for any tb which might potentially
800 * overlap the flushed pages, which includes the previous.
802 d.addr -= TARGET_PAGE_SIZE;
803 for (target_ulong i = 0, n = d.len / TARGET_PAGE_SIZE + 1; i < n; i++) {
804 tb_jmp_cache_clear_page(cpu, d.addr);
805 d.addr += TARGET_PAGE_SIZE;
809 static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu,
810 run_on_cpu_data data)
812 TLBFlushRangeData *d = data.host_ptr;
813 tlb_flush_range_by_mmuidx_async_0(cpu, *d);
814 g_free(d);
817 void tlb_flush_range_by_mmuidx(CPUState *cpu, target_ulong addr,
818 target_ulong len, uint16_t idxmap,
819 unsigned bits)
821 TLBFlushRangeData d;
824 * If all bits are significant, and len is small,
825 * this devolves to tlb_flush_page.
827 if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
828 tlb_flush_page_by_mmuidx(cpu, addr, idxmap);
829 return;
831 /* If no page bits are significant, this devolves to tlb_flush. */
832 if (bits < TARGET_PAGE_BITS) {
833 tlb_flush_by_mmuidx(cpu, idxmap);
834 return;
837 /* This should already be page aligned */
838 d.addr = addr & TARGET_PAGE_MASK;
839 d.len = len;
840 d.idxmap = idxmap;
841 d.bits = bits;
843 if (qemu_cpu_is_self(cpu)) {
844 tlb_flush_range_by_mmuidx_async_0(cpu, d);
845 } else {
846 /* Otherwise allocate a structure, freed by the worker. */
847 TLBFlushRangeData *p = g_memdup(&d, sizeof(d));
848 async_run_on_cpu(cpu, tlb_flush_range_by_mmuidx_async_1,
849 RUN_ON_CPU_HOST_PTR(p));
853 void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, target_ulong addr,
854 uint16_t idxmap, unsigned bits)
856 tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits);
859 void tlb_flush_range_by_mmuidx_all_cpus(CPUState *src_cpu,
860 target_ulong addr, target_ulong len,
861 uint16_t idxmap, unsigned bits)
863 TLBFlushRangeData d;
864 CPUState *dst_cpu;
867 * If all bits are significant, and len is small,
868 * this devolves to tlb_flush_page.
870 if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
871 tlb_flush_page_by_mmuidx_all_cpus(src_cpu, addr, idxmap);
872 return;
874 /* If no page bits are significant, this devolves to tlb_flush. */
875 if (bits < TARGET_PAGE_BITS) {
876 tlb_flush_by_mmuidx_all_cpus(src_cpu, idxmap);
877 return;
880 /* This should already be page aligned */
881 d.addr = addr & TARGET_PAGE_MASK;
882 d.len = len;
883 d.idxmap = idxmap;
884 d.bits = bits;
886 /* Allocate a separate data block for each destination cpu. */
887 CPU_FOREACH(dst_cpu) {
888 if (dst_cpu != src_cpu) {
889 TLBFlushRangeData *p = g_memdup(&d, sizeof(d));
890 async_run_on_cpu(dst_cpu,
891 tlb_flush_range_by_mmuidx_async_1,
892 RUN_ON_CPU_HOST_PTR(p));
896 tlb_flush_range_by_mmuidx_async_0(src_cpu, d);
899 void tlb_flush_page_bits_by_mmuidx_all_cpus(CPUState *src_cpu,
900 target_ulong addr,
901 uint16_t idxmap, unsigned bits)
903 tlb_flush_range_by_mmuidx_all_cpus(src_cpu, addr, TARGET_PAGE_SIZE,
904 idxmap, bits);
907 void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
908 target_ulong addr,
909 target_ulong len,
910 uint16_t idxmap,
911 unsigned bits)
913 TLBFlushRangeData d, *p;
914 CPUState *dst_cpu;
917 * If all bits are significant, and len is small,
918 * this devolves to tlb_flush_page.
920 if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
921 tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap);
922 return;
924 /* If no page bits are significant, this devolves to tlb_flush. */
925 if (bits < TARGET_PAGE_BITS) {
926 tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap);
927 return;
930 /* This should already be page aligned */
931 d.addr = addr & TARGET_PAGE_MASK;
932 d.len = len;
933 d.idxmap = idxmap;
934 d.bits = bits;
936 /* Allocate a separate data block for each destination cpu. */
937 CPU_FOREACH(dst_cpu) {
938 if (dst_cpu != src_cpu) {
939 p = g_memdup(&d, sizeof(d));
940 async_run_on_cpu(dst_cpu, tlb_flush_range_by_mmuidx_async_1,
941 RUN_ON_CPU_HOST_PTR(p));
945 p = g_memdup(&d, sizeof(d));
946 async_safe_run_on_cpu(src_cpu, tlb_flush_range_by_mmuidx_async_1,
947 RUN_ON_CPU_HOST_PTR(p));
950 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
951 target_ulong addr,
952 uint16_t idxmap,
953 unsigned bits)
955 tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE,
956 idxmap, bits);
959 /* update the TLBs so that writes to code in the virtual page 'addr'
960 can be detected */
961 void tlb_protect_code(ram_addr_t ram_addr)
963 cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK,
964 TARGET_PAGE_SIZE,
965 DIRTY_MEMORY_CODE);
968 /* update the TLB so that writes in physical page 'phys_addr' are no longer
969 tested for self modifying code */
970 void tlb_unprotect_code(ram_addr_t ram_addr)
972 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
977 * Dirty write flag handling
979 * When the TCG code writes to a location it looks up the address in
980 * the TLB and uses that data to compute the final address. If any of
981 * the lower bits of the address are set then the slow path is forced.
982 * There are a number of reasons to do this but for normal RAM the
983 * most usual is detecting writes to code regions which may invalidate
984 * generated code.
986 * Other vCPUs might be reading their TLBs during guest execution, so we update
987 * te->addr_write with qatomic_set. We don't need to worry about this for
988 * oversized guests as MTTCG is disabled for them.
990 * Called with tlb_c.lock held.
992 static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
993 uintptr_t start, uintptr_t length)
995 uintptr_t addr = tlb_entry->addr_write;
997 if ((addr & (TLB_INVALID_MASK | TLB_MMIO |
998 TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {
999 addr &= TARGET_PAGE_MASK;
1000 addr += tlb_entry->addend;
1001 if ((addr - start) < length) {
1002 #if TCG_OVERSIZED_GUEST
1003 tlb_entry->addr_write |= TLB_NOTDIRTY;
1004 #else
1005 qatomic_set(&tlb_entry->addr_write,
1006 tlb_entry->addr_write | TLB_NOTDIRTY);
1007 #endif
1013 * Called with tlb_c.lock held.
1014 * Called only from the vCPU context, i.e. the TLB's owner thread.
1016 static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
1018 *d = *s;
1021 /* This is a cross vCPU call (i.e. another vCPU resetting the flags of
1022 * the target vCPU).
1023 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
1024 * thing actually updated is the target TLB entry ->addr_write flags.
1026 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
1028 CPUArchState *env;
1030 int mmu_idx;
1032 env = cpu->env_ptr;
1033 qemu_spin_lock(&env_tlb(env)->c.lock);
1034 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1035 unsigned int i;
1036 unsigned int n = tlb_n_entries(&env_tlb(env)->f[mmu_idx]);
1038 for (i = 0; i < n; i++) {
1039 tlb_reset_dirty_range_locked(&env_tlb(env)->f[mmu_idx].table[i],
1040 start1, length);
1043 for (i = 0; i < CPU_VTLB_SIZE; i++) {
1044 tlb_reset_dirty_range_locked(&env_tlb(env)->d[mmu_idx].vtable[i],
1045 start1, length);
1048 qemu_spin_unlock(&env_tlb(env)->c.lock);
1051 /* Called with tlb_c.lock held */
1052 static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
1053 target_ulong vaddr)
1055 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
1056 tlb_entry->addr_write = vaddr;
1060 /* update the TLB corresponding to virtual page vaddr
1061 so that it is no longer dirty */
1062 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
1064 CPUArchState *env = cpu->env_ptr;
1065 int mmu_idx;
1067 assert_cpu_is_self(cpu);
1069 vaddr &= TARGET_PAGE_MASK;
1070 qemu_spin_lock(&env_tlb(env)->c.lock);
1071 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1072 tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr);
1075 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1076 int k;
1077 for (k = 0; k < CPU_VTLB_SIZE; k++) {
1078 tlb_set_dirty1_locked(&env_tlb(env)->d[mmu_idx].vtable[k], vaddr);
1081 qemu_spin_unlock(&env_tlb(env)->c.lock);
1084 /* Our TLB does not support large pages, so remember the area covered by
1085 large pages and trigger a full TLB flush if these are invalidated. */
1086 static void tlb_add_large_page(CPUArchState *env, int mmu_idx,
1087 target_ulong vaddr, target_ulong size)
1089 target_ulong lp_addr = env_tlb(env)->d[mmu_idx].large_page_addr;
1090 target_ulong lp_mask = ~(size - 1);
1092 if (lp_addr == (target_ulong)-1) {
1093 /* No previous large page. */
1094 lp_addr = vaddr;
1095 } else {
1096 /* Extend the existing region to include the new page.
1097 This is a compromise between unnecessary flushes and
1098 the cost of maintaining a full variable size TLB. */
1099 lp_mask &= env_tlb(env)->d[mmu_idx].large_page_mask;
1100 while (((lp_addr ^ vaddr) & lp_mask) != 0) {
1101 lp_mask <<= 1;
1104 env_tlb(env)->d[mmu_idx].large_page_addr = lp_addr & lp_mask;
1105 env_tlb(env)->d[mmu_idx].large_page_mask = lp_mask;
1109 * Add a new TLB entry. At most one entry for a given virtual address
1110 * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1111 * supplied size is only used by tlb_flush_page.
1113 * Called from TCG-generated code, which is under an RCU read-side
1114 * critical section.
1116 void tlb_set_page_full(CPUState *cpu, int mmu_idx,
1117 target_ulong vaddr, CPUTLBEntryFull *full)
1119 CPUArchState *env = cpu->env_ptr;
1120 CPUTLB *tlb = env_tlb(env);
1121 CPUTLBDesc *desc = &tlb->d[mmu_idx];
1122 MemoryRegionSection *section;
1123 unsigned int index;
1124 target_ulong address;
1125 target_ulong write_address;
1126 uintptr_t addend;
1127 CPUTLBEntry *te, tn;
1128 hwaddr iotlb, xlat, sz, paddr_page;
1129 target_ulong vaddr_page;
1130 int asidx, wp_flags, prot;
1131 bool is_ram, is_romd;
1133 assert_cpu_is_self(cpu);
1135 if (full->lg_page_size <= TARGET_PAGE_BITS) {
1136 sz = TARGET_PAGE_SIZE;
1137 } else {
1138 sz = (hwaddr)1 << full->lg_page_size;
1139 tlb_add_large_page(env, mmu_idx, vaddr, sz);
1141 vaddr_page = vaddr & TARGET_PAGE_MASK;
1142 paddr_page = full->phys_addr & TARGET_PAGE_MASK;
1144 prot = full->prot;
1145 asidx = cpu_asidx_from_attrs(cpu, full->attrs);
1146 section = address_space_translate_for_iotlb(cpu, asidx, paddr_page,
1147 &xlat, &sz, full->attrs, &prot);
1148 assert(sz >= TARGET_PAGE_SIZE);
1150 tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" HWADDR_FMT_plx
1151 " prot=%x idx=%d\n",
1152 vaddr, full->phys_addr, prot, mmu_idx);
1154 address = vaddr_page;
1155 if (full->lg_page_size < TARGET_PAGE_BITS) {
1156 /* Repeat the MMU check and TLB fill on every access. */
1157 address |= TLB_INVALID_MASK;
1159 if (full->attrs.byte_swap) {
1160 address |= TLB_BSWAP;
1163 is_ram = memory_region_is_ram(section->mr);
1164 is_romd = memory_region_is_romd(section->mr);
1166 if (is_ram || is_romd) {
1167 /* RAM and ROMD both have associated host memory. */
1168 addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
1169 } else {
1170 /* I/O does not; force the host address to NULL. */
1171 addend = 0;
1174 write_address = address;
1175 if (is_ram) {
1176 iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1178 * Computing is_clean is expensive; avoid all that unless
1179 * the page is actually writable.
1181 if (prot & PAGE_WRITE) {
1182 if (section->readonly) {
1183 write_address |= TLB_DISCARD_WRITE;
1184 } else if (cpu_physical_memory_is_clean(iotlb)) {
1185 write_address |= TLB_NOTDIRTY;
1188 } else {
1189 /* I/O or ROMD */
1190 iotlb = memory_region_section_get_iotlb(cpu, section) + xlat;
1192 * Writes to romd devices must go through MMIO to enable write.
1193 * Reads to romd devices go through the ram_ptr found above,
1194 * but of course reads to I/O must go through MMIO.
1196 write_address |= TLB_MMIO;
1197 if (!is_romd) {
1198 address = write_address;
1202 wp_flags = cpu_watchpoint_address_matches(cpu, vaddr_page,
1203 TARGET_PAGE_SIZE);
1205 index = tlb_index(env, mmu_idx, vaddr_page);
1206 te = tlb_entry(env, mmu_idx, vaddr_page);
1209 * Hold the TLB lock for the rest of the function. We could acquire/release
1210 * the lock several times in the function, but it is faster to amortize the
1211 * acquisition cost by acquiring it just once. Note that this leads to
1212 * a longer critical section, but this is not a concern since the TLB lock
1213 * is unlikely to be contended.
1215 qemu_spin_lock(&tlb->c.lock);
1217 /* Note that the tlb is no longer clean. */
1218 tlb->c.dirty |= 1 << mmu_idx;
1220 /* Make sure there's no cached translation for the new page. */
1221 tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page);
1224 * Only evict the old entry to the victim tlb if it's for a
1225 * different page; otherwise just overwrite the stale data.
1227 if (!tlb_hit_page_anyprot(te, vaddr_page) && !tlb_entry_is_empty(te)) {
1228 unsigned vidx = desc->vindex++ % CPU_VTLB_SIZE;
1229 CPUTLBEntry *tv = &desc->vtable[vidx];
1231 /* Evict the old entry into the victim tlb. */
1232 copy_tlb_helper_locked(tv, te);
1233 desc->vfulltlb[vidx] = desc->fulltlb[index];
1234 tlb_n_used_entries_dec(env, mmu_idx);
1237 /* refill the tlb */
1239 * At this point iotlb contains a physical section number in the lower
1240 * TARGET_PAGE_BITS, and either
1241 * + the ram_addr_t of the page base of the target RAM (RAM)
1242 * + the offset within section->mr of the page base (I/O, ROMD)
1243 * We subtract the vaddr_page (which is page aligned and thus won't
1244 * disturb the low bits) to give an offset which can be added to the
1245 * (non-page-aligned) vaddr of the eventual memory access to get
1246 * the MemoryRegion offset for the access. Note that the vaddr we
1247 * subtract here is that of the page base, and not the same as the
1248 * vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
1250 desc->fulltlb[index] = *full;
1251 desc->fulltlb[index].xlat_section = iotlb - vaddr_page;
1252 desc->fulltlb[index].phys_addr = paddr_page;
1254 /* Now calculate the new entry */
1255 tn.addend = addend - vaddr_page;
1256 if (prot & PAGE_READ) {
1257 tn.addr_read = address;
1258 if (wp_flags & BP_MEM_READ) {
1259 tn.addr_read |= TLB_WATCHPOINT;
1261 } else {
1262 tn.addr_read = -1;
1265 if (prot & PAGE_EXEC) {
1266 tn.addr_code = address;
1267 } else {
1268 tn.addr_code = -1;
1271 tn.addr_write = -1;
1272 if (prot & PAGE_WRITE) {
1273 tn.addr_write = write_address;
1274 if (prot & PAGE_WRITE_INV) {
1275 tn.addr_write |= TLB_INVALID_MASK;
1277 if (wp_flags & BP_MEM_WRITE) {
1278 tn.addr_write |= TLB_WATCHPOINT;
1282 copy_tlb_helper_locked(te, &tn);
1283 tlb_n_used_entries_inc(env, mmu_idx);
1284 qemu_spin_unlock(&tlb->c.lock);
1287 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
1288 hwaddr paddr, MemTxAttrs attrs, int prot,
1289 int mmu_idx, target_ulong size)
1291 CPUTLBEntryFull full = {
1292 .phys_addr = paddr,
1293 .attrs = attrs,
1294 .prot = prot,
1295 .lg_page_size = ctz64(size)
1298 assert(is_power_of_2(size));
1299 tlb_set_page_full(cpu, mmu_idx, vaddr, &full);
1302 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
1303 hwaddr paddr, int prot,
1304 int mmu_idx, target_ulong size)
1306 tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
1307 prot, mmu_idx, size);
1311 * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
1312 * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
1313 * be discarded and looked up again (e.g. via tlb_entry()).
1315 static void tlb_fill(CPUState *cpu, target_ulong addr, int size,
1316 MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
1318 bool ok;
1321 * This is not a probe, so only valid return is success; failure
1322 * should result in exception + longjmp to the cpu loop.
1324 ok = cpu->cc->tcg_ops->tlb_fill(cpu, addr, size,
1325 access_type, mmu_idx, false, retaddr);
1326 assert(ok);
1329 static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
1330 MMUAccessType access_type,
1331 int mmu_idx, uintptr_t retaddr)
1333 cpu->cc->tcg_ops->do_unaligned_access(cpu, addr, access_type,
1334 mmu_idx, retaddr);
1337 static inline void cpu_transaction_failed(CPUState *cpu, hwaddr physaddr,
1338 vaddr addr, unsigned size,
1339 MMUAccessType access_type,
1340 int mmu_idx, MemTxAttrs attrs,
1341 MemTxResult response,
1342 uintptr_t retaddr)
1344 CPUClass *cc = CPU_GET_CLASS(cpu);
1346 if (!cpu->ignore_memory_transaction_failures &&
1347 cc->tcg_ops->do_transaction_failed) {
1348 cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size,
1349 access_type, mmu_idx, attrs,
1350 response, retaddr);
1354 static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full,
1355 int mmu_idx, target_ulong addr, uintptr_t retaddr,
1356 MMUAccessType access_type, MemOp op)
1358 CPUState *cpu = env_cpu(env);
1359 hwaddr mr_offset;
1360 MemoryRegionSection *section;
1361 MemoryRegion *mr;
1362 uint64_t val;
1363 MemTxResult r;
1365 section = iotlb_to_section(cpu, full->xlat_section, full->attrs);
1366 mr = section->mr;
1367 mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
1368 cpu->mem_io_pc = retaddr;
1369 if (!cpu->can_do_io) {
1370 cpu_io_recompile(cpu, retaddr);
1374 QEMU_IOTHREAD_LOCK_GUARD();
1375 r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs);
1378 if (r != MEMTX_OK) {
1379 hwaddr physaddr = mr_offset +
1380 section->offset_within_address_space -
1381 section->offset_within_region;
1383 cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type,
1384 mmu_idx, full->attrs, r, retaddr);
1386 return val;
1390 * Save a potentially trashed CPUTLBEntryFull for later lookup by plugin.
1391 * This is read by tlb_plugin_lookup if the fulltlb entry doesn't match
1392 * because of the side effect of io_writex changing memory layout.
1394 static void save_iotlb_data(CPUState *cs, MemoryRegionSection *section,
1395 hwaddr mr_offset)
1397 #ifdef CONFIG_PLUGIN
1398 SavedIOTLB *saved = &cs->saved_iotlb;
1399 saved->section = section;
1400 saved->mr_offset = mr_offset;
1401 #endif
1404 static void io_writex(CPUArchState *env, CPUTLBEntryFull *full,
1405 int mmu_idx, uint64_t val, target_ulong addr,
1406 uintptr_t retaddr, MemOp op)
1408 CPUState *cpu = env_cpu(env);
1409 hwaddr mr_offset;
1410 MemoryRegionSection *section;
1411 MemoryRegion *mr;
1412 MemTxResult r;
1414 section = iotlb_to_section(cpu, full->xlat_section, full->attrs);
1415 mr = section->mr;
1416 mr_offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
1417 if (!cpu->can_do_io) {
1418 cpu_io_recompile(cpu, retaddr);
1420 cpu->mem_io_pc = retaddr;
1423 * The memory_region_dispatch may trigger a flush/resize
1424 * so for plugins we save the iotlb_data just in case.
1426 save_iotlb_data(cpu, section, mr_offset);
1429 QEMU_IOTHREAD_LOCK_GUARD();
1430 r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs);
1433 if (r != MEMTX_OK) {
1434 hwaddr physaddr = mr_offset +
1435 section->offset_within_address_space -
1436 section->offset_within_region;
1438 cpu_transaction_failed(cpu, physaddr, addr, memop_size(op),
1439 MMU_DATA_STORE, mmu_idx, full->attrs, r,
1440 retaddr);
1444 static inline target_ulong tlb_read_ofs(CPUTLBEntry *entry, size_t ofs)
1446 #if TCG_OVERSIZED_GUEST
1447 return *(target_ulong *)((uintptr_t)entry + ofs);
1448 #else
1449 /* ofs might correspond to .addr_write, so use qatomic_read */
1450 return qatomic_read((target_ulong *)((uintptr_t)entry + ofs));
1451 #endif
1454 /* Return true if ADDR is present in the victim tlb, and has been copied
1455 back to the main tlb. */
1456 static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
1457 size_t elt_ofs, target_ulong page)
1459 size_t vidx;
1461 assert_cpu_is_self(env_cpu(env));
1462 for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
1463 CPUTLBEntry *vtlb = &env_tlb(env)->d[mmu_idx].vtable[vidx];
1464 target_ulong cmp;
1466 /* elt_ofs might correspond to .addr_write, so use qatomic_read */
1467 #if TCG_OVERSIZED_GUEST
1468 cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
1469 #else
1470 cmp = qatomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
1471 #endif
1473 if (cmp == page) {
1474 /* Found entry in victim tlb, swap tlb and iotlb. */
1475 CPUTLBEntry tmptlb, *tlb = &env_tlb(env)->f[mmu_idx].table[index];
1477 qemu_spin_lock(&env_tlb(env)->c.lock);
1478 copy_tlb_helper_locked(&tmptlb, tlb);
1479 copy_tlb_helper_locked(tlb, vtlb);
1480 copy_tlb_helper_locked(vtlb, &tmptlb);
1481 qemu_spin_unlock(&env_tlb(env)->c.lock);
1483 CPUTLBEntryFull *f1 = &env_tlb(env)->d[mmu_idx].fulltlb[index];
1484 CPUTLBEntryFull *f2 = &env_tlb(env)->d[mmu_idx].vfulltlb[vidx];
1485 CPUTLBEntryFull tmpf;
1486 tmpf = *f1; *f1 = *f2; *f2 = tmpf;
1487 return true;
1490 return false;
1493 /* Macro to call the above, with local variables from the use context. */
1494 #define VICTIM_TLB_HIT(TY, ADDR) \
1495 victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
1496 (ADDR) & TARGET_PAGE_MASK)
1498 static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,
1499 CPUTLBEntryFull *full, uintptr_t retaddr)
1501 ram_addr_t ram_addr = mem_vaddr + full->xlat_section;
1503 trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);
1505 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1506 tb_invalidate_phys_range_fast(ram_addr, size, retaddr);
1510 * Set both VGA and migration bits for simplicity and to remove
1511 * the notdirty callback faster.
1513 cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);
1515 /* We remove the notdirty callback only if the code has been flushed. */
1516 if (!cpu_physical_memory_is_clean(ram_addr)) {
1517 trace_memory_notdirty_set_dirty(mem_vaddr);
1518 tlb_set_dirty(cpu, mem_vaddr);
1522 static int probe_access_internal(CPUArchState *env, target_ulong addr,
1523 int fault_size, MMUAccessType access_type,
1524 int mmu_idx, bool nonfault,
1525 void **phost, CPUTLBEntryFull **pfull,
1526 uintptr_t retaddr)
1528 uintptr_t index = tlb_index(env, mmu_idx, addr);
1529 CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr);
1530 target_ulong tlb_addr, page_addr;
1531 size_t elt_ofs;
1532 int flags;
1534 switch (access_type) {
1535 case MMU_DATA_LOAD:
1536 elt_ofs = offsetof(CPUTLBEntry, addr_read);
1537 break;
1538 case MMU_DATA_STORE:
1539 elt_ofs = offsetof(CPUTLBEntry, addr_write);
1540 break;
1541 case MMU_INST_FETCH:
1542 elt_ofs = offsetof(CPUTLBEntry, addr_code);
1543 break;
1544 default:
1545 g_assert_not_reached();
1547 tlb_addr = tlb_read_ofs(entry, elt_ofs);
1549 flags = TLB_FLAGS_MASK;
1550 page_addr = addr & TARGET_PAGE_MASK;
1551 if (!tlb_hit_page(tlb_addr, page_addr)) {
1552 if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page_addr)) {
1553 CPUState *cs = env_cpu(env);
1555 if (!cs->cc->tcg_ops->tlb_fill(cs, addr, fault_size, access_type,
1556 mmu_idx, nonfault, retaddr)) {
1557 /* Non-faulting page table read failed. */
1558 *phost = NULL;
1559 *pfull = NULL;
1560 return TLB_INVALID_MASK;
1563 /* TLB resize via tlb_fill may have moved the entry. */
1564 index = tlb_index(env, mmu_idx, addr);
1565 entry = tlb_entry(env, mmu_idx, addr);
1568 * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
1569 * to force the next access through tlb_fill. We've just
1570 * called tlb_fill, so we know that this entry *is* valid.
1572 flags &= ~TLB_INVALID_MASK;
1574 tlb_addr = tlb_read_ofs(entry, elt_ofs);
1576 flags &= tlb_addr;
1578 *pfull = &env_tlb(env)->d[mmu_idx].fulltlb[index];
1580 /* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
1581 if (unlikely(flags & ~(TLB_WATCHPOINT | TLB_NOTDIRTY))) {
1582 *phost = NULL;
1583 return TLB_MMIO;
1586 /* Everything else is RAM. */
1587 *phost = (void *)((uintptr_t)addr + entry->addend);
1588 return flags;
1591 int probe_access_full(CPUArchState *env, target_ulong addr, int size,
1592 MMUAccessType access_type, int mmu_idx,
1593 bool nonfault, void **phost, CPUTLBEntryFull **pfull,
1594 uintptr_t retaddr)
1596 int flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
1597 nonfault, phost, pfull, retaddr);
1599 /* Handle clean RAM pages. */
1600 if (unlikely(flags & TLB_NOTDIRTY)) {
1601 notdirty_write(env_cpu(env), addr, 1, *pfull, retaddr);
1602 flags &= ~TLB_NOTDIRTY;
1605 return flags;
1608 int probe_access_flags(CPUArchState *env, target_ulong addr, int size,
1609 MMUAccessType access_type, int mmu_idx,
1610 bool nonfault, void **phost, uintptr_t retaddr)
1612 CPUTLBEntryFull *full;
1613 int flags;
1615 g_assert(-(addr | TARGET_PAGE_MASK) >= size);
1617 flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
1618 nonfault, phost, &full, retaddr);
1620 /* Handle clean RAM pages. */
1621 if (unlikely(flags & TLB_NOTDIRTY)) {
1622 notdirty_write(env_cpu(env), addr, 1, full, retaddr);
1623 flags &= ~TLB_NOTDIRTY;
1626 return flags;
1629 void *probe_access(CPUArchState *env, target_ulong addr, int size,
1630 MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
1632 CPUTLBEntryFull *full;
1633 void *host;
1634 int flags;
1636 g_assert(-(addr | TARGET_PAGE_MASK) >= size);
1638 flags = probe_access_internal(env, addr, size, access_type, mmu_idx,
1639 false, &host, &full, retaddr);
1641 /* Per the interface, size == 0 merely faults the access. */
1642 if (size == 0) {
1643 return NULL;
1646 if (unlikely(flags & (TLB_NOTDIRTY | TLB_WATCHPOINT))) {
1647 /* Handle watchpoints. */
1648 if (flags & TLB_WATCHPOINT) {
1649 int wp_access = (access_type == MMU_DATA_STORE
1650 ? BP_MEM_WRITE : BP_MEM_READ);
1651 cpu_check_watchpoint(env_cpu(env), addr, size,
1652 full->attrs, wp_access, retaddr);
1655 /* Handle clean RAM pages. */
1656 if (flags & TLB_NOTDIRTY) {
1657 notdirty_write(env_cpu(env), addr, 1, full, retaddr);
1661 return host;
1664 void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
1665 MMUAccessType access_type, int mmu_idx)
1667 CPUTLBEntryFull *full;
1668 void *host;
1669 int flags;
1671 flags = probe_access_internal(env, addr, 0, access_type,
1672 mmu_idx, true, &host, &full, 0);
1674 /* No combination of flags are expected by the caller. */
1675 return flags ? NULL : host;
1679 * Return a ram_addr_t for the virtual address for execution.
1681 * Return -1 if we can't translate and execute from an entire page
1682 * of RAM. This will force us to execute by loading and translating
1683 * one insn at a time, without caching.
1685 * NOTE: This function will trigger an exception if the page is
1686 * not executable.
1688 tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr,
1689 void **hostp)
1691 CPUTLBEntryFull *full;
1692 void *p;
1694 (void)probe_access_internal(env, addr, 1, MMU_INST_FETCH,
1695 cpu_mmu_index(env, true), false, &p, &full, 0);
1696 if (p == NULL) {
1697 return -1;
1699 if (hostp) {
1700 *hostp = p;
1702 return qemu_ram_addr_from_host_nofail(p);
1705 #ifdef CONFIG_PLUGIN
1707 * Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1708 * This should be a hot path as we will have just looked this path up
1709 * in the softmmu lookup code (or helper). We don't handle re-fills or
1710 * checking the victim table. This is purely informational.
1712 * This almost never fails as the memory access being instrumented
1713 * should have just filled the TLB. The one corner case is io_writex
1714 * which can cause TLB flushes and potential resizing of the TLBs
1715 * losing the information we need. In those cases we need to recover
1716 * data from a copy of the CPUTLBEntryFull. As long as this always occurs
1717 * from the same thread (which a mem callback will be) this is safe.
1720 bool tlb_plugin_lookup(CPUState *cpu, target_ulong addr, int mmu_idx,
1721 bool is_store, struct qemu_plugin_hwaddr *data)
1723 CPUArchState *env = cpu->env_ptr;
1724 CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr);
1725 uintptr_t index = tlb_index(env, mmu_idx, addr);
1726 target_ulong tlb_addr = is_store ? tlb_addr_write(tlbe) : tlbe->addr_read;
1728 if (likely(tlb_hit(tlb_addr, addr))) {
1729 /* We must have an iotlb entry for MMIO */
1730 if (tlb_addr & TLB_MMIO) {
1731 CPUTLBEntryFull *full;
1732 full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
1733 data->is_io = true;
1734 data->v.io.section =
1735 iotlb_to_section(cpu, full->xlat_section, full->attrs);
1736 data->v.io.offset = (full->xlat_section & TARGET_PAGE_MASK) + addr;
1737 } else {
1738 data->is_io = false;
1739 data->v.ram.hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
1741 return true;
1742 } else {
1743 SavedIOTLB *saved = &cpu->saved_iotlb;
1744 data->is_io = true;
1745 data->v.io.section = saved->section;
1746 data->v.io.offset = saved->mr_offset;
1747 return true;
1751 #endif
1754 * Probe for an atomic operation. Do not allow unaligned operations,
1755 * or io operations to proceed. Return the host address.
1757 * @prot may be PAGE_READ, PAGE_WRITE, or PAGE_READ|PAGE_WRITE.
1759 static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
1760 MemOpIdx oi, int size, int prot,
1761 uintptr_t retaddr)
1763 uintptr_t mmu_idx = get_mmuidx(oi);
1764 MemOp mop = get_memop(oi);
1765 int a_bits = get_alignment_bits(mop);
1766 uintptr_t index;
1767 CPUTLBEntry *tlbe;
1768 target_ulong tlb_addr;
1769 void *hostaddr;
1770 CPUTLBEntryFull *full;
1772 tcg_debug_assert(mmu_idx < NB_MMU_MODES);
1774 /* Adjust the given return address. */
1775 retaddr -= GETPC_ADJ;
1777 /* Enforce guest required alignment. */
1778 if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
1779 /* ??? Maybe indicate atomic op to cpu_unaligned_access */
1780 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
1781 mmu_idx, retaddr);
1784 /* Enforce qemu required alignment. */
1785 if (unlikely(addr & (size - 1))) {
1786 /* We get here if guest alignment was not requested,
1787 or was not enforced by cpu_unaligned_access above.
1788 We might widen the access and emulate, but for now
1789 mark an exception and exit the cpu loop. */
1790 goto stop_the_world;
1793 index = tlb_index(env, mmu_idx, addr);
1794 tlbe = tlb_entry(env, mmu_idx, addr);
1796 /* Check TLB entry and enforce page permissions. */
1797 if (prot & PAGE_WRITE) {
1798 tlb_addr = tlb_addr_write(tlbe);
1799 if (!tlb_hit(tlb_addr, addr)) {
1800 if (!VICTIM_TLB_HIT(addr_write, addr)) {
1801 tlb_fill(env_cpu(env), addr, size,
1802 MMU_DATA_STORE, mmu_idx, retaddr);
1803 index = tlb_index(env, mmu_idx, addr);
1804 tlbe = tlb_entry(env, mmu_idx, addr);
1806 tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
1809 if (prot & PAGE_READ) {
1811 * Let the guest notice RMW on a write-only page.
1812 * We have just verified that the page is writable.
1813 * Subpage lookups may have left TLB_INVALID_MASK set,
1814 * but addr_read will only be -1 if PAGE_READ was unset.
1816 if (unlikely(tlbe->addr_read == -1)) {
1817 tlb_fill(env_cpu(env), addr, size,
1818 MMU_DATA_LOAD, mmu_idx, retaddr);
1820 * Since we don't support reads and writes to different
1821 * addresses, and we do have the proper page loaded for
1822 * write, this shouldn't ever return. But just in case,
1823 * handle via stop-the-world.
1825 goto stop_the_world;
1827 /* Collect TLB_WATCHPOINT for read. */
1828 tlb_addr |= tlbe->addr_read;
1830 } else /* if (prot & PAGE_READ) */ {
1831 tlb_addr = tlbe->addr_read;
1832 if (!tlb_hit(tlb_addr, addr)) {
1833 if (!VICTIM_TLB_HIT(addr_write, addr)) {
1834 tlb_fill(env_cpu(env), addr, size,
1835 MMU_DATA_LOAD, mmu_idx, retaddr);
1836 index = tlb_index(env, mmu_idx, addr);
1837 tlbe = tlb_entry(env, mmu_idx, addr);
1839 tlb_addr = tlbe->addr_read & ~TLB_INVALID_MASK;
1843 /* Notice an IO access or a needs-MMU-lookup access */
1844 if (unlikely(tlb_addr & (TLB_MMIO | TLB_DISCARD_WRITE))) {
1845 /* There's really nothing that can be done to
1846 support this apart from stop-the-world. */
1847 goto stop_the_world;
1850 hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
1851 full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
1853 if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
1854 notdirty_write(env_cpu(env), addr, size, full, retaddr);
1857 if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
1858 QEMU_BUILD_BUG_ON(PAGE_READ != BP_MEM_READ);
1859 QEMU_BUILD_BUG_ON(PAGE_WRITE != BP_MEM_WRITE);
1860 /* therefore prot == watchpoint bits */
1861 cpu_check_watchpoint(env_cpu(env), addr, size,
1862 full->attrs, prot, retaddr);
1865 return hostaddr;
1867 stop_the_world:
1868 cpu_loop_exit_atomic(env_cpu(env), retaddr);
1872 * Verify that we have passed the correct MemOp to the correct function.
1874 * In the case of the helper_*_mmu functions, we will have done this by
1875 * using the MemOp to look up the helper during code generation.
1877 * In the case of the cpu_*_mmu functions, this is up to the caller.
1878 * We could present one function to target code, and dispatch based on
1879 * the MemOp, but so far we have worked hard to avoid an indirect function
1880 * call along the memory path.
1882 static void validate_memop(MemOpIdx oi, MemOp expected)
1884 #ifdef CONFIG_DEBUG_TCG
1885 MemOp have = get_memop(oi) & (MO_SIZE | MO_BSWAP);
1886 assert(have == expected);
1887 #endif
1891 * Load Helpers
1893 * We support two different access types. SOFTMMU_CODE_ACCESS is
1894 * specifically for reading instructions from system memory. It is
1895 * called by the translation loop and in some helpers where the code
1896 * is disassembled. It shouldn't be called directly by guest code.
1899 typedef uint64_t FullLoadHelper(CPUArchState *env, target_ulong addr,
1900 MemOpIdx oi, uintptr_t retaddr);
1902 static inline uint64_t QEMU_ALWAYS_INLINE
1903 load_memop(const void *haddr, MemOp op)
1905 switch (op) {
1906 case MO_UB:
1907 return ldub_p(haddr);
1908 case MO_BEUW:
1909 return lduw_be_p(haddr);
1910 case MO_LEUW:
1911 return lduw_le_p(haddr);
1912 case MO_BEUL:
1913 return (uint32_t)ldl_be_p(haddr);
1914 case MO_LEUL:
1915 return (uint32_t)ldl_le_p(haddr);
1916 case MO_BEUQ:
1917 return ldq_be_p(haddr);
1918 case MO_LEUQ:
1919 return ldq_le_p(haddr);
1920 default:
1921 qemu_build_not_reached();
1925 static inline uint64_t QEMU_ALWAYS_INLINE
1926 load_helper(CPUArchState *env, target_ulong addr, MemOpIdx oi,
1927 uintptr_t retaddr, MemOp op, bool code_read,
1928 FullLoadHelper *full_load)
1930 const size_t tlb_off = code_read ?
1931 offsetof(CPUTLBEntry, addr_code) : offsetof(CPUTLBEntry, addr_read);
1932 const MMUAccessType access_type =
1933 code_read ? MMU_INST_FETCH : MMU_DATA_LOAD;
1934 const unsigned a_bits = get_alignment_bits(get_memop(oi));
1935 const size_t size = memop_size(op);
1936 uintptr_t mmu_idx = get_mmuidx(oi);
1937 uintptr_t index;
1938 CPUTLBEntry *entry;
1939 target_ulong tlb_addr;
1940 void *haddr;
1941 uint64_t res;
1943 tcg_debug_assert(mmu_idx < NB_MMU_MODES);
1945 /* Handle CPU specific unaligned behaviour */
1946 if (addr & ((1 << a_bits) - 1)) {
1947 cpu_unaligned_access(env_cpu(env), addr, access_type,
1948 mmu_idx, retaddr);
1951 index = tlb_index(env, mmu_idx, addr);
1952 entry = tlb_entry(env, mmu_idx, addr);
1953 tlb_addr = code_read ? entry->addr_code : entry->addr_read;
1955 /* If the TLB entry is for a different page, reload and try again. */
1956 if (!tlb_hit(tlb_addr, addr)) {
1957 if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
1958 addr & TARGET_PAGE_MASK)) {
1959 tlb_fill(env_cpu(env), addr, size,
1960 access_type, mmu_idx, retaddr);
1961 index = tlb_index(env, mmu_idx, addr);
1962 entry = tlb_entry(env, mmu_idx, addr);
1964 tlb_addr = code_read ? entry->addr_code : entry->addr_read;
1965 tlb_addr &= ~TLB_INVALID_MASK;
1968 /* Handle anything that isn't just a straight memory access. */
1969 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
1970 CPUTLBEntryFull *full;
1971 bool need_swap;
1973 /* For anything that is unaligned, recurse through full_load. */
1974 if ((addr & (size - 1)) != 0) {
1975 goto do_unaligned_access;
1978 full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
1980 /* Handle watchpoints. */
1981 if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
1982 /* On watchpoint hit, this will longjmp out. */
1983 cpu_check_watchpoint(env_cpu(env), addr, size,
1984 full->attrs, BP_MEM_READ, retaddr);
1987 need_swap = size > 1 && (tlb_addr & TLB_BSWAP);
1989 /* Handle I/O access. */
1990 if (likely(tlb_addr & TLB_MMIO)) {
1991 return io_readx(env, full, mmu_idx, addr, retaddr,
1992 access_type, op ^ (need_swap * MO_BSWAP));
1995 haddr = (void *)((uintptr_t)addr + entry->addend);
1998 * Keep these two load_memop separate to ensure that the compiler
1999 * is able to fold the entire function to a single instruction.
2000 * There is a build-time assert inside to remind you of this. ;-)
2002 if (unlikely(need_swap)) {
2003 return load_memop(haddr, op ^ MO_BSWAP);
2005 return load_memop(haddr, op);
2008 /* Handle slow unaligned access (it spans two pages or IO). */
2009 if (size > 1
2010 && unlikely((addr & ~TARGET_PAGE_MASK) + size - 1
2011 >= TARGET_PAGE_SIZE)) {
2012 target_ulong addr1, addr2;
2013 uint64_t r1, r2;
2014 unsigned shift;
2015 do_unaligned_access:
2016 addr1 = addr & ~((target_ulong)size - 1);
2017 addr2 = addr1 + size;
2018 r1 = full_load(env, addr1, oi, retaddr);
2019 r2 = full_load(env, addr2, oi, retaddr);
2020 shift = (addr & (size - 1)) * 8;
2022 if (memop_big_endian(op)) {
2023 /* Big-endian combine. */
2024 res = (r1 << shift) | (r2 >> ((size * 8) - shift));
2025 } else {
2026 /* Little-endian combine. */
2027 res = (r1 >> shift) | (r2 << ((size * 8) - shift));
2029 return res & MAKE_64BIT_MASK(0, size * 8);
2032 haddr = (void *)((uintptr_t)addr + entry->addend);
2033 return load_memop(haddr, op);
2037 * For the benefit of TCG generated code, we want to avoid the
2038 * complication of ABI-specific return type promotion and always
2039 * return a value extended to the register size of the host. This is
2040 * tcg_target_long, except in the case of a 32-bit host and 64-bit
2041 * data, and for that we always have uint64_t.
2043 * We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
2046 static uint64_t full_ldub_mmu(CPUArchState *env, target_ulong addr,
2047 MemOpIdx oi, uintptr_t retaddr)
2049 validate_memop(oi, MO_UB);
2050 return load_helper(env, addr, oi, retaddr, MO_UB, false, full_ldub_mmu);
2053 tcg_target_ulong helper_ret_ldub_mmu(CPUArchState *env, target_ulong addr,
2054 MemOpIdx oi, uintptr_t retaddr)
2056 return full_ldub_mmu(env, addr, oi, retaddr);
2059 static uint64_t full_le_lduw_mmu(CPUArchState *env, target_ulong addr,
2060 MemOpIdx oi, uintptr_t retaddr)
2062 validate_memop(oi, MO_LEUW);
2063 return load_helper(env, addr, oi, retaddr, MO_LEUW, false,
2064 full_le_lduw_mmu);
2067 tcg_target_ulong helper_le_lduw_mmu(CPUArchState *env, target_ulong addr,
2068 MemOpIdx oi, uintptr_t retaddr)
2070 return full_le_lduw_mmu(env, addr, oi, retaddr);
2073 static uint64_t full_be_lduw_mmu(CPUArchState *env, target_ulong addr,
2074 MemOpIdx oi, uintptr_t retaddr)
2076 validate_memop(oi, MO_BEUW);
2077 return load_helper(env, addr, oi, retaddr, MO_BEUW, false,
2078 full_be_lduw_mmu);
2081 tcg_target_ulong helper_be_lduw_mmu(CPUArchState *env, target_ulong addr,
2082 MemOpIdx oi, uintptr_t retaddr)
2084 return full_be_lduw_mmu(env, addr, oi, retaddr);
2087 static uint64_t full_le_ldul_mmu(CPUArchState *env, target_ulong addr,
2088 MemOpIdx oi, uintptr_t retaddr)
2090 validate_memop(oi, MO_LEUL);
2091 return load_helper(env, addr, oi, retaddr, MO_LEUL, false,
2092 full_le_ldul_mmu);
2095 tcg_target_ulong helper_le_ldul_mmu(CPUArchState *env, target_ulong addr,
2096 MemOpIdx oi, uintptr_t retaddr)
2098 return full_le_ldul_mmu(env, addr, oi, retaddr);
2101 static uint64_t full_be_ldul_mmu(CPUArchState *env, target_ulong addr,
2102 MemOpIdx oi, uintptr_t retaddr)
2104 validate_memop(oi, MO_BEUL);
2105 return load_helper(env, addr, oi, retaddr, MO_BEUL, false,
2106 full_be_ldul_mmu);
2109 tcg_target_ulong helper_be_ldul_mmu(CPUArchState *env, target_ulong addr,
2110 MemOpIdx oi, uintptr_t retaddr)
2112 return full_be_ldul_mmu(env, addr, oi, retaddr);
2115 uint64_t helper_le_ldq_mmu(CPUArchState *env, target_ulong addr,
2116 MemOpIdx oi, uintptr_t retaddr)
2118 validate_memop(oi, MO_LEUQ);
2119 return load_helper(env, addr, oi, retaddr, MO_LEUQ, false,
2120 helper_le_ldq_mmu);
2123 uint64_t helper_be_ldq_mmu(CPUArchState *env, target_ulong addr,
2124 MemOpIdx oi, uintptr_t retaddr)
2126 validate_memop(oi, MO_BEUQ);
2127 return load_helper(env, addr, oi, retaddr, MO_BEUQ, false,
2128 helper_be_ldq_mmu);
2132 * Provide signed versions of the load routines as well. We can of course
2133 * avoid this for 64-bit data, or for 32-bit data on 32-bit host.
2137 tcg_target_ulong helper_ret_ldsb_mmu(CPUArchState *env, target_ulong addr,
2138 MemOpIdx oi, uintptr_t retaddr)
2140 return (int8_t)helper_ret_ldub_mmu(env, addr, oi, retaddr);
2143 tcg_target_ulong helper_le_ldsw_mmu(CPUArchState *env, target_ulong addr,
2144 MemOpIdx oi, uintptr_t retaddr)
2146 return (int16_t)helper_le_lduw_mmu(env, addr, oi, retaddr);
2149 tcg_target_ulong helper_be_ldsw_mmu(CPUArchState *env, target_ulong addr,
2150 MemOpIdx oi, uintptr_t retaddr)
2152 return (int16_t)helper_be_lduw_mmu(env, addr, oi, retaddr);
2155 tcg_target_ulong helper_le_ldsl_mmu(CPUArchState *env, target_ulong addr,
2156 MemOpIdx oi, uintptr_t retaddr)
2158 return (int32_t)helper_le_ldul_mmu(env, addr, oi, retaddr);
2161 tcg_target_ulong helper_be_ldsl_mmu(CPUArchState *env, target_ulong addr,
2162 MemOpIdx oi, uintptr_t retaddr)
2164 return (int32_t)helper_be_ldul_mmu(env, addr, oi, retaddr);
2168 * Load helpers for cpu_ldst.h.
2171 static inline uint64_t cpu_load_helper(CPUArchState *env, abi_ptr addr,
2172 MemOpIdx oi, uintptr_t retaddr,
2173 FullLoadHelper *full_load)
2175 uint64_t ret;
2177 ret = full_load(env, addr, oi, retaddr);
2178 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
2179 return ret;
2182 uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
2184 return cpu_load_helper(env, addr, oi, ra, full_ldub_mmu);
2187 uint16_t cpu_ldw_be_mmu(CPUArchState *env, abi_ptr addr,
2188 MemOpIdx oi, uintptr_t ra)
2190 return cpu_load_helper(env, addr, oi, ra, full_be_lduw_mmu);
2193 uint32_t cpu_ldl_be_mmu(CPUArchState *env, abi_ptr addr,
2194 MemOpIdx oi, uintptr_t ra)
2196 return cpu_load_helper(env, addr, oi, ra, full_be_ldul_mmu);
2199 uint64_t cpu_ldq_be_mmu(CPUArchState *env, abi_ptr addr,
2200 MemOpIdx oi, uintptr_t ra)
2202 return cpu_load_helper(env, addr, oi, ra, helper_be_ldq_mmu);
2205 uint16_t cpu_ldw_le_mmu(CPUArchState *env, abi_ptr addr,
2206 MemOpIdx oi, uintptr_t ra)
2208 return cpu_load_helper(env, addr, oi, ra, full_le_lduw_mmu);
2211 uint32_t cpu_ldl_le_mmu(CPUArchState *env, abi_ptr addr,
2212 MemOpIdx oi, uintptr_t ra)
2214 return cpu_load_helper(env, addr, oi, ra, full_le_ldul_mmu);
2217 uint64_t cpu_ldq_le_mmu(CPUArchState *env, abi_ptr addr,
2218 MemOpIdx oi, uintptr_t ra)
2220 return cpu_load_helper(env, addr, oi, ra, helper_le_ldq_mmu);
2223 Int128 cpu_ld16_be_mmu(CPUArchState *env, abi_ptr addr,
2224 MemOpIdx oi, uintptr_t ra)
2226 MemOp mop = get_memop(oi);
2227 int mmu_idx = get_mmuidx(oi);
2228 MemOpIdx new_oi;
2229 unsigned a_bits;
2230 uint64_t h, l;
2232 tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_BE|MO_128));
2233 a_bits = get_alignment_bits(mop);
2235 /* Handle CPU specific unaligned behaviour */
2236 if (addr & ((1 << a_bits) - 1)) {
2237 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_LOAD,
2238 mmu_idx, ra);
2241 /* Construct an unaligned 64-bit replacement MemOpIdx. */
2242 mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN;
2243 new_oi = make_memop_idx(mop, mmu_idx);
2245 h = helper_be_ldq_mmu(env, addr, new_oi, ra);
2246 l = helper_be_ldq_mmu(env, addr + 8, new_oi, ra);
2248 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
2249 return int128_make128(l, h);
2252 Int128 cpu_ld16_le_mmu(CPUArchState *env, abi_ptr addr,
2253 MemOpIdx oi, uintptr_t ra)
2255 MemOp mop = get_memop(oi);
2256 int mmu_idx = get_mmuidx(oi);
2257 MemOpIdx new_oi;
2258 unsigned a_bits;
2259 uint64_t h, l;
2261 tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_LE|MO_128));
2262 a_bits = get_alignment_bits(mop);
2264 /* Handle CPU specific unaligned behaviour */
2265 if (addr & ((1 << a_bits) - 1)) {
2266 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_LOAD,
2267 mmu_idx, ra);
2270 /* Construct an unaligned 64-bit replacement MemOpIdx. */
2271 mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN;
2272 new_oi = make_memop_idx(mop, mmu_idx);
2274 l = helper_le_ldq_mmu(env, addr, new_oi, ra);
2275 h = helper_le_ldq_mmu(env, addr + 8, new_oi, ra);
2277 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
2278 return int128_make128(l, h);
2282 * Store Helpers
2285 static inline void QEMU_ALWAYS_INLINE
2286 store_memop(void *haddr, uint64_t val, MemOp op)
2288 switch (op) {
2289 case MO_UB:
2290 stb_p(haddr, val);
2291 break;
2292 case MO_BEUW:
2293 stw_be_p(haddr, val);
2294 break;
2295 case MO_LEUW:
2296 stw_le_p(haddr, val);
2297 break;
2298 case MO_BEUL:
2299 stl_be_p(haddr, val);
2300 break;
2301 case MO_LEUL:
2302 stl_le_p(haddr, val);
2303 break;
2304 case MO_BEUQ:
2305 stq_be_p(haddr, val);
2306 break;
2307 case MO_LEUQ:
2308 stq_le_p(haddr, val);
2309 break;
2310 default:
2311 qemu_build_not_reached();
2315 static void full_stb_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2316 MemOpIdx oi, uintptr_t retaddr);
2318 static void __attribute__((noinline))
2319 store_helper_unaligned(CPUArchState *env, target_ulong addr, uint64_t val,
2320 uintptr_t retaddr, size_t size, uintptr_t mmu_idx,
2321 bool big_endian)
2323 const size_t tlb_off = offsetof(CPUTLBEntry, addr_write);
2324 uintptr_t index, index2;
2325 CPUTLBEntry *entry, *entry2;
2326 target_ulong page1, page2, tlb_addr, tlb_addr2;
2327 MemOpIdx oi;
2328 size_t size2;
2329 int i;
2332 * Ensure the second page is in the TLB. Note that the first page
2333 * is already guaranteed to be filled, and that the second page
2334 * cannot evict the first. An exception to this rule is PAGE_WRITE_INV
2335 * handling: the first page could have evicted itself.
2337 page1 = addr & TARGET_PAGE_MASK;
2338 page2 = (addr + size) & TARGET_PAGE_MASK;
2339 size2 = (addr + size) & ~TARGET_PAGE_MASK;
2340 index2 = tlb_index(env, mmu_idx, page2);
2341 entry2 = tlb_entry(env, mmu_idx, page2);
2343 tlb_addr2 = tlb_addr_write(entry2);
2344 if (page1 != page2 && !tlb_hit_page(tlb_addr2, page2)) {
2345 if (!victim_tlb_hit(env, mmu_idx, index2, tlb_off, page2)) {
2346 tlb_fill(env_cpu(env), page2, size2, MMU_DATA_STORE,
2347 mmu_idx, retaddr);
2348 index2 = tlb_index(env, mmu_idx, page2);
2349 entry2 = tlb_entry(env, mmu_idx, page2);
2351 tlb_addr2 = tlb_addr_write(entry2);
2354 index = tlb_index(env, mmu_idx, addr);
2355 entry = tlb_entry(env, mmu_idx, addr);
2356 tlb_addr = tlb_addr_write(entry);
2359 * Handle watchpoints. Since this may trap, all checks
2360 * must happen before any store.
2362 if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
2363 cpu_check_watchpoint(env_cpu(env), addr, size - size2,
2364 env_tlb(env)->d[mmu_idx].fulltlb[index].attrs,
2365 BP_MEM_WRITE, retaddr);
2367 if (unlikely(tlb_addr2 & TLB_WATCHPOINT)) {
2368 cpu_check_watchpoint(env_cpu(env), page2, size2,
2369 env_tlb(env)->d[mmu_idx].fulltlb[index2].attrs,
2370 BP_MEM_WRITE, retaddr);
2374 * XXX: not efficient, but simple.
2375 * This loop must go in the forward direction to avoid issues
2376 * with self-modifying code in Windows 64-bit.
2378 oi = make_memop_idx(MO_UB, mmu_idx);
2379 if (big_endian) {
2380 for (i = 0; i < size; ++i) {
2381 /* Big-endian extract. */
2382 uint8_t val8 = val >> (((size - 1) * 8) - (i * 8));
2383 full_stb_mmu(env, addr + i, val8, oi, retaddr);
2385 } else {
2386 for (i = 0; i < size; ++i) {
2387 /* Little-endian extract. */
2388 uint8_t val8 = val >> (i * 8);
2389 full_stb_mmu(env, addr + i, val8, oi, retaddr);
2394 static inline void QEMU_ALWAYS_INLINE
2395 store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
2396 MemOpIdx oi, uintptr_t retaddr, MemOp op)
2398 const size_t tlb_off = offsetof(CPUTLBEntry, addr_write);
2399 const unsigned a_bits = get_alignment_bits(get_memop(oi));
2400 const size_t size = memop_size(op);
2401 uintptr_t mmu_idx = get_mmuidx(oi);
2402 uintptr_t index;
2403 CPUTLBEntry *entry;
2404 target_ulong tlb_addr;
2405 void *haddr;
2407 tcg_debug_assert(mmu_idx < NB_MMU_MODES);
2409 /* Handle CPU specific unaligned behaviour */
2410 if (addr & ((1 << a_bits) - 1)) {
2411 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
2412 mmu_idx, retaddr);
2415 index = tlb_index(env, mmu_idx, addr);
2416 entry = tlb_entry(env, mmu_idx, addr);
2417 tlb_addr = tlb_addr_write(entry);
2419 /* If the TLB entry is for a different page, reload and try again. */
2420 if (!tlb_hit(tlb_addr, addr)) {
2421 if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
2422 addr & TARGET_PAGE_MASK)) {
2423 tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
2424 mmu_idx, retaddr);
2425 index = tlb_index(env, mmu_idx, addr);
2426 entry = tlb_entry(env, mmu_idx, addr);
2428 tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK;
2431 /* Handle anything that isn't just a straight memory access. */
2432 if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
2433 CPUTLBEntryFull *full;
2434 bool need_swap;
2436 /* For anything that is unaligned, recurse through byte stores. */
2437 if ((addr & (size - 1)) != 0) {
2438 goto do_unaligned_access;
2441 full = &env_tlb(env)->d[mmu_idx].fulltlb[index];
2443 /* Handle watchpoints. */
2444 if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
2445 /* On watchpoint hit, this will longjmp out. */
2446 cpu_check_watchpoint(env_cpu(env), addr, size,
2447 full->attrs, BP_MEM_WRITE, retaddr);
2450 need_swap = size > 1 && (tlb_addr & TLB_BSWAP);
2452 /* Handle I/O access. */
2453 if (tlb_addr & TLB_MMIO) {
2454 io_writex(env, full, mmu_idx, val, addr, retaddr,
2455 op ^ (need_swap * MO_BSWAP));
2456 return;
2459 /* Ignore writes to ROM. */
2460 if (unlikely(tlb_addr & TLB_DISCARD_WRITE)) {
2461 return;
2464 /* Handle clean RAM pages. */
2465 if (tlb_addr & TLB_NOTDIRTY) {
2466 notdirty_write(env_cpu(env), addr, size, full, retaddr);
2469 haddr = (void *)((uintptr_t)addr + entry->addend);
2472 * Keep these two store_memop separate to ensure that the compiler
2473 * is able to fold the entire function to a single instruction.
2474 * There is a build-time assert inside to remind you of this. ;-)
2476 if (unlikely(need_swap)) {
2477 store_memop(haddr, val, op ^ MO_BSWAP);
2478 } else {
2479 store_memop(haddr, val, op);
2481 return;
2484 /* Handle slow unaligned access (it spans two pages or IO). */
2485 if (size > 1
2486 && unlikely((addr & ~TARGET_PAGE_MASK) + size - 1
2487 >= TARGET_PAGE_SIZE)) {
2488 do_unaligned_access:
2489 store_helper_unaligned(env, addr, val, retaddr, size,
2490 mmu_idx, memop_big_endian(op));
2491 return;
2494 haddr = (void *)((uintptr_t)addr + entry->addend);
2495 store_memop(haddr, val, op);
2498 static void __attribute__((noinline))
2499 full_stb_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2500 MemOpIdx oi, uintptr_t retaddr)
2502 validate_memop(oi, MO_UB);
2503 store_helper(env, addr, val, oi, retaddr, MO_UB);
2506 void helper_ret_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
2507 MemOpIdx oi, uintptr_t retaddr)
2509 full_stb_mmu(env, addr, val, oi, retaddr);
2512 static void full_le_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2513 MemOpIdx oi, uintptr_t retaddr)
2515 validate_memop(oi, MO_LEUW);
2516 store_helper(env, addr, val, oi, retaddr, MO_LEUW);
2519 void helper_le_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
2520 MemOpIdx oi, uintptr_t retaddr)
2522 full_le_stw_mmu(env, addr, val, oi, retaddr);
2525 static void full_be_stw_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2526 MemOpIdx oi, uintptr_t retaddr)
2528 validate_memop(oi, MO_BEUW);
2529 store_helper(env, addr, val, oi, retaddr, MO_BEUW);
2532 void helper_be_stw_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
2533 MemOpIdx oi, uintptr_t retaddr)
2535 full_be_stw_mmu(env, addr, val, oi, retaddr);
2538 static void full_le_stl_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2539 MemOpIdx oi, uintptr_t retaddr)
2541 validate_memop(oi, MO_LEUL);
2542 store_helper(env, addr, val, oi, retaddr, MO_LEUL);
2545 void helper_le_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
2546 MemOpIdx oi, uintptr_t retaddr)
2548 full_le_stl_mmu(env, addr, val, oi, retaddr);
2551 static void full_be_stl_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2552 MemOpIdx oi, uintptr_t retaddr)
2554 validate_memop(oi, MO_BEUL);
2555 store_helper(env, addr, val, oi, retaddr, MO_BEUL);
2558 void helper_be_stl_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
2559 MemOpIdx oi, uintptr_t retaddr)
2561 full_be_stl_mmu(env, addr, val, oi, retaddr);
2564 void helper_le_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2565 MemOpIdx oi, uintptr_t retaddr)
2567 validate_memop(oi, MO_LEUQ);
2568 store_helper(env, addr, val, oi, retaddr, MO_LEUQ);
2571 void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2572 MemOpIdx oi, uintptr_t retaddr)
2574 validate_memop(oi, MO_BEUQ);
2575 store_helper(env, addr, val, oi, retaddr, MO_BEUQ);
2579 * Store Helpers for cpu_ldst.h
2582 typedef void FullStoreHelper(CPUArchState *env, target_ulong addr,
2583 uint64_t val, MemOpIdx oi, uintptr_t retaddr);
2585 static inline void cpu_store_helper(CPUArchState *env, target_ulong addr,
2586 uint64_t val, MemOpIdx oi, uintptr_t ra,
2587 FullStoreHelper *full_store)
2589 full_store(env, addr, val, oi, ra);
2590 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
2593 void cpu_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
2594 MemOpIdx oi, uintptr_t retaddr)
2596 cpu_store_helper(env, addr, val, oi, retaddr, full_stb_mmu);
2599 void cpu_stw_be_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
2600 MemOpIdx oi, uintptr_t retaddr)
2602 cpu_store_helper(env, addr, val, oi, retaddr, full_be_stw_mmu);
2605 void cpu_stl_be_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
2606 MemOpIdx oi, uintptr_t retaddr)
2608 cpu_store_helper(env, addr, val, oi, retaddr, full_be_stl_mmu);
2611 void cpu_stq_be_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2612 MemOpIdx oi, uintptr_t retaddr)
2614 cpu_store_helper(env, addr, val, oi, retaddr, helper_be_stq_mmu);
2617 void cpu_stw_le_mmu(CPUArchState *env, target_ulong addr, uint16_t val,
2618 MemOpIdx oi, uintptr_t retaddr)
2620 cpu_store_helper(env, addr, val, oi, retaddr, full_le_stw_mmu);
2623 void cpu_stl_le_mmu(CPUArchState *env, target_ulong addr, uint32_t val,
2624 MemOpIdx oi, uintptr_t retaddr)
2626 cpu_store_helper(env, addr, val, oi, retaddr, full_le_stl_mmu);
2629 void cpu_stq_le_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
2630 MemOpIdx oi, uintptr_t retaddr)
2632 cpu_store_helper(env, addr, val, oi, retaddr, helper_le_stq_mmu);
2635 void cpu_st16_be_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
2636 MemOpIdx oi, uintptr_t ra)
2638 MemOp mop = get_memop(oi);
2639 int mmu_idx = get_mmuidx(oi);
2640 MemOpIdx new_oi;
2641 unsigned a_bits;
2643 tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_BE|MO_128));
2644 a_bits = get_alignment_bits(mop);
2646 /* Handle CPU specific unaligned behaviour */
2647 if (addr & ((1 << a_bits) - 1)) {
2648 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
2649 mmu_idx, ra);
2652 /* Construct an unaligned 64-bit replacement MemOpIdx. */
2653 mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN;
2654 new_oi = make_memop_idx(mop, mmu_idx);
2656 helper_be_stq_mmu(env, addr, int128_gethi(val), new_oi, ra);
2657 helper_be_stq_mmu(env, addr + 8, int128_getlo(val), new_oi, ra);
2659 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
2662 void cpu_st16_le_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
2663 MemOpIdx oi, uintptr_t ra)
2665 MemOp mop = get_memop(oi);
2666 int mmu_idx = get_mmuidx(oi);
2667 MemOpIdx new_oi;
2668 unsigned a_bits;
2670 tcg_debug_assert((mop & (MO_BSWAP|MO_SSIZE)) == (MO_LE|MO_128));
2671 a_bits = get_alignment_bits(mop);
2673 /* Handle CPU specific unaligned behaviour */
2674 if (addr & ((1 << a_bits) - 1)) {
2675 cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
2676 mmu_idx, ra);
2679 /* Construct an unaligned 64-bit replacement MemOpIdx. */
2680 mop = (mop & ~(MO_SIZE | MO_AMASK)) | MO_64 | MO_UNALN;
2681 new_oi = make_memop_idx(mop, mmu_idx);
2683 helper_le_stq_mmu(env, addr, int128_getlo(val), new_oi, ra);
2684 helper_le_stq_mmu(env, addr + 8, int128_gethi(val), new_oi, ra);
2686 qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
2689 #include "ldst_common.c.inc"
2692 * First set of functions passes in OI and RETADDR.
2693 * This makes them callable from other helpers.
2696 #define ATOMIC_NAME(X) \
2697 glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
2699 #define ATOMIC_MMU_CLEANUP
2701 #include "atomic_common.c.inc"
2703 #define DATA_SIZE 1
2704 #include "atomic_template.h"
2706 #define DATA_SIZE 2
2707 #include "atomic_template.h"
2709 #define DATA_SIZE 4
2710 #include "atomic_template.h"
2712 #ifdef CONFIG_ATOMIC64
2713 #define DATA_SIZE 8
2714 #include "atomic_template.h"
2715 #endif
2717 #if HAVE_CMPXCHG128 || HAVE_ATOMIC128
2718 #define DATA_SIZE 16
2719 #include "atomic_template.h"
2720 #endif
2722 /* Code access functions. */
2724 static uint64_t full_ldub_code(CPUArchState *env, target_ulong addr,
2725 MemOpIdx oi, uintptr_t retaddr)
2727 return load_helper(env, addr, oi, retaddr, MO_8, true, full_ldub_code);
2730 uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
2732 MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(env, true));
2733 return full_ldub_code(env, addr, oi, 0);
2736 static uint64_t full_lduw_code(CPUArchState *env, target_ulong addr,
2737 MemOpIdx oi, uintptr_t retaddr)
2739 return load_helper(env, addr, oi, retaddr, MO_TEUW, true, full_lduw_code);
2742 uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
2744 MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(env, true));
2745 return full_lduw_code(env, addr, oi, 0);
2748 static uint64_t full_ldl_code(CPUArchState *env, target_ulong addr,
2749 MemOpIdx oi, uintptr_t retaddr)
2751 return load_helper(env, addr, oi, retaddr, MO_TEUL, true, full_ldl_code);
2754 uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
2756 MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(env, true));
2757 return full_ldl_code(env, addr, oi, 0);
2760 static uint64_t full_ldq_code(CPUArchState *env, target_ulong addr,
2761 MemOpIdx oi, uintptr_t retaddr)
2763 return load_helper(env, addr, oi, retaddr, MO_TEUQ, true, full_ldq_code);
2766 uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
2768 MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(env, true));
2769 return full_ldq_code(env, addr, oi, 0);