include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24 #include "exec/ramlist.h"
  25
  26 struct RAMBlock {
  27     struct rcu_head rcu;
  28     struct MemoryRegion *mr;
  29     uint8_t *host;
  30     ram_addr_t offset;
  31     ram_addr_t used_length;
  32     ram_addr_t max_length;
  33     void (*resized)(const char*, uint64_t length, void *host);
  34     uint32_t flags;
  35     /* Protected by iothread lock.  */
  36     char idstr[256];
  37     /* RCU-enabled, writes protected by the ramlist lock */
  38     QLIST_ENTRY(RAMBlock) next;
  39     QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers;
  40     int fd;
  41     size_t page_size;
  42     /* dirty bitmap used during migration */
  43     unsigned long *bmap;
  44     /* bitmap of pages that haven't been sent even once
  45      * only maintained and used in postcopy at the moment
  46      * where it's used to send the dirtymap at the start
  47      * of the postcopy phase
  48      */
  49     unsigned long *unsentmap;
  50     /* bitmap of already received pages in postcopy */
  51     unsigned long *receivedmap;
  52 };
  53
  54 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  55 {
  56     return (b && b->host && offset < b->used_length) ? true : false;
  57 }
  58
  59 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  60 {
  61     assert(offset_in_ramblock(block, offset));
  62     return (char *)block->host + offset;
  63 }
  64
  65 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
  66                                                             RAMBlock *rb)
  67 {
  68     uint64_t host_addr_offset =
  69             (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
  70     return host_addr_offset >> TARGET_PAGE_BITS;
  71 }
  72
  73 long qemu_getrampagesize(void);
  74 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  75                                    bool share, const char *mem_path,
  76                                    Error **errp);
  77 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
  78                                  bool share, int fd,
  79                                  Error **errp);
  80 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
  81                                   MemoryRegion *mr, Error **errp);
  82 RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr,
  83                          Error **errp);
  84 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
  85                                     void (*resized)(const char*,
  86                                                     uint64_t length,
  87                                                     void *host),
  88                                     MemoryRegion *mr, Error **errp);
  89 void qemu_ram_free(RAMBlock *block);
  90
  91 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
  92
  93 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
  94 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
  95
  96 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
  97
  98 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
  99                                                  ram_addr_t length,
 100                                                  unsigned client)
 101 {
 102     DirtyMemoryBlocks *blocks;
 103     unsigned long end, page;
 104     unsigned long idx, offset, base;
 105     bool dirty = false;
 106
 107     assert(client < DIRTY_MEMORY_NUM);
 108
 109     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 110     page = start >> TARGET_PAGE_BITS;
 111
 112     rcu_read_lock();
 113
 114     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 115
 116     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 117     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 118     base = page - offset;
 119     while (page < end) {
 120         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 121         unsigned long num = next - base;
 122         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
 123         if (found < num) {
 124             dirty = true;
 125             break;
 126         }
 127
 128         page = next;
 129         idx++;
 130         offset = 0;
 131         base += DIRTY_MEMORY_BLOCK_SIZE;
 132     }
 133
 134     rcu_read_unlock();
 135
 136     return dirty;
 137 }
 138
 139 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 140                                                  ram_addr_t length,
 141                                                  unsigned client)
 142 {
 143     DirtyMemoryBlocks *blocks;
 144     unsigned long end, page;
 145     unsigned long idx, offset, base;
 146     bool dirty = true;
 147
 148     assert(client < DIRTY_MEMORY_NUM);
 149
 150     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 151     page = start >> TARGET_PAGE_BITS;
 152
 153     rcu_read_lock();
 154
 155     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 156
 157     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 158     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 159     base = page - offset;
 160     while (page < end) {
 161         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 162         unsigned long num = next - base;
 163         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 164         if (found < num) {
 165             dirty = false;
 166             break;
 167         }
 168
 169         page = next;
 170         idx++;
 171         offset = 0;
 172         base += DIRTY_MEMORY_BLOCK_SIZE;
 173     }
 174
 175     rcu_read_unlock();
 176
 177     return dirty;
 178 }
 179
 180 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 181                                                       unsigned client)
 182 {
 183     return cpu_physical_memory_get_dirty(addr, 1, client);
 184 }
 185
 186 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 187 {
 188     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 189     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 190     bool migration =
 191         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 192     return !(vga && code && migration);
 193 }
 194
 195 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 196                                                                ram_addr_t length,
 197                                                                uint8_t mask)
 198 {
 199     uint8_t ret = 0;
 200
 201     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 202         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 203         ret |= (1 << DIRTY_MEMORY_VGA);
 204     }
 205     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 206         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 207         ret |= (1 << DIRTY_MEMORY_CODE);
 208     }
 209     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 210         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 211         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 212     }
 213     return ret;
 214 }
 215
 216 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 217                                                       unsigned client)
 218 {
 219     unsigned long page, idx, offset;
 220     DirtyMemoryBlocks *blocks;
 221
 222     assert(client < DIRTY_MEMORY_NUM);
 223
 224     page = addr >> TARGET_PAGE_BITS;
 225     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 226     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 227
 228     rcu_read_lock();
 229
 230     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 231
 232     set_bit_atomic(offset, blocks->blocks[idx]);
 233
 234     rcu_read_unlock();
 235 }
 236
 237 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 238                                                        ram_addr_t length,
 239                                                        uint8_t mask)
 240 {
 241     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 242     unsigned long end, page;
 243     unsigned long idx, offset, base;
 244     int i;
 245
 246     if (!mask && !xen_enabled()) {
 247         return;
 248     }
 249
 250     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 251     page = start >> TARGET_PAGE_BITS;
 252
 253     rcu_read_lock();
 254
 255     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 256         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 257     }
 258
 259     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 260     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 261     base = page - offset;
 262     while (page < end) {
 263         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 264
 265         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 266             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 267                               offset, next - page);
 268         }
 269         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 270             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 271                               offset, next - page);
 272         }
 273         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 274             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 275                               offset, next - page);
 276         }
 277
 278         page = next;
 279         idx++;
 280         offset = 0;
 281         base += DIRTY_MEMORY_BLOCK_SIZE;
 282     }
 283
 284     rcu_read_unlock();
 285
 286     xen_hvm_modified_memory(start, length);
 287 }
 288
 289 #if !defined(_WIN32)
 290 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 291                                                           ram_addr_t start,
 292                                                           ram_addr_t pages)
 293 {
 294     unsigned long i, j;
 295     unsigned long page_number, c;
 296     hwaddr addr;
 297     ram_addr_t ram_addr;
 298     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 299     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 300     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 301
 302     /* start address is aligned at the start of a word? */
 303     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 304         (hpratio == 1)) {
 305         unsigned long **blocks[DIRTY_MEMORY_NUM];
 306         unsigned long idx;
 307         unsigned long offset;
 308         long k;
 309         long nr = BITS_TO_LONGS(pages);
 310
 311         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 312         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 313                           DIRTY_MEMORY_BLOCK_SIZE);
 314
 315         rcu_read_lock();
 316
 317         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 318             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 319         }
 320
 321         for (k = 0; k < nr; k++) {
 322             if (bitmap[k]) {
 323                 unsigned long temp = leul_to_cpu(bitmap[k]);
 324
 325                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 326                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 327                 if (tcg_enabled()) {
 328                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 329                 }
 330             }
 331
 332             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 333                 offset = 0;
 334                 idx++;
 335             }
 336         }
 337
 338         rcu_read_unlock();
 339
 340         xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
 341     } else {
 342         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 343         /*
 344          * bitmap-traveling is faster than memory-traveling (for addr...)
 345          * especially when most of the memory is not dirty.
 346          */
 347         for (i = 0; i < len; i++) {
 348             if (bitmap[i] != 0) {
 349                 c = leul_to_cpu(bitmap[i]);
 350                 do {
 351                     j = ctzl(c);
 352                     c &= ~(1ul << j);
 353                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 354                     addr = page_number * TARGET_PAGE_SIZE;
 355                     ram_addr = start + addr;
 356                     cpu_physical_memory_set_dirty_range(ram_addr,
 357                                        TARGET_PAGE_SIZE * hpratio, clients);
 358                 } while (c != 0);
 359             }
 360         }
 361     }
 362 }
 363 #endif /* not _WIN32 */
 364
 365 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 366                                               ram_addr_t length,
 367                                               unsigned client);
 368
 369 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
 370     (ram_addr_t start, ram_addr_t length, unsigned client);
 371
 372 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
 373                                             ram_addr_t start,
 374                                             ram_addr_t length);
 375
 376 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 377                                                          ram_addr_t length)
 378 {
 379     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 380     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 381     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 382 }
 383
 384
 385 static inline
 386 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
 387                                                ram_addr_t start,
 388                                                ram_addr_t length,
 389                                                uint64_t *real_dirty_pages)
 390 {
 391     ram_addr_t addr;
 392     unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
 393     uint64_t num_dirty = 0;
 394     unsigned long *dest = rb->bmap;
 395
 396     /* start address and length is aligned at the start of a word? */
 397     if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
 398          (start + rb->offset) &&
 399         !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
 400         int k;
 401         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 402         unsigned long * const *src;
 403         unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 404         unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
 405                                         DIRTY_MEMORY_BLOCK_SIZE);
 406         unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 407
 408         rcu_read_lock();
 409
 410         src = atomic_rcu_read(
 411                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 412
 413         for (k = page; k < page + nr; k++) {
 414             if (src[idx][offset]) {
 415                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 416                 unsigned long new_dirty;
 417                 *real_dirty_pages += ctpopl(bits);
 418                 new_dirty = ~dest[k];
 419                 dest[k] |= bits;
 420                 new_dirty &= bits;
 421                 num_dirty += ctpopl(new_dirty);
 422             }
 423
 424             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 425                 offset = 0;
 426                 idx++;
 427             }
 428         }
 429
 430         rcu_read_unlock();
 431     } else {
 432         ram_addr_t offset = rb->offset;
 433
 434         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 435             if (cpu_physical_memory_test_and_clear_dirty(
 436                         start + addr + offset,
 437                         TARGET_PAGE_SIZE,
 438                         DIRTY_MEMORY_MIGRATION)) {
 439                 *real_dirty_pages += 1;
 440                 long k = (start + addr) >> TARGET_PAGE_BITS;
 441                 if (!test_and_set_bit(k, dest)) {
 442                     num_dirty++;
 443                 }
 444             }
 445         }
 446     }
 447
 448     return num_dirty;
 449 }
 450 #endif
 451 #endif