include/exec/ram_addr.h

   1 /*
   2  * Declarations for cpu physical memory functions
   3  *
   4  * Copyright 2011 Red Hat, Inc. and/or its affiliates
   5  *
   6  * Authors:
   7  *  Avi Kivity <avi@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or
  10  * later.  See the COPYING file in the top-level directory.
  11  *
  12  */
  13
  14 /*
  15  * This header is for use by exec.c and memory.c ONLY.  Do not include it.
  16  * The functions declared here will be removed soon.
  17  */
  18
  19 #ifndef RAM_ADDR_H
  20 #define RAM_ADDR_H
  21
  22 #ifndef CONFIG_USER_ONLY
  23 #include "hw/xen/xen.h"
  24 #include "exec/ramlist.h"
  25
  26 struct RAMBlock {
  27     struct rcu_head rcu;
  28     struct MemoryRegion *mr;
  29     uint8_t *host;
  30     ram_addr_t offset;
  31     ram_addr_t used_length;
  32     ram_addr_t max_length;
  33     void (*resized)(const char*, uint64_t length, void *host);
  34     uint32_t flags;
  35     /* Protected by iothread lock.  */
  36     char idstr[256];
  37     /* RCU-enabled, writes protected by the ramlist lock */
  38     QLIST_ENTRY(RAMBlock) next;
  39     QLIST_HEAD(, RAMBlockNotifier) ramblock_notifiers;
  40     int fd;
  41     size_t page_size;
  42     /* dirty bitmap used during migration */
  43     unsigned long *bmap;
  44     /* bitmap of pages that haven't been sent even once
  45      * only maintained and used in postcopy at the moment
  46      * where it's used to send the dirtymap at the start
  47      * of the postcopy phase
  48      */
  49     unsigned long *unsentmap;
  50     /* bitmap of already received pages in postcopy */
  51     unsigned long *receivedmap;
  52 };
  53
  54 static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
  55 {
  56     return (b && b->host && offset < b->used_length) ? true : false;
  57 }
  58
  59 static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
  60 {
  61     assert(offset_in_ramblock(block, offset));
  62     return (char *)block->host + offset;
  63 }
  64
  65 static inline unsigned long int ramblock_recv_bitmap_offset(void *host_addr,
  66                                                             RAMBlock *rb)
  67 {
  68     uint64_t host_addr_offset =
  69             (uint64_t)(uintptr_t)(host_addr - (void *)rb->host);
  70     return host_addr_offset >> TARGET_PAGE_BITS;
  71 }
  72
  73 bool ramblock_is_pmem(RAMBlock *rb);
  74
  75 long qemu_getrampagesize(void);
  76
  77 /**
  78  * qemu_ram_alloc_from_file,
  79  * qemu_ram_alloc_from_fd:  Allocate a ram block from the specified backing
  80  *                          file or device
  81  *
  82  * Parameters:
  83  *  @size: the size in bytes of the ram block
  84  *  @mr: the memory region where the ram block is
  85  *  @ram_flags: specify the properties of the ram block, which can be one
  86  *              or bit-or of following values
  87  *              - RAM_SHARED: mmap the backing file or device with MAP_SHARED
  88  *              - RAM_PMEM: the backend @mem_path or @fd is persistent memory
  89  *              Other bits are ignored.
  90  *  @mem_path or @fd: specify the backing file or device
  91  *  @errp: pointer to Error*, to store an error if it happens
  92  *
  93  * Return:
  94  *  On success, return a pointer to the ram block.
  95  *  On failure, return NULL.
  96  */
  97 RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
  98                                    uint32_t ram_flags, const char *mem_path,
  99                                    Error **errp);
 100 RAMBlock *qemu_ram_alloc_from_fd(ram_addr_t size, MemoryRegion *mr,
 101                                  uint32_t ram_flags, int fd,
 102                                  Error **errp);
 103
 104 RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
 105                                   MemoryRegion *mr, Error **errp);
 106 RAMBlock *qemu_ram_alloc(ram_addr_t size, bool share, MemoryRegion *mr,
 107                          Error **errp);
 108 RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
 109                                     void (*resized)(const char*,
 110                                                     uint64_t length,
 111                                                     void *host),
 112                                     MemoryRegion *mr, Error **errp);
 113 void qemu_ram_free(RAMBlock *block);
 114
 115 int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
 116
 117 #define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
 118 #define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
 119
 120 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end);
 121
 122 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
 123                                                  ram_addr_t length,
 124                                                  unsigned client)
 125 {
 126     DirtyMemoryBlocks *blocks;
 127     unsigned long end, page;
 128     unsigned long idx, offset, base;
 129     bool dirty = false;
 130
 131     assert(client < DIRTY_MEMORY_NUM);
 132
 133     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 134     page = start >> TARGET_PAGE_BITS;
 135
 136     rcu_read_lock();
 137
 138     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 139
 140     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 141     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 142     base = page - offset;
 143     while (page < end) {
 144         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 145         unsigned long num = next - base;
 146         unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
 147         if (found < num) {
 148             dirty = true;
 149             break;
 150         }
 151
 152         page = next;
 153         idx++;
 154         offset = 0;
 155         base += DIRTY_MEMORY_BLOCK_SIZE;
 156     }
 157
 158     rcu_read_unlock();
 159
 160     return dirty;
 161 }
 162
 163 static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
 164                                                  ram_addr_t length,
 165                                                  unsigned client)
 166 {
 167     DirtyMemoryBlocks *blocks;
 168     unsigned long end, page;
 169     unsigned long idx, offset, base;
 170     bool dirty = true;
 171
 172     assert(client < DIRTY_MEMORY_NUM);
 173
 174     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 175     page = start >> TARGET_PAGE_BITS;
 176
 177     rcu_read_lock();
 178
 179     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 180
 181     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 182     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 183     base = page - offset;
 184     while (page < end) {
 185         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 186         unsigned long num = next - base;
 187         unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
 188         if (found < num) {
 189             dirty = false;
 190             break;
 191         }
 192
 193         page = next;
 194         idx++;
 195         offset = 0;
 196         base += DIRTY_MEMORY_BLOCK_SIZE;
 197     }
 198
 199     rcu_read_unlock();
 200
 201     return dirty;
 202 }
 203
 204 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
 205                                                       unsigned client)
 206 {
 207     return cpu_physical_memory_get_dirty(addr, 1, client);
 208 }
 209
 210 static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
 211 {
 212     bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
 213     bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
 214     bool migration =
 215         cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
 216     return !(vga && code && migration);
 217 }
 218
 219 static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
 220                                                                ram_addr_t length,
 221                                                                uint8_t mask)
 222 {
 223     uint8_t ret = 0;
 224
 225     if (mask & (1 << DIRTY_MEMORY_VGA) &&
 226         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
 227         ret |= (1 << DIRTY_MEMORY_VGA);
 228     }
 229     if (mask & (1 << DIRTY_MEMORY_CODE) &&
 230         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
 231         ret |= (1 << DIRTY_MEMORY_CODE);
 232     }
 233     if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
 234         !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
 235         ret |= (1 << DIRTY_MEMORY_MIGRATION);
 236     }
 237     return ret;
 238 }
 239
 240 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
 241                                                       unsigned client)
 242 {
 243     unsigned long page, idx, offset;
 244     DirtyMemoryBlocks *blocks;
 245
 246     assert(client < DIRTY_MEMORY_NUM);
 247
 248     page = addr >> TARGET_PAGE_BITS;
 249     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 250     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 251
 252     rcu_read_lock();
 253
 254     blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
 255
 256     set_bit_atomic(offset, blocks->blocks[idx]);
 257
 258     rcu_read_unlock();
 259 }
 260
 261 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
 262                                                        ram_addr_t length,
 263                                                        uint8_t mask)
 264 {
 265     DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
 266     unsigned long end, page;
 267     unsigned long idx, offset, base;
 268     int i;
 269
 270     if (!mask && !xen_enabled()) {
 271         return;
 272     }
 273
 274     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
 275     page = start >> TARGET_PAGE_BITS;
 276
 277     rcu_read_lock();
 278
 279     for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 280         blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
 281     }
 282
 283     idx = page / DIRTY_MEMORY_BLOCK_SIZE;
 284     offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 285     base = page - offset;
 286     while (page < end) {
 287         unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
 288
 289         if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
 290             bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
 291                               offset, next - page);
 292         }
 293         if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
 294             bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
 295                               offset, next - page);
 296         }
 297         if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
 298             bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
 299                               offset, next - page);
 300         }
 301
 302         page = next;
 303         idx++;
 304         offset = 0;
 305         base += DIRTY_MEMORY_BLOCK_SIZE;
 306     }
 307
 308     rcu_read_unlock();
 309
 310     xen_hvm_modified_memory(start, length);
 311 }
 312
 313 #if !defined(_WIN32)
 314 static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
 315                                                           ram_addr_t start,
 316                                                           ram_addr_t pages)
 317 {
 318     unsigned long i, j;
 319     unsigned long page_number, c;
 320     hwaddr addr;
 321     ram_addr_t ram_addr;
 322     unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 323     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 324     unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 325
 326     /* start address is aligned at the start of a word? */
 327     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
 328         (hpratio == 1)) {
 329         unsigned long **blocks[DIRTY_MEMORY_NUM];
 330         unsigned long idx;
 331         unsigned long offset;
 332         long k;
 333         long nr = BITS_TO_LONGS(pages);
 334
 335         idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
 336         offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
 337                           DIRTY_MEMORY_BLOCK_SIZE);
 338
 339         rcu_read_lock();
 340
 341         for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
 342             blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
 343         }
 344
 345         for (k = 0; k < nr; k++) {
 346             if (bitmap[k]) {
 347                 unsigned long temp = leul_to_cpu(bitmap[k]);
 348
 349                 atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
 350                 atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
 351                 if (tcg_enabled()) {
 352                     atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
 353                 }
 354             }
 355
 356             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 357                 offset = 0;
 358                 idx++;
 359             }
 360         }
 361
 362         rcu_read_unlock();
 363
 364         xen_hvm_modified_memory(start, pages << TARGET_PAGE_BITS);
 365     } else {
 366         uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
 367         /*
 368          * bitmap-traveling is faster than memory-traveling (for addr...)
 369          * especially when most of the memory is not dirty.
 370          */
 371         for (i = 0; i < len; i++) {
 372             if (bitmap[i] != 0) {
 373                 c = leul_to_cpu(bitmap[i]);
 374                 do {
 375                     j = ctzl(c);
 376                     c &= ~(1ul << j);
 377                     page_number = (i * HOST_LONG_BITS + j) * hpratio;
 378                     addr = page_number * TARGET_PAGE_SIZE;
 379                     ram_addr = start + addr;
 380                     cpu_physical_memory_set_dirty_range(ram_addr,
 381                                        TARGET_PAGE_SIZE * hpratio, clients);
 382                 } while (c != 0);
 383             }
 384         }
 385     }
 386 }
 387 #endif /* not _WIN32 */
 388
 389 bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
 390                                               ram_addr_t length,
 391                                               unsigned client);
 392
 393 DirtyBitmapSnapshot *cpu_physical_memory_snapshot_and_clear_dirty
 394     (ram_addr_t start, ram_addr_t length, unsigned client);
 395
 396 bool cpu_physical_memory_snapshot_get_dirty(DirtyBitmapSnapshot *snap,
 397                                             ram_addr_t start,
 398                                             ram_addr_t length);
 399
 400 static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
 401                                                          ram_addr_t length)
 402 {
 403     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
 404     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
 405     cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 406 }
 407
 408
 409 static inline
 410 uint64_t cpu_physical_memory_sync_dirty_bitmap(RAMBlock *rb,
 411                                                ram_addr_t start,
 412                                                ram_addr_t length,
 413                                                uint64_t *real_dirty_pages)
 414 {
 415     ram_addr_t addr;
 416     unsigned long word = BIT_WORD((start + rb->offset) >> TARGET_PAGE_BITS);
 417     uint64_t num_dirty = 0;
 418     unsigned long *dest = rb->bmap;
 419
 420     /* start address and length is aligned at the start of a word? */
 421     if (((word * BITS_PER_LONG) << TARGET_PAGE_BITS) ==
 422          (start + rb->offset) &&
 423         !(length & ((BITS_PER_LONG << TARGET_PAGE_BITS) - 1))) {
 424         int k;
 425         int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
 426         unsigned long * const *src;
 427         unsigned long idx = (word * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
 428         unsigned long offset = BIT_WORD((word * BITS_PER_LONG) %
 429                                         DIRTY_MEMORY_BLOCK_SIZE);
 430         unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
 431
 432         rcu_read_lock();
 433
 434         src = atomic_rcu_read(
 435                 &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
 436
 437         for (k = page; k < page + nr; k++) {
 438             if (src[idx][offset]) {
 439                 unsigned long bits = atomic_xchg(&src[idx][offset], 0);
 440                 unsigned long new_dirty;
 441                 *real_dirty_pages += ctpopl(bits);
 442                 new_dirty = ~dest[k];
 443                 dest[k] |= bits;
 444                 new_dirty &= bits;
 445                 num_dirty += ctpopl(new_dirty);
 446             }
 447
 448             if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
 449                 offset = 0;
 450                 idx++;
 451             }
 452         }
 453
 454         rcu_read_unlock();
 455     } else {
 456         ram_addr_t offset = rb->offset;
 457
 458         for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
 459             if (cpu_physical_memory_test_and_clear_dirty(
 460                         start + addr + offset,
 461                         TARGET_PAGE_SIZE,
 462                         DIRTY_MEMORY_MIGRATION)) {
 463                 *real_dirty_pages += 1;
 464                 long k = (start + addr) >> TARGET_PAGE_BITS;
 465                 if (!test_and_set_bit(k, dest)) {
 466                     num_dirty++;
 467                 }
 468             }
 469         }
 470     }
 471
 472     return num_dirty;
 473 }
 474 #endif
 475 #endif