migration/migration.c

   1 /*
   2  * QEMU live migration
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *
   6  * Authors:
   7  *  Anthony Liguori   <aliguori@us.ibm.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2.  See
  10  * the COPYING file in the top-level directory.
  11  *
  12  * Contributions after 2012-01-13 are licensed under the terms of the
  13  * GNU GPL, version 2 or (at your option) any later version.
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include "qemu/cutils.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/main-loop.h"
  20 #include "migration/blocker.h"
  21 #include "exec.h"
  22 #include "fd.h"
  23 #include "socket.h"
  24 #include "sysemu/runstate.h"
  25 #include "sysemu/sysemu.h"
  26 #include "rdma.h"
  27 #include "ram.h"
  28 #include "migration/global_state.h"
  29 #include "migration/misc.h"
  30 #include "migration.h"
  31 #include "savevm.h"
  32 #include "qemu-file-channel.h"
  33 #include "qemu-file.h"
  34 #include "migration/vmstate.h"
  35 #include "block/block.h"
  36 #include "qapi/error.h"
  37 #include "qapi/clone-visitor.h"
  38 #include "qapi/qapi-visit-sockets.h"
  39 #include "qapi/qapi-commands-migration.h"
  40 #include "qapi/qapi-events-migration.h"
  41 #include "qapi/qmp/qerror.h"
  42 #include "qapi/qmp/qnull.h"
  43 #include "qemu/rcu.h"
  44 #include "block.h"
  45 #include "postcopy-ram.h"
  46 #include "qemu/thread.h"
  47 #include "trace.h"
  48 #include "exec/target_page.h"
  49 #include "io/channel-buffer.h"
  50 #include "migration/colo.h"
  51 #include "hw/boards.h"
  52 #include "hw/qdev-properties.h"
  53 #include "monitor/monitor.h"
  54 #include "net/announce.h"
  55 #include "qemu/queue.h"
  56 #include "multifd.h"
  57
  58 #define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
  59
  60 /* Amount of time to allocate to each "chunk" of bandwidth-throttled
  61  * data. */
  62 #define BUFFER_DELAY     100
  63 #define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
  64
  65 /* Time in milliseconds we are allowed to stop the source,
  66  * for sending the last part */
  67 #define DEFAULT_MIGRATE_SET_DOWNTIME 300
  68
  69 /* Maximum migrate downtime set to 2000 seconds */
  70 #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
  71 #define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000)
  72
  73 /* Default compression thread count */
  74 #define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
  75 /* Default decompression thread count, usually decompression is at
  76  * least 4 times as fast as compression.*/
  77 #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
  78 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */
  79 #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
  80 /* Define default autoconverge cpu throttle migration parameters */
  81 #define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50
  82 #define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
  83 #define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
  84 #define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99
  85
  86 /* Migration XBZRLE default cache size */
  87 #define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
  88
  89 /* The delay time (in ms) between two COLO checkpoints */
  90 #define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
  91 #define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
  92 #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE
  93 /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */
  94 #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1
  95 /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */
  96 #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1
  97
  98 /* Background transfer rate for postcopy, 0 means unlimited, note
  99  * that page requests can still exceed this limit.
 100  */
 101 #define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0
 102
 103 /*
 104  * Parameters for self_announce_delay giving a stream of RARP/ARP
 105  * packets after migration.
 106  */
 107 #define DEFAULT_MIGRATE_ANNOUNCE_INITIAL  50
 108 #define DEFAULT_MIGRATE_ANNOUNCE_MAX     550
 109 #define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS    5
 110 #define DEFAULT_MIGRATE_ANNOUNCE_STEP    100
 111
 112 static NotifierList migration_state_notifiers =
 113     NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
 114
 115 static bool deferred_incoming;
 116
 117 /* Messages sent on the return path from destination to source */
 118 enum mig_rp_message_type {
 119     MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
 120     MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
 121     MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
 122
 123     MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
 124     MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
 125     MIG_RP_MSG_RECV_BITMAP,  /* send recved_bitmap back to source */
 126     MIG_RP_MSG_RESUME_ACK,   /* tell source that we are ready to resume */
 127
 128     MIG_RP_MSG_MAX
 129 };
 130
 131 /* When we add fault tolerance, we could have several
 132    migrations at once.  For now we don't need to add
 133    dynamic creation of migration */
 134
 135 static MigrationState *current_migration;
 136 static MigrationIncomingState *current_incoming;
 137
 138 static bool migration_object_check(MigrationState *ms, Error **errp);
 139 static int migration_maybe_pause(MigrationState *s,
 140                                  int *current_active_state,
 141                                  int new_state);
 142 static void migrate_fd_cancel(MigrationState *s);
 143
 144 void migration_object_init(void)
 145 {
 146     MachineState *ms = MACHINE(qdev_get_machine());
 147     Error *err = NULL;
 148
 149     /* This can only be called once. */
 150     assert(!current_migration);
 151     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 152
 153     /*
 154      * Init the migrate incoming object as well no matter whether
 155      * we'll use it or not.
 156      */
 157     assert(!current_incoming);
 158     current_incoming = g_new0(MigrationIncomingState, 1);
 159     current_incoming->state = MIGRATION_STATUS_NONE;
 160     current_incoming->postcopy_remote_fds =
 161         g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD));
 162     qemu_mutex_init(&current_incoming->rp_mutex);
 163     qemu_event_init(&current_incoming->main_thread_load_event, false);
 164     qemu_sem_init(&current_incoming->postcopy_pause_sem_dst, 0);
 165     qemu_sem_init(&current_incoming->postcopy_pause_sem_fault, 0);
 166
 167     init_dirty_bitmap_incoming_migration();
 168
 169     if (!migration_object_check(current_migration, &err)) {
 170         error_report_err(err);
 171         exit(1);
 172     }
 173
 174     /*
 175      * We cannot really do this in migration_instance_init() since at
 176      * that time global properties are not yet applied, then this
 177      * value will be definitely replaced by something else.
 178      */
 179     if (ms->enforce_config_section) {
 180         current_migration->send_configuration = true;
 181     }
 182 }
 183
 184 void migration_shutdown(void)
 185 {
 186     /*
 187      * Cancel the current migration - that will (eventually)
 188      * stop the migration using this structure
 189      */
 190     migrate_fd_cancel(current_migration);
 191     object_unref(OBJECT(current_migration));
 192 }
 193
 194 /* For outgoing */
 195 MigrationState *migrate_get_current(void)
 196 {
 197     /* This can only be called after the object created. */
 198     assert(current_migration);
 199     return current_migration;
 200 }
 201
 202 MigrationIncomingState *migration_incoming_get_current(void)
 203 {
 204     assert(current_incoming);
 205     return current_incoming;
 206 }
 207
 208 void migration_incoming_state_destroy(void)
 209 {
 210     struct MigrationIncomingState *mis = migration_incoming_get_current();
 211
 212     if (mis->to_src_file) {
 213         /* Tell source that we are done */
 214         migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
 215         qemu_fclose(mis->to_src_file);
 216         mis->to_src_file = NULL;
 217     }
 218
 219     if (mis->from_src_file) {
 220         qemu_fclose(mis->from_src_file);
 221         mis->from_src_file = NULL;
 222     }
 223     if (mis->postcopy_remote_fds) {
 224         g_array_free(mis->postcopy_remote_fds, TRUE);
 225         mis->postcopy_remote_fds = NULL;
 226     }
 227
 228     qemu_event_reset(&mis->main_thread_load_event);
 229
 230     if (mis->socket_address_list) {
 231         qapi_free_SocketAddressList(mis->socket_address_list);
 232         mis->socket_address_list = NULL;
 233     }
 234 }
 235
 236 static void migrate_generate_event(int new_state)
 237 {
 238     if (migrate_use_events()) {
 239         qapi_event_send_migration(new_state);
 240     }
 241 }
 242
 243 static bool migrate_late_block_activate(void)
 244 {
 245     MigrationState *s;
 246
 247     s = migrate_get_current();
 248
 249     return s->enabled_capabilities[
 250         MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE];
 251 }
 252
 253 /*
 254  * Called on -incoming with a defer: uri.
 255  * The migration can be started later after any parameters have been
 256  * changed.
 257  */
 258 static void deferred_incoming_migration(Error **errp)
 259 {
 260     if (deferred_incoming) {
 261         error_setg(errp, "Incoming migration already deferred");
 262     }
 263     deferred_incoming = true;
 264 }
 265
 266 /*
 267  * Send a message on the return channel back to the source
 268  * of the migration.
 269  */
 270 static int migrate_send_rp_message(MigrationIncomingState *mis,
 271                                    enum mig_rp_message_type message_type,
 272                                    uint16_t len, void *data)
 273 {
 274     int ret = 0;
 275
 276     trace_migrate_send_rp_message((int)message_type, len);
 277     qemu_mutex_lock(&mis->rp_mutex);
 278
 279     /*
 280      * It's possible that the file handle got lost due to network
 281      * failures.
 282      */
 283     if (!mis->to_src_file) {
 284         ret = -EIO;
 285         goto error;
 286     }
 287
 288     qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
 289     qemu_put_be16(mis->to_src_file, len);
 290     qemu_put_buffer(mis->to_src_file, data, len);
 291     qemu_fflush(mis->to_src_file);
 292
 293     /* It's possible that qemu file got error during sending */
 294     ret = qemu_file_get_error(mis->to_src_file);
 295
 296 error:
 297     qemu_mutex_unlock(&mis->rp_mutex);
 298     return ret;
 299 }
 300
 301 /* Request a range of pages from the source VM at the given
 302  * start address.
 303  *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
 304  *           as the last request (a name must have been given previously)
 305  *   Start: Address offset within the RB
 306  *   Len: Length in bytes required - must be a multiple of pagesize
 307  */
 308 int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
 309                               ram_addr_t start, size_t len)
 310 {
 311     uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
 312     size_t msglen = 12; /* start + len */
 313     enum mig_rp_message_type msg_type;
 314
 315     *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
 316     *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
 317
 318     if (rbname) {
 319         int rbname_len = strlen(rbname);
 320         assert(rbname_len < 256);
 321
 322         bufc[msglen++] = rbname_len;
 323         memcpy(bufc + msglen, rbname, rbname_len);
 324         msglen += rbname_len;
 325         msg_type = MIG_RP_MSG_REQ_PAGES_ID;
 326     } else {
 327         msg_type = MIG_RP_MSG_REQ_PAGES;
 328     }
 329
 330     return migrate_send_rp_message(mis, msg_type, msglen, bufc);
 331 }
 332
 333 static bool migration_colo_enabled;
 334 bool migration_incoming_colo_enabled(void)
 335 {
 336     return migration_colo_enabled;
 337 }
 338
 339 void migration_incoming_disable_colo(void)
 340 {
 341     migration_colo_enabled = false;
 342 }
 343
 344 void migration_incoming_enable_colo(void)
 345 {
 346     migration_colo_enabled = true;
 347 }
 348
 349 void migrate_add_address(SocketAddress *address)
 350 {
 351     MigrationIncomingState *mis = migration_incoming_get_current();
 352     SocketAddressList *addrs;
 353
 354     addrs = g_new0(SocketAddressList, 1);
 355     addrs->next = mis->socket_address_list;
 356     mis->socket_address_list = addrs;
 357     addrs->value = QAPI_CLONE(SocketAddress, address);
 358 }
 359
 360 void qemu_start_incoming_migration(const char *uri, Error **errp)
 361 {
 362     const char *p;
 363
 364     qapi_event_send_migration(MIGRATION_STATUS_SETUP);
 365     if (!strcmp(uri, "defer")) {
 366         deferred_incoming_migration(errp);
 367     } else if (strstart(uri, "tcp:", &p)) {
 368         tcp_start_incoming_migration(p, errp);
 369 #ifdef CONFIG_RDMA
 370     } else if (strstart(uri, "rdma:", &p)) {
 371         rdma_start_incoming_migration(p, errp);
 372 #endif
 373     } else if (strstart(uri, "exec:", &p)) {
 374         exec_start_incoming_migration(p, errp);
 375     } else if (strstart(uri, "unix:", &p)) {
 376         unix_start_incoming_migration(p, errp);
 377     } else if (strstart(uri, "fd:", &p)) {
 378         fd_start_incoming_migration(p, errp);
 379     } else {
 380         error_setg(errp, "unknown migration protocol: %s", uri);
 381     }
 382 }
 383
 384 static void process_incoming_migration_bh(void *opaque)
 385 {
 386     Error *local_err = NULL;
 387     MigrationIncomingState *mis = opaque;
 388
 389     /* If capability late_block_activate is set:
 390      * Only fire up the block code now if we're going to restart the
 391      * VM, else 'cont' will do it.
 392      * This causes file locking to happen; so we don't want it to happen
 393      * unless we really are starting the VM.
 394      */
 395     if (!migrate_late_block_activate() ||
 396          (autostart && (!global_state_received() ||
 397             global_state_get_runstate() == RUN_STATE_RUNNING))) {
 398         /* Make sure all file formats flush their mutable metadata.
 399          * If we get an error here, just don't restart the VM yet. */
 400         bdrv_invalidate_cache_all(&local_err);
 401         if (local_err) {
 402             error_report_err(local_err);
 403             local_err = NULL;
 404             autostart = false;
 405         }
 406     }
 407
 408     /*
 409      * This must happen after all error conditions are dealt with and
 410      * we're sure the VM is going to be running on this host.
 411      */
 412     qemu_announce_self(&mis->announce_timer, migrate_announce_params());
 413
 414     if (multifd_load_cleanup(&local_err) != 0) {
 415         error_report_err(local_err);
 416         autostart = false;
 417     }
 418     /* If global state section was not received or we are in running
 419        state, we need to obey autostart. Any other state is set with
 420        runstate_set. */
 421
 422     dirty_bitmap_mig_before_vm_start();
 423
 424     if (!global_state_received() ||
 425         global_state_get_runstate() == RUN_STATE_RUNNING) {
 426         if (autostart) {
 427             vm_start();
 428         } else {
 429             runstate_set(RUN_STATE_PAUSED);
 430         }
 431     } else if (migration_incoming_colo_enabled()) {
 432         migration_incoming_disable_colo();
 433         vm_start();
 434     } else {
 435         runstate_set(global_state_get_runstate());
 436     }
 437     /*
 438      * This must happen after any state changes since as soon as an external
 439      * observer sees this event they might start to prod at the VM assuming
 440      * it's ready to use.
 441      */
 442     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 443                       MIGRATION_STATUS_COMPLETED);
 444     qemu_bh_delete(mis->bh);
 445     migration_incoming_state_destroy();
 446 }
 447
 448 static void process_incoming_migration_co(void *opaque)
 449 {
 450     MigrationIncomingState *mis = migration_incoming_get_current();
 451     PostcopyState ps;
 452     int ret;
 453     Error *local_err = NULL;
 454
 455     assert(mis->from_src_file);
 456     mis->migration_incoming_co = qemu_coroutine_self();
 457     mis->largest_page_size = qemu_ram_pagesize_largest();
 458     postcopy_state_set(POSTCOPY_INCOMING_NONE);
 459     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
 460                       MIGRATION_STATUS_ACTIVE);
 461     ret = qemu_loadvm_state(mis->from_src_file);
 462
 463     ps = postcopy_state_get();
 464     trace_process_incoming_migration_co_end(ret, ps);
 465     if (ps != POSTCOPY_INCOMING_NONE) {
 466         if (ps == POSTCOPY_INCOMING_ADVISE) {
 467             /*
 468              * Where a migration had postcopy enabled (and thus went to advise)
 469              * but managed to complete within the precopy period, we can use
 470              * the normal exit.
 471              */
 472             postcopy_ram_incoming_cleanup(mis);
 473         } else if (ret >= 0) {
 474             /*
 475              * Postcopy was started, cleanup should happen at the end of the
 476              * postcopy thread.
 477              */
 478             trace_process_incoming_migration_co_postcopy_end_main();
 479             return;
 480         }
 481         /* Else if something went wrong then just fall out of the normal exit */
 482     }
 483
 484     /* we get COLO info, and know if we are in COLO mode */
 485     if (!ret && migration_incoming_colo_enabled()) {
 486         /* Make sure all file formats flush their mutable metadata */
 487         bdrv_invalidate_cache_all(&local_err);
 488         if (local_err) {
 489             error_report_err(local_err);
 490             goto fail;
 491         }
 492
 493         qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
 494              colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
 495         mis->have_colo_incoming_thread = true;
 496         qemu_coroutine_yield();
 497
 498         /* Wait checkpoint incoming thread exit before free resource */
 499         qemu_thread_join(&mis->colo_incoming_thread);
 500         /* We hold the global iothread lock, so it is safe here */
 501         colo_release_ram_cache();
 502     }
 503
 504     if (ret < 0) {
 505         error_report("load of migration failed: %s", strerror(-ret));
 506         goto fail;
 507     }
 508     mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
 509     qemu_bh_schedule(mis->bh);
 510     mis->migration_incoming_co = NULL;
 511     return;
 512 fail:
 513     local_err = NULL;
 514     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
 515                       MIGRATION_STATUS_FAILED);
 516     qemu_fclose(mis->from_src_file);
 517     if (multifd_load_cleanup(&local_err) != 0) {
 518         error_report_err(local_err);
 519     }
 520     exit(EXIT_FAILURE);
 521 }
 522
 523 /**
 524  * @migration_incoming_setup: Setup incoming migration
 525  *
 526  * Returns 0 for no error or 1 for error
 527  *
 528  * @f: file for main migration channel
 529  * @errp: where to put errors
 530  */
 531 static int migration_incoming_setup(QEMUFile *f, Error **errp)
 532 {
 533     MigrationIncomingState *mis = migration_incoming_get_current();
 534     Error *local_err = NULL;
 535
 536     if (multifd_load_setup(&local_err) != 0) {
 537         /* We haven't been able to create multifd threads
 538            nothing better to do */
 539         error_report_err(local_err);
 540         exit(EXIT_FAILURE);
 541     }
 542
 543     if (!mis->from_src_file) {
 544         mis->from_src_file = f;
 545     }
 546     qemu_file_set_blocking(f, false);
 547     return 0;
 548 }
 549
 550 void migration_incoming_process(void)
 551 {
 552     Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL);
 553     qemu_coroutine_enter(co);
 554 }
 555
 556 /* Returns true if recovered from a paused migration, otherwise false */
 557 static bool postcopy_try_recover(QEMUFile *f)
 558 {
 559     MigrationIncomingState *mis = migration_incoming_get_current();
 560
 561     if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
 562         /* Resumed from a paused postcopy migration */
 563
 564         mis->from_src_file = f;
 565         /* Postcopy has standalone thread to do vm load */
 566         qemu_file_set_blocking(f, true);
 567
 568         /* Re-configure the return path */
 569         mis->to_src_file = qemu_file_get_return_path(f);
 570
 571         migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
 572                           MIGRATION_STATUS_POSTCOPY_RECOVER);
 573
 574         /*
 575          * Here, we only wake up the main loading thread (while the
 576          * fault thread will still be waiting), so that we can receive
 577          * commands from source now, and answer it if needed. The
 578          * fault thread will be woken up afterwards until we are sure
 579          * that source is ready to reply to page requests.
 580          */
 581         qemu_sem_post(&mis->postcopy_pause_sem_dst);
 582         return true;
 583     }
 584
 585     return false;
 586 }
 587
 588 void migration_fd_process_incoming(QEMUFile *f, Error **errp)
 589 {
 590     Error *local_err = NULL;
 591
 592     if (postcopy_try_recover(f)) {
 593         return;
 594     }
 595
 596     if (migration_incoming_setup(f, &local_err)) {
 597         if (local_err) {
 598             error_propagate(errp, local_err);
 599         }
 600         return;
 601     }
 602     migration_incoming_process();
 603 }
 604
 605 void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp)
 606 {
 607     MigrationIncomingState *mis = migration_incoming_get_current();
 608     Error *local_err = NULL;
 609     bool start_migration;
 610
 611     if (!mis->from_src_file) {
 612         /* The first connection (multifd may have multiple) */
 613         QEMUFile *f = qemu_fopen_channel_input(ioc);
 614
 615         /* If it's a recovery, we're done */
 616         if (postcopy_try_recover(f)) {
 617             return;
 618         }
 619
 620         if (migration_incoming_setup(f, &local_err)) {
 621             if (local_err) {
 622                 error_propagate(errp, local_err);
 623             }
 624             return;
 625         }
 626
 627         /*
 628          * Common migration only needs one channel, so we can start
 629          * right now.  Multifd needs more than one channel, we wait.
 630          */
 631         start_migration = !migrate_use_multifd();
 632     } else {
 633         /* Multiple connections */
 634         assert(migrate_use_multifd());
 635         start_migration = multifd_recv_new_channel(ioc, &local_err);
 636         if (local_err) {
 637             error_propagate(errp, local_err);
 638             return;
 639         }
 640     }
 641
 642     if (start_migration) {
 643         migration_incoming_process();
 644     }
 645 }
 646
 647 /**
 648  * @migration_has_all_channels: We have received all channels that we need
 649  *
 650  * Returns true when we have got connections to all the channels that
 651  * we need for migration.
 652  */
 653 bool migration_has_all_channels(void)
 654 {
 655     MigrationIncomingState *mis = migration_incoming_get_current();
 656     bool all_channels;
 657
 658     all_channels = multifd_recv_all_channels_created();
 659
 660     return all_channels && mis->from_src_file != NULL;
 661 }
 662
 663 /*
 664  * Send a 'SHUT' message on the return channel with the given value
 665  * to indicate that we've finished with the RP.  Non-0 value indicates
 666  * error.
 667  */
 668 void migrate_send_rp_shut(MigrationIncomingState *mis,
 669                           uint32_t value)
 670 {
 671     uint32_t buf;
 672
 673     buf = cpu_to_be32(value);
 674     migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
 675 }
 676
 677 /*
 678  * Send a 'PONG' message on the return channel with the given value
 679  * (normally in response to a 'PING')
 680  */
 681 void migrate_send_rp_pong(MigrationIncomingState *mis,
 682                           uint32_t value)
 683 {
 684     uint32_t buf;
 685
 686     buf = cpu_to_be32(value);
 687     migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
 688 }
 689
 690 void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 691                                  char *block_name)
 692 {
 693     char buf[512];
 694     int len;
 695     int64_t res;
 696
 697     /*
 698      * First, we send the header part. It contains only the len of
 699      * idstr, and the idstr itself.
 700      */
 701     len = strlen(block_name);
 702     buf[0] = len;
 703     memcpy(buf + 1, block_name, len);
 704
 705     if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) {
 706         error_report("%s: MSG_RP_RECV_BITMAP only used for recovery",
 707                      __func__);
 708         return;
 709     }
 710
 711     migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf);
 712
 713     /*
 714      * Next, we dump the received bitmap to the stream.
 715      *
 716      * TODO: currently we are safe since we are the only one that is
 717      * using the to_src_file handle (fault thread is still paused),
 718      * and it's ok even not taking the mutex. However the best way is
 719      * to take the lock before sending the message header, and release
 720      * the lock after sending the bitmap.
 721      */
 722     qemu_mutex_lock(&mis->rp_mutex);
 723     res = ramblock_recv_bitmap_send(mis->to_src_file, block_name);
 724     qemu_mutex_unlock(&mis->rp_mutex);
 725
 726     trace_migrate_send_rp_recv_bitmap(block_name, res);
 727 }
 728
 729 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value)
 730 {
 731     uint32_t buf;
 732
 733     buf = cpu_to_be32(value);
 734     migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf);
 735 }
 736
 737 MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
 738 {
 739     MigrationCapabilityStatusList *head = NULL;
 740     MigrationCapabilityStatusList *caps;
 741     MigrationState *s = migrate_get_current();
 742     int i;
 743
 744     caps = NULL; /* silence compiler warning */
 745     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
 746 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
 747         if (i == MIGRATION_CAPABILITY_BLOCK) {
 748             continue;
 749         }
 750 #endif
 751         if (head == NULL) {
 752             head = g_malloc0(sizeof(*caps));
 753             caps = head;
 754         } else {
 755             caps->next = g_malloc0(sizeof(*caps));
 756             caps = caps->next;
 757         }
 758         caps->value =
 759             g_malloc(sizeof(*caps->value));
 760         caps->value->capability = i;
 761         caps->value->state = s->enabled_capabilities[i];
 762     }
 763
 764     return head;
 765 }
 766
 767 MigrationParameters *qmp_query_migrate_parameters(Error **errp)
 768 {
 769     MigrationParameters *params;
 770     MigrationState *s = migrate_get_current();
 771
 772     /* TODO use QAPI_CLONE() instead of duplicating it inline */
 773     params = g_malloc0(sizeof(*params));
 774     params->has_compress_level = true;
 775     params->compress_level = s->parameters.compress_level;
 776     params->has_compress_threads = true;
 777     params->compress_threads = s->parameters.compress_threads;
 778     params->has_compress_wait_thread = true;
 779     params->compress_wait_thread = s->parameters.compress_wait_thread;
 780     params->has_decompress_threads = true;
 781     params->decompress_threads = s->parameters.decompress_threads;
 782     params->has_throttle_trigger_threshold = true;
 783     params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold;
 784     params->has_cpu_throttle_initial = true;
 785     params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
 786     params->has_cpu_throttle_increment = true;
 787     params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
 788     params->has_cpu_throttle_tailslow = true;
 789     params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow;
 790     params->has_tls_creds = true;
 791     params->tls_creds = g_strdup(s->parameters.tls_creds);
 792     params->has_tls_hostname = true;
 793     params->tls_hostname = g_strdup(s->parameters.tls_hostname);
 794     params->has_tls_authz = true;
 795     params->tls_authz = g_strdup(s->parameters.tls_authz ?
 796                                  s->parameters.tls_authz : "");
 797     params->has_max_bandwidth = true;
 798     params->max_bandwidth = s->parameters.max_bandwidth;
 799     params->has_downtime_limit = true;
 800     params->downtime_limit = s->parameters.downtime_limit;
 801     params->has_x_checkpoint_delay = true;
 802     params->x_checkpoint_delay = s->parameters.x_checkpoint_delay;
 803     params->has_block_incremental = true;
 804     params->block_incremental = s->parameters.block_incremental;
 805     params->has_multifd_channels = true;
 806     params->multifd_channels = s->parameters.multifd_channels;
 807     params->has_multifd_compression = true;
 808     params->multifd_compression = s->parameters.multifd_compression;
 809     params->has_multifd_zlib_level = true;
 810     params->multifd_zlib_level = s->parameters.multifd_zlib_level;
 811     params->has_multifd_zstd_level = true;
 812     params->multifd_zstd_level = s->parameters.multifd_zstd_level;
 813     params->has_xbzrle_cache_size = true;
 814     params->xbzrle_cache_size = s->parameters.xbzrle_cache_size;
 815     params->has_max_postcopy_bandwidth = true;
 816     params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth;
 817     params->has_max_cpu_throttle = true;
 818     params->max_cpu_throttle = s->parameters.max_cpu_throttle;
 819     params->has_announce_initial = true;
 820     params->announce_initial = s->parameters.announce_initial;
 821     params->has_announce_max = true;
 822     params->announce_max = s->parameters.announce_max;
 823     params->has_announce_rounds = true;
 824     params->announce_rounds = s->parameters.announce_rounds;
 825     params->has_announce_step = true;
 826     params->announce_step = s->parameters.announce_step;
 827
 828     return params;
 829 }
 830
 831 AnnounceParameters *migrate_announce_params(void)
 832 {
 833     static AnnounceParameters ap;
 834
 835     MigrationState *s = migrate_get_current();
 836
 837     ap.initial = s->parameters.announce_initial;
 838     ap.max = s->parameters.announce_max;
 839     ap.rounds = s->parameters.announce_rounds;
 840     ap.step = s->parameters.announce_step;
 841
 842     return &ap;
 843 }
 844
 845 /*
 846  * Return true if we're already in the middle of a migration
 847  * (i.e. any of the active or setup states)
 848  */
 849 bool migration_is_setup_or_active(int state)
 850 {
 851     switch (state) {
 852     case MIGRATION_STATUS_ACTIVE:
 853     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 854     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 855     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 856     case MIGRATION_STATUS_SETUP:
 857     case MIGRATION_STATUS_PRE_SWITCHOVER:
 858     case MIGRATION_STATUS_DEVICE:
 859     case MIGRATION_STATUS_WAIT_UNPLUG:
 860     case MIGRATION_STATUS_COLO:
 861         return true;
 862
 863     default:
 864         return false;
 865
 866     }
 867 }
 868
 869 bool migration_is_running(int state)
 870 {
 871     switch (state) {
 872     case MIGRATION_STATUS_ACTIVE:
 873     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 874     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 875     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 876     case MIGRATION_STATUS_SETUP:
 877     case MIGRATION_STATUS_PRE_SWITCHOVER:
 878     case MIGRATION_STATUS_DEVICE:
 879     case MIGRATION_STATUS_WAIT_UNPLUG:
 880     case MIGRATION_STATUS_CANCELLING:
 881         return true;
 882
 883     default:
 884         return false;
 885
 886     }
 887 }
 888
 889 static void populate_time_info(MigrationInfo *info, MigrationState *s)
 890 {
 891     info->has_status = true;
 892     info->has_setup_time = true;
 893     info->setup_time = s->setup_time;
 894     if (s->state == MIGRATION_STATUS_COMPLETED) {
 895         info->has_total_time = true;
 896         info->total_time = s->total_time;
 897         info->has_downtime = true;
 898         info->downtime = s->downtime;
 899     } else {
 900         info->has_total_time = true;
 901         info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) -
 902                            s->start_time;
 903         info->has_expected_downtime = true;
 904         info->expected_downtime = s->expected_downtime;
 905     }
 906 }
 907
 908 static void populate_ram_info(MigrationInfo *info, MigrationState *s)
 909 {
 910     info->has_ram = true;
 911     info->ram = g_malloc0(sizeof(*info->ram));
 912     info->ram->transferred = ram_counters.transferred;
 913     info->ram->total = ram_bytes_total();
 914     info->ram->duplicate = ram_counters.duplicate;
 915     /* legacy value.  It is not used anymore */
 916     info->ram->skipped = 0;
 917     info->ram->normal = ram_counters.normal;
 918     info->ram->normal_bytes = ram_counters.normal *
 919         qemu_target_page_size();
 920     info->ram->mbps = s->mbps;
 921     info->ram->dirty_sync_count = ram_counters.dirty_sync_count;
 922     info->ram->postcopy_requests = ram_counters.postcopy_requests;
 923     info->ram->page_size = qemu_target_page_size();
 924     info->ram->multifd_bytes = ram_counters.multifd_bytes;
 925     info->ram->pages_per_second = s->pages_per_second;
 926
 927     if (migrate_use_xbzrle()) {
 928         info->has_xbzrle_cache = true;
 929         info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
 930         info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
 931         info->xbzrle_cache->bytes = xbzrle_counters.bytes;
 932         info->xbzrle_cache->pages = xbzrle_counters.pages;
 933         info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss;
 934         info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate;
 935         info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate;
 936         info->xbzrle_cache->overflow = xbzrle_counters.overflow;
 937     }
 938
 939     if (migrate_use_compression()) {
 940         info->has_compression = true;
 941         info->compression = g_malloc0(sizeof(*info->compression));
 942         info->compression->pages = compression_counters.pages;
 943         info->compression->busy = compression_counters.busy;
 944         info->compression->busy_rate = compression_counters.busy_rate;
 945         info->compression->compressed_size =
 946                                     compression_counters.compressed_size;
 947         info->compression->compression_rate =
 948                                     compression_counters.compression_rate;
 949     }
 950
 951     if (cpu_throttle_active()) {
 952         info->has_cpu_throttle_percentage = true;
 953         info->cpu_throttle_percentage = cpu_throttle_get_percentage();
 954     }
 955
 956     if (s->state != MIGRATION_STATUS_COMPLETED) {
 957         info->ram->remaining = ram_bytes_remaining();
 958         info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate;
 959     }
 960 }
 961
 962 static void populate_disk_info(MigrationInfo *info)
 963 {
 964     if (blk_mig_active()) {
 965         info->has_disk = true;
 966         info->disk = g_malloc0(sizeof(*info->disk));
 967         info->disk->transferred = blk_mig_bytes_transferred();
 968         info->disk->remaining = blk_mig_bytes_remaining();
 969         info->disk->total = blk_mig_bytes_total();
 970     }
 971 }
 972
 973 static void fill_source_migration_info(MigrationInfo *info)
 974 {
 975     MigrationState *s = migrate_get_current();
 976
 977     switch (s->state) {
 978     case MIGRATION_STATUS_NONE:
 979         /* no migration has happened ever */
 980         /* do not overwrite destination migration status */
 981         return;
 982         break;
 983     case MIGRATION_STATUS_SETUP:
 984         info->has_status = true;
 985         info->has_total_time = false;
 986         break;
 987     case MIGRATION_STATUS_ACTIVE:
 988     case MIGRATION_STATUS_CANCELLING:
 989     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
 990     case MIGRATION_STATUS_PRE_SWITCHOVER:
 991     case MIGRATION_STATUS_DEVICE:
 992     case MIGRATION_STATUS_POSTCOPY_PAUSED:
 993     case MIGRATION_STATUS_POSTCOPY_RECOVER:
 994         /* TODO add some postcopy stats */
 995         populate_time_info(info, s);
 996         populate_ram_info(info, s);
 997         populate_disk_info(info);
 998         break;
 999     case MIGRATION_STATUS_COLO:
1000         info->has_status = true;
1001         /* TODO: display COLO specific information (checkpoint info etc.) */
1002         break;
1003     case MIGRATION_STATUS_COMPLETED:
1004         populate_time_info(info, s);
1005         populate_ram_info(info, s);
1006         break;
1007     case MIGRATION_STATUS_FAILED:
1008         info->has_status = true;
1009         if (s->error) {
1010             info->has_error_desc = true;
1011             info->error_desc = g_strdup(error_get_pretty(s->error));
1012         }
1013         break;
1014     case MIGRATION_STATUS_CANCELLED:
1015         info->has_status = true;
1016         break;
1017     case MIGRATION_STATUS_WAIT_UNPLUG:
1018         info->has_status = true;
1019         break;
1020     }
1021     info->status = s->state;
1022 }
1023
1024 /**
1025  * @migration_caps_check - check capability validity
1026  *
1027  * @cap_list: old capability list, array of bool
1028  * @params: new capabilities to be applied soon
1029  * @errp: set *errp if the check failed, with reason
1030  *
1031  * Returns true if check passed, otherwise false.
1032  */
1033 static bool migrate_caps_check(bool *cap_list,
1034                                MigrationCapabilityStatusList *params,
1035                                Error **errp)
1036 {
1037     MigrationCapabilityStatusList *cap;
1038     bool old_postcopy_cap;
1039     MigrationIncomingState *mis = migration_incoming_get_current();
1040
1041     old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM];
1042
1043     for (cap = params; cap; cap = cap->next) {
1044         cap_list[cap->value->capability] = cap->value->state;
1045     }
1046
1047 #ifndef CONFIG_LIVE_BLOCK_MIGRATION
1048     if (cap_list[MIGRATION_CAPABILITY_BLOCK]) {
1049         error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) "
1050                    "block migration");
1051         error_append_hint(errp, "Use drive_mirror+NBD instead.\n");
1052         return false;
1053     }
1054 #endif
1055
1056 #ifndef CONFIG_REPLICATION
1057     if (cap_list[MIGRATION_CAPABILITY_X_COLO]) {
1058         error_setg(errp, "QEMU compiled without replication module"
1059                    " can't enable COLO");
1060         error_append_hint(errp, "Please enable replication before COLO.\n");
1061         return false;
1062     }
1063 #endif
1064
1065     if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
1066         /* This check is reasonably expensive, so only when it's being
1067          * set the first time, also it's only the destination that needs
1068          * special support.
1069          */
1070         if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
1071             !postcopy_ram_supported_by_host(mis)) {
1072             /* postcopy_ram_supported_by_host will have emitted a more
1073              * detailed message
1074              */
1075             error_setg(errp, "Postcopy is not supported");
1076             return false;
1077         }
1078
1079         if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) {
1080             error_setg(errp, "Postcopy is not compatible with ignore-shared");
1081             return false;
1082         }
1083     }
1084
1085     return true;
1086 }
1087
1088 static void fill_destination_migration_info(MigrationInfo *info)
1089 {
1090     MigrationIncomingState *mis = migration_incoming_get_current();
1091
1092     if (mis->socket_address_list) {
1093         info->has_socket_address = true;
1094         info->socket_address =
1095             QAPI_CLONE(SocketAddressList, mis->socket_address_list);
1096     }
1097
1098     switch (mis->state) {
1099     case MIGRATION_STATUS_NONE:
1100         return;
1101         break;
1102     case MIGRATION_STATUS_SETUP:
1103     case MIGRATION_STATUS_CANCELLING:
1104     case MIGRATION_STATUS_CANCELLED:
1105     case MIGRATION_STATUS_ACTIVE:
1106     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1107     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1108     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1109     case MIGRATION_STATUS_FAILED:
1110     case MIGRATION_STATUS_COLO:
1111         info->has_status = true;
1112         break;
1113     case MIGRATION_STATUS_COMPLETED:
1114         info->has_status = true;
1115         fill_destination_postcopy_migration_info(info);
1116         break;
1117     }
1118     info->status = mis->state;
1119 }
1120
1121 MigrationInfo *qmp_query_migrate(Error **errp)
1122 {
1123     MigrationInfo *info = g_malloc0(sizeof(*info));
1124
1125     fill_destination_migration_info(info);
1126     fill_source_migration_info(info);
1127
1128     return info;
1129 }
1130
1131 void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
1132                                   Error **errp)
1133 {
1134     MigrationState *s = migrate_get_current();
1135     MigrationCapabilityStatusList *cap;
1136     bool cap_list[MIGRATION_CAPABILITY__MAX];
1137
1138     if (migration_is_running(s->state)) {
1139         error_setg(errp, QERR_MIGRATION_ACTIVE);
1140         return;
1141     }
1142
1143     memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list));
1144     if (!migrate_caps_check(cap_list, params, errp)) {
1145         return;
1146     }
1147
1148     for (cap = params; cap; cap = cap->next) {
1149         s->enabled_capabilities[cap->value->capability] = cap->value->state;
1150     }
1151 }
1152
1153 /*
1154  * Check whether the parameters are valid. Error will be put into errp
1155  * (if provided). Return true if valid, otherwise false.
1156  */
1157 static bool migrate_params_check(MigrationParameters *params, Error **errp)
1158 {
1159     if (params->has_compress_level &&
1160         (params->compress_level > 9)) {
1161         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
1162                    "is invalid, it should be in the range of 0 to 9");
1163         return false;
1164     }
1165
1166     if (params->has_compress_threads && (params->compress_threads < 1)) {
1167         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1168                    "compress_threads",
1169                    "is invalid, it should be in the range of 1 to 255");
1170         return false;
1171     }
1172
1173     if (params->has_decompress_threads && (params->decompress_threads < 1)) {
1174         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1175                    "decompress_threads",
1176                    "is invalid, it should be in the range of 1 to 255");
1177         return false;
1178     }
1179
1180     if (params->has_throttle_trigger_threshold &&
1181         (params->throttle_trigger_threshold < 1 ||
1182          params->throttle_trigger_threshold > 100)) {
1183         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1184                    "throttle_trigger_threshold",
1185                    "an integer in the range of 1 to 100");
1186         return false;
1187     }
1188
1189     if (params->has_cpu_throttle_initial &&
1190         (params->cpu_throttle_initial < 1 ||
1191          params->cpu_throttle_initial > 99)) {
1192         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1193                    "cpu_throttle_initial",
1194                    "an integer in the range of 1 to 99");
1195         return false;
1196     }
1197
1198     if (params->has_cpu_throttle_increment &&
1199         (params->cpu_throttle_increment < 1 ||
1200          params->cpu_throttle_increment > 99)) {
1201         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1202                    "cpu_throttle_increment",
1203                    "an integer in the range of 1 to 99");
1204         return false;
1205     }
1206
1207     if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) {
1208         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1209                    "max_bandwidth",
1210                    "an integer in the range of 0 to "stringify(SIZE_MAX)
1211                    " bytes/second");
1212         return false;
1213     }
1214
1215     if (params->has_downtime_limit &&
1216         (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) {
1217         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1218                    "downtime_limit",
1219                    "an integer in the range of 0 to "
1220                     stringify(MAX_MIGRATE_DOWNTIME)" ms");
1221         return false;
1222     }
1223
1224     /* x_checkpoint_delay is now always positive */
1225
1226     if (params->has_multifd_channels && (params->multifd_channels < 1)) {
1227         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1228                    "multifd_channels",
1229                    "is invalid, it should be in the range of 1 to 255");
1230         return false;
1231     }
1232
1233     if (params->has_multifd_zlib_level &&
1234         (params->multifd_zlib_level > 9)) {
1235         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level",
1236                    "is invalid, it should be in the range of 0 to 9");
1237         return false;
1238     }
1239
1240     if (params->has_multifd_zstd_level &&
1241         (params->multifd_zstd_level > 20)) {
1242         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level",
1243                    "is invalid, it should be in the range of 0 to 20");
1244         return false;
1245     }
1246
1247     if (params->has_xbzrle_cache_size &&
1248         (params->xbzrle_cache_size < qemu_target_page_size() ||
1249          !is_power_of_2(params->xbzrle_cache_size))) {
1250         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1251                    "xbzrle_cache_size",
1252                    "is invalid, it should be bigger than target page size"
1253                    " and a power of 2");
1254         return false;
1255     }
1256
1257     if (params->has_max_cpu_throttle &&
1258         (params->max_cpu_throttle < params->cpu_throttle_initial ||
1259          params->max_cpu_throttle > 99)) {
1260         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1261                    "max_cpu_throttle",
1262                    "an integer in the range of cpu_throttle_initial to 99");
1263         return false;
1264     }
1265
1266     if (params->has_announce_initial &&
1267         params->announce_initial > 100000) {
1268         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1269                    "announce_initial",
1270                    "is invalid, it must be less than 100000 ms");
1271         return false;
1272     }
1273     if (params->has_announce_max &&
1274         params->announce_max > 100000) {
1275         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1276                    "announce_max",
1277                    "is invalid, it must be less than 100000 ms");
1278        return false;
1279     }
1280     if (params->has_announce_rounds &&
1281         params->announce_rounds > 1000) {
1282         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1283                    "announce_rounds",
1284                    "is invalid, it must be in the range of 0 to 1000");
1285        return false;
1286     }
1287     if (params->has_announce_step &&
1288         (params->announce_step < 1 ||
1289         params->announce_step > 10000)) {
1290         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
1291                    "announce_step",
1292                    "is invalid, it must be in the range of 1 to 10000 ms");
1293        return false;
1294     }
1295     return true;
1296 }
1297
1298 static void migrate_params_test_apply(MigrateSetParameters *params,
1299                                       MigrationParameters *dest)
1300 {
1301     *dest = migrate_get_current()->parameters;
1302
1303     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1304
1305     if (params->has_compress_level) {
1306         dest->compress_level = params->compress_level;
1307     }
1308
1309     if (params->has_compress_threads) {
1310         dest->compress_threads = params->compress_threads;
1311     }
1312
1313     if (params->has_compress_wait_thread) {
1314         dest->compress_wait_thread = params->compress_wait_thread;
1315     }
1316
1317     if (params->has_decompress_threads) {
1318         dest->decompress_threads = params->decompress_threads;
1319     }
1320
1321     if (params->has_throttle_trigger_threshold) {
1322         dest->throttle_trigger_threshold = params->throttle_trigger_threshold;
1323     }
1324
1325     if (params->has_cpu_throttle_initial) {
1326         dest->cpu_throttle_initial = params->cpu_throttle_initial;
1327     }
1328
1329     if (params->has_cpu_throttle_increment) {
1330         dest->cpu_throttle_increment = params->cpu_throttle_increment;
1331     }
1332
1333     if (params->has_cpu_throttle_tailslow) {
1334         dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1335     }
1336
1337     if (params->has_tls_creds) {
1338         assert(params->tls_creds->type == QTYPE_QSTRING);
1339         dest->tls_creds = g_strdup(params->tls_creds->u.s);
1340     }
1341
1342     if (params->has_tls_hostname) {
1343         assert(params->tls_hostname->type == QTYPE_QSTRING);
1344         dest->tls_hostname = g_strdup(params->tls_hostname->u.s);
1345     }
1346
1347     if (params->has_max_bandwidth) {
1348         dest->max_bandwidth = params->max_bandwidth;
1349     }
1350
1351     if (params->has_downtime_limit) {
1352         dest->downtime_limit = params->downtime_limit;
1353     }
1354
1355     if (params->has_x_checkpoint_delay) {
1356         dest->x_checkpoint_delay = params->x_checkpoint_delay;
1357     }
1358
1359     if (params->has_block_incremental) {
1360         dest->block_incremental = params->block_incremental;
1361     }
1362     if (params->has_multifd_channels) {
1363         dest->multifd_channels = params->multifd_channels;
1364     }
1365     if (params->has_multifd_compression) {
1366         dest->multifd_compression = params->multifd_compression;
1367     }
1368     if (params->has_xbzrle_cache_size) {
1369         dest->xbzrle_cache_size = params->xbzrle_cache_size;
1370     }
1371     if (params->has_max_postcopy_bandwidth) {
1372         dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1373     }
1374     if (params->has_max_cpu_throttle) {
1375         dest->max_cpu_throttle = params->max_cpu_throttle;
1376     }
1377     if (params->has_announce_initial) {
1378         dest->announce_initial = params->announce_initial;
1379     }
1380     if (params->has_announce_max) {
1381         dest->announce_max = params->announce_max;
1382     }
1383     if (params->has_announce_rounds) {
1384         dest->announce_rounds = params->announce_rounds;
1385     }
1386     if (params->has_announce_step) {
1387         dest->announce_step = params->announce_step;
1388     }
1389 }
1390
1391 static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
1392 {
1393     MigrationState *s = migrate_get_current();
1394
1395     /* TODO use QAPI_CLONE() instead of duplicating it inline */
1396
1397     if (params->has_compress_level) {
1398         s->parameters.compress_level = params->compress_level;
1399     }
1400
1401     if (params->has_compress_threads) {
1402         s->parameters.compress_threads = params->compress_threads;
1403     }
1404
1405     if (params->has_compress_wait_thread) {
1406         s->parameters.compress_wait_thread = params->compress_wait_thread;
1407     }
1408
1409     if (params->has_decompress_threads) {
1410         s->parameters.decompress_threads = params->decompress_threads;
1411     }
1412
1413     if (params->has_throttle_trigger_threshold) {
1414         s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold;
1415     }
1416
1417     if (params->has_cpu_throttle_initial) {
1418         s->parameters.cpu_throttle_initial = params->cpu_throttle_initial;
1419     }
1420
1421     if (params->has_cpu_throttle_increment) {
1422         s->parameters.cpu_throttle_increment = params->cpu_throttle_increment;
1423     }
1424
1425     if (params->has_cpu_throttle_tailslow) {
1426         s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow;
1427     }
1428
1429     if (params->has_tls_creds) {
1430         g_free(s->parameters.tls_creds);
1431         assert(params->tls_creds->type == QTYPE_QSTRING);
1432         s->parameters.tls_creds = g_strdup(params->tls_creds->u.s);
1433     }
1434
1435     if (params->has_tls_hostname) {
1436         g_free(s->parameters.tls_hostname);
1437         assert(params->tls_hostname->type == QTYPE_QSTRING);
1438         s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s);
1439     }
1440
1441     if (params->has_tls_authz) {
1442         g_free(s->parameters.tls_authz);
1443         assert(params->tls_authz->type == QTYPE_QSTRING);
1444         s->parameters.tls_authz = g_strdup(params->tls_authz->u.s);
1445     }
1446
1447     if (params->has_max_bandwidth) {
1448         s->parameters.max_bandwidth = params->max_bandwidth;
1449         if (s->to_dst_file && !migration_in_postcopy()) {
1450             qemu_file_set_rate_limit(s->to_dst_file,
1451                                 s->parameters.max_bandwidth / XFER_LIMIT_RATIO);
1452         }
1453     }
1454
1455     if (params->has_downtime_limit) {
1456         s->parameters.downtime_limit = params->downtime_limit;
1457     }
1458
1459     if (params->has_x_checkpoint_delay) {
1460         s->parameters.x_checkpoint_delay = params->x_checkpoint_delay;
1461         if (migration_in_colo_state()) {
1462             colo_checkpoint_notify(s);
1463         }
1464     }
1465
1466     if (params->has_block_incremental) {
1467         s->parameters.block_incremental = params->block_incremental;
1468     }
1469     if (params->has_multifd_channels) {
1470         s->parameters.multifd_channels = params->multifd_channels;
1471     }
1472     if (params->has_multifd_compression) {
1473         s->parameters.multifd_compression = params->multifd_compression;
1474     }
1475     if (params->has_xbzrle_cache_size) {
1476         s->parameters.xbzrle_cache_size = params->xbzrle_cache_size;
1477         xbzrle_cache_resize(params->xbzrle_cache_size, errp);
1478     }
1479     if (params->has_max_postcopy_bandwidth) {
1480         s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth;
1481         if (s->to_dst_file && migration_in_postcopy()) {
1482             qemu_file_set_rate_limit(s->to_dst_file,
1483                     s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO);
1484         }
1485     }
1486     if (params->has_max_cpu_throttle) {
1487         s->parameters.max_cpu_throttle = params->max_cpu_throttle;
1488     }
1489     if (params->has_announce_initial) {
1490         s->parameters.announce_initial = params->announce_initial;
1491     }
1492     if (params->has_announce_max) {
1493         s->parameters.announce_max = params->announce_max;
1494     }
1495     if (params->has_announce_rounds) {
1496         s->parameters.announce_rounds = params->announce_rounds;
1497     }
1498     if (params->has_announce_step) {
1499         s->parameters.announce_step = params->announce_step;
1500     }
1501 }
1502
1503 void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
1504 {
1505     MigrationParameters tmp;
1506
1507     /* TODO Rewrite "" to null instead */
1508     if (params->has_tls_creds
1509         && params->tls_creds->type == QTYPE_QNULL) {
1510         qobject_unref(params->tls_creds->u.n);
1511         params->tls_creds->type = QTYPE_QSTRING;
1512         params->tls_creds->u.s = strdup("");
1513     }
1514     /* TODO Rewrite "" to null instead */
1515     if (params->has_tls_hostname
1516         && params->tls_hostname->type == QTYPE_QNULL) {
1517         qobject_unref(params->tls_hostname->u.n);
1518         params->tls_hostname->type = QTYPE_QSTRING;
1519         params->tls_hostname->u.s = strdup("");
1520     }
1521
1522     migrate_params_test_apply(params, &tmp);
1523
1524     if (!migrate_params_check(&tmp, errp)) {
1525         /* Invalid parameter */
1526         return;
1527     }
1528
1529     migrate_params_apply(params, errp);
1530 }
1531
1532
1533 void qmp_migrate_start_postcopy(Error **errp)
1534 {
1535     MigrationState *s = migrate_get_current();
1536
1537     if (!migrate_postcopy()) {
1538         error_setg(errp, "Enable postcopy with migrate_set_capability before"
1539                          " the start of migration");
1540         return;
1541     }
1542
1543     if (s->state == MIGRATION_STATUS_NONE) {
1544         error_setg(errp, "Postcopy must be started after migration has been"
1545                          " started");
1546         return;
1547     }
1548     /*
1549      * we don't error if migration has finished since that would be racy
1550      * with issuing this command.
1551      */
1552     atomic_set(&s->start_postcopy, true);
1553 }
1554
1555 /* shared migration helpers */
1556
1557 void migrate_set_state(int *state, int old_state, int new_state)
1558 {
1559     assert(new_state < MIGRATION_STATUS__MAX);
1560     if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
1561         trace_migrate_set_state(MigrationStatus_str(new_state));
1562         migrate_generate_event(new_state);
1563     }
1564 }
1565
1566 static MigrationCapabilityStatusList *migrate_cap_add(
1567     MigrationCapabilityStatusList *list,
1568     MigrationCapability index,
1569     bool state)
1570 {
1571     MigrationCapabilityStatusList *cap;
1572
1573     cap = g_new0(MigrationCapabilityStatusList, 1);
1574     cap->value = g_new0(MigrationCapabilityStatus, 1);
1575     cap->value->capability = index;
1576     cap->value->state = state;
1577     cap->next = list;
1578
1579     return cap;
1580 }
1581
1582 void migrate_set_block_enabled(bool value, Error **errp)
1583 {
1584     MigrationCapabilityStatusList *cap;
1585
1586     cap = migrate_cap_add(NULL, MIGRATION_CAPABILITY_BLOCK, value);
1587     qmp_migrate_set_capabilities(cap, errp);
1588     qapi_free_MigrationCapabilityStatusList(cap);
1589 }
1590
1591 static void migrate_set_block_incremental(MigrationState *s, bool value)
1592 {
1593     s->parameters.block_incremental = value;
1594 }
1595
1596 static void block_cleanup_parameters(MigrationState *s)
1597 {
1598     if (s->must_remove_block_options) {
1599         /* setting to false can never fail */
1600         migrate_set_block_enabled(false, &error_abort);
1601         migrate_set_block_incremental(s, false);
1602         s->must_remove_block_options = false;
1603     }
1604 }
1605
1606 static void migrate_fd_cleanup(MigrationState *s)
1607 {
1608     qemu_bh_delete(s->cleanup_bh);
1609     s->cleanup_bh = NULL;
1610
1611     qemu_savevm_state_cleanup();
1612
1613     if (s->to_dst_file) {
1614         QEMUFile *tmp;
1615
1616         trace_migrate_fd_cleanup();
1617         qemu_mutex_unlock_iothread();
1618         if (s->migration_thread_running) {
1619             qemu_thread_join(&s->thread);
1620             s->migration_thread_running = false;
1621         }
1622         qemu_mutex_lock_iothread();
1623
1624         multifd_save_cleanup();
1625         qemu_mutex_lock(&s->qemu_file_lock);
1626         tmp = s->to_dst_file;
1627         s->to_dst_file = NULL;
1628         qemu_mutex_unlock(&s->qemu_file_lock);
1629         /*
1630          * Close the file handle without the lock to make sure the
1631          * critical section won't block for long.
1632          */
1633         qemu_fclose(tmp);
1634     }
1635
1636     assert(!migration_is_active(s));
1637
1638     if (s->state == MIGRATION_STATUS_CANCELLING) {
1639         migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
1640                           MIGRATION_STATUS_CANCELLED);
1641     }
1642
1643     if (s->error) {
1644         /* It is used on info migrate.  We can't free it */
1645         error_report_err(error_copy(s->error));
1646     }
1647     notifier_list_notify(&migration_state_notifiers, s);
1648     block_cleanup_parameters(s);
1649 }
1650
1651 static void migrate_fd_cleanup_schedule(MigrationState *s)
1652 {
1653     /*
1654      * Ref the state for bh, because it may be called when
1655      * there're already no other refs
1656      */
1657     object_ref(OBJECT(s));
1658     qemu_bh_schedule(s->cleanup_bh);
1659 }
1660
1661 static void migrate_fd_cleanup_bh(void *opaque)
1662 {
1663     MigrationState *s = opaque;
1664     migrate_fd_cleanup(s);
1665     object_unref(OBJECT(s));
1666 }
1667
1668 void migrate_set_error(MigrationState *s, const Error *error)
1669 {
1670     QEMU_LOCK_GUARD(&s->error_mutex);
1671     if (!s->error) {
1672         s->error = error_copy(error);
1673     }
1674 }
1675
1676 void migrate_fd_error(MigrationState *s, const Error *error)
1677 {
1678     trace_migrate_fd_error(error_get_pretty(error));
1679     assert(s->to_dst_file == NULL);
1680     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
1681                       MIGRATION_STATUS_FAILED);
1682     migrate_set_error(s, error);
1683 }
1684
1685 static void migrate_fd_cancel(MigrationState *s)
1686 {
1687     int old_state ;
1688     QEMUFile *f = migrate_get_current()->to_dst_file;
1689     trace_migrate_fd_cancel();
1690
1691     if (s->rp_state.from_dst_file) {
1692         /* shutdown the rp socket, so causing the rp thread to shutdown */
1693         qemu_file_shutdown(s->rp_state.from_dst_file);
1694     }
1695
1696     do {
1697         old_state = s->state;
1698         if (!migration_is_running(old_state)) {
1699             break;
1700         }
1701         /* If the migration is paused, kick it out of the pause */
1702         if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) {
1703             qemu_sem_post(&s->pause_sem);
1704         }
1705         migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
1706     } while (s->state != MIGRATION_STATUS_CANCELLING);
1707
1708     /*
1709      * If we're unlucky the migration code might be stuck somewhere in a
1710      * send/write while the network has failed and is waiting to timeout;
1711      * if we've got shutdown(2) available then we can force it to quit.
1712      * The outgoing qemu file gets closed in migrate_fd_cleanup that is
1713      * called in a bh, so there is no race against this cancel.
1714      */
1715     if (s->state == MIGRATION_STATUS_CANCELLING && f) {
1716         qemu_file_shutdown(f);
1717     }
1718     if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) {
1719         Error *local_err = NULL;
1720
1721         bdrv_invalidate_cache_all(&local_err);
1722         if (local_err) {
1723             error_report_err(local_err);
1724         } else {
1725             s->block_inactive = false;
1726         }
1727     }
1728 }
1729
1730 void add_migration_state_change_notifier(Notifier *notify)
1731 {
1732     notifier_list_add(&migration_state_notifiers, notify);
1733 }
1734
1735 void remove_migration_state_change_notifier(Notifier *notify)
1736 {
1737     notifier_remove(notify);
1738 }
1739
1740 bool migration_in_setup(MigrationState *s)
1741 {
1742     return s->state == MIGRATION_STATUS_SETUP;
1743 }
1744
1745 bool migration_has_finished(MigrationState *s)
1746 {
1747     return s->state == MIGRATION_STATUS_COMPLETED;
1748 }
1749
1750 bool migration_has_failed(MigrationState *s)
1751 {
1752     return (s->state == MIGRATION_STATUS_CANCELLED ||
1753             s->state == MIGRATION_STATUS_FAILED);
1754 }
1755
1756 bool migration_in_postcopy(void)
1757 {
1758     MigrationState *s = migrate_get_current();
1759
1760     switch (s->state) {
1761     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1762     case MIGRATION_STATUS_POSTCOPY_PAUSED:
1763     case MIGRATION_STATUS_POSTCOPY_RECOVER:
1764         return true;
1765     default:
1766         return false;
1767     }
1768 }
1769
1770 bool migration_in_postcopy_after_devices(MigrationState *s)
1771 {
1772     return migration_in_postcopy() && s->postcopy_after_devices;
1773 }
1774
1775 bool migration_is_idle(void)
1776 {
1777     MigrationState *s = current_migration;
1778
1779     if (!s) {
1780         return true;
1781     }
1782
1783     switch (s->state) {
1784     case MIGRATION_STATUS_NONE:
1785     case MIGRATION_STATUS_CANCELLED:
1786     case MIGRATION_STATUS_COMPLETED:
1787     case MIGRATION_STATUS_FAILED:
1788         return true;
1789     case MIGRATION_STATUS_SETUP:
1790     case MIGRATION_STATUS_CANCELLING:
1791     case MIGRATION_STATUS_ACTIVE:
1792     case MIGRATION_STATUS_POSTCOPY_ACTIVE:
1793     case MIGRATION_STATUS_COLO:
1794     case MIGRATION_STATUS_PRE_SWITCHOVER:
1795     case MIGRATION_STATUS_DEVICE:
1796     case MIGRATION_STATUS_WAIT_UNPLUG:
1797         return false;
1798     case MIGRATION_STATUS__MAX:
1799         g_assert_not_reached();
1800     }
1801
1802     return false;
1803 }
1804
1805 bool migration_is_active(MigrationState *s)
1806 {
1807     return (s->state == MIGRATION_STATUS_ACTIVE ||
1808             s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
1809 }
1810
1811 void migrate_init(MigrationState *s)
1812 {
1813     /*
1814      * Reinitialise all migration state, except
1815      * parameters/capabilities that the user set, and
1816      * locks.
1817      */
1818     s->cleanup_bh = 0;
1819     s->to_dst_file = NULL;
1820     s->state = MIGRATION_STATUS_NONE;
1821     s->rp_state.from_dst_file = NULL;
1822     s->rp_state.error = false;
1823     s->mbps = 0.0;
1824     s->pages_per_second = 0.0;
1825     s->downtime = 0;
1826     s->expected_downtime = 0;
1827     s->setup_time = 0;
1828     s->start_postcopy = false;
1829     s->postcopy_after_devices = false;
1830     s->migration_thread_running = false;
1831     error_free(s->error);
1832     s->error = NULL;
1833
1834     migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
1835
1836     s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
1837     s->total_time = 0;
1838     s->vm_was_running = false;
1839     s->iteration_initial_bytes = 0;
1840     s->threshold_size = 0;
1841 }
1842
1843 static GSList *migration_blockers;
1844
1845 int migrate_add_blocker(Error *reason, Error **errp)
1846 {
1847     if (only_migratable) {
1848         error_propagate_prepend(errp, error_copy(reason),
1849                                 "disallowing migration blocker "
1850                                 "(--only-migratable) for: ");
1851         return -EACCES;
1852     }
1853
1854     if (migration_is_idle()) {
1855         migration_blockers = g_slist_prepend(migration_blockers, reason);
1856         return 0;
1857     }
1858
1859     error_propagate_prepend(errp, error_copy(reason),
1860                             "disallowing migration blocker "
1861                             "(migration in progress) for: ");
1862     return -EBUSY;
1863 }
1864
1865 void migrate_del_blocker(Error *reason)
1866 {
1867     migration_blockers = g_slist_remove(migration_blockers, reason);
1868 }
1869
1870 void qmp_migrate_incoming(const char *uri, Error **errp)
1871 {
1872     Error *local_err = NULL;
1873     static bool once = true;
1874
1875     if (!deferred_incoming) {
1876         error_setg(errp, "For use with '-incoming defer'");
1877         return;
1878     }
1879     if (!once) {
1880         error_setg(errp, "The incoming migration has already been started");
1881         return;
1882     }
1883
1884     qemu_start_incoming_migration(uri, &local_err);
1885
1886     if (local_err) {
1887         error_propagate(errp, local_err);
1888         return;
1889     }
1890
1891     once = false;
1892 }
1893
1894 void qmp_migrate_recover(const char *uri, Error **errp)
1895 {
1896     MigrationIncomingState *mis = migration_incoming_get_current();
1897
1898     if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1899         error_setg(errp, "Migrate recover can only be run "
1900                    "when postcopy is paused.");
1901         return;
1902     }
1903
1904     if (atomic_cmpxchg(&mis->postcopy_recover_triggered,
1905                        false, true) == true) {
1906         error_setg(errp, "Migrate recovery is triggered already");
1907         return;
1908     }
1909
1910     /*
1911      * Note that this call will never start a real migration; it will
1912      * only re-setup the migration stream and poke existing migration
1913      * to continue using that newly established channel.
1914      */
1915     qemu_start_incoming_migration(uri, errp);
1916 }
1917
1918 void qmp_migrate_pause(Error **errp)
1919 {
1920     MigrationState *ms = migrate_get_current();
1921     MigrationIncomingState *mis = migration_incoming_get_current();
1922     int ret;
1923
1924     if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1925         /* Source side, during postcopy */
1926         qemu_mutex_lock(&ms->qemu_file_lock);
1927         ret = qemu_file_shutdown(ms->to_dst_file);
1928         qemu_mutex_unlock(&ms->qemu_file_lock);
1929         if (ret) {
1930             error_setg(errp, "Failed to pause source migration");
1931         }
1932         return;
1933     }
1934
1935     if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
1936         ret = qemu_file_shutdown(mis->from_src_file);
1937         if (ret) {
1938             error_setg(errp, "Failed to pause destination migration");
1939         }
1940         return;
1941     }
1942
1943     error_setg(errp, "migrate-pause is currently only supported "
1944                "during postcopy-active state");
1945 }
1946
1947 bool migration_is_blocked(Error **errp)
1948 {
1949     if (qemu_savevm_state_blocked(errp)) {
1950         return true;
1951     }
1952
1953     if (migration_blockers) {
1954         error_propagate(errp, error_copy(migration_blockers->data));
1955         return true;
1956     }
1957
1958     return false;
1959 }
1960
1961 /* Returns true if continue to migrate, or false if error detected */
1962 static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
1963                             bool resume, Error **errp)
1964 {
1965     Error *local_err = NULL;
1966
1967     if (resume) {
1968         if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) {
1969             error_setg(errp, "Cannot resume if there is no "
1970                        "paused migration");
1971             return false;
1972         }
1973
1974         /*
1975          * Postcopy recovery won't work well with release-ram
1976          * capability since release-ram will drop the page buffer as
1977          * long as the page is put into the send buffer.  So if there
1978          * is a network failure happened, any page buffers that have
1979          * not yet reached the destination VM but have already been
1980          * sent from the source VM will be lost forever.  Let's refuse
1981          * the client from resuming such a postcopy migration.
1982          * Luckily release-ram was designed to only be used when src
1983          * and destination VMs are on the same host, so it should be
1984          * fine.
1985          */
1986         if (migrate_release_ram()) {
1987             error_setg(errp, "Postcopy recovery cannot work "
1988                        "when release-ram capability is set");
1989             return false;
1990         }
1991
1992         /* This is a resume, skip init status */
1993         return true;
1994     }
1995
1996     if (migration_is_running(s->state)) {
1997         error_setg(errp, QERR_MIGRATION_ACTIVE);
1998         return false;
1999     }
2000
2001     if (runstate_check(RUN_STATE_INMIGRATE)) {
2002         error_setg(errp, "Guest is waiting for an incoming migration");
2003         return false;
2004     }
2005
2006     if (migration_is_blocked(errp)) {
2007         return false;
2008     }
2009
2010     if (blk || blk_inc) {
2011         if (migrate_use_block() || migrate_use_block_incremental()) {
2012             error_setg(errp, "Command options are incompatible with "
2013                        "current migration capabilities");
2014             return false;
2015         }
2016         migrate_set_block_enabled(true, &local_err);
2017         if (local_err) {
2018             error_propagate(errp, local_err);
2019             return false;
2020         }
2021         s->must_remove_block_options = true;
2022     }
2023
2024     if (blk_inc) {
2025         migrate_set_block_incremental(s, true);
2026     }
2027
2028     migrate_init(s);
2029     /*
2030      * set ram_counters memory to zero for a
2031      * new migration
2032      */
2033     memset(&ram_counters, 0, sizeof(ram_counters));
2034
2035     return true;
2036 }
2037
2038 void qmp_migrate(const char *uri, bool has_blk, bool blk,
2039                  bool has_inc, bool inc, bool has_detach, bool detach,
2040                  bool has_resume, bool resume, Error **errp)
2041 {
2042     Error *local_err = NULL;
2043     MigrationState *s = migrate_get_current();
2044     const char *p;
2045
2046     if (!migrate_prepare(s, has_blk && blk, has_inc && inc,
2047                          has_resume && resume, errp)) {
2048         /* Error detected, put into errp */
2049         return;
2050     }
2051
2052     if (strstart(uri, "tcp:", &p)) {
2053         tcp_start_outgoing_migration(s, p, &local_err);
2054 #ifdef CONFIG_RDMA
2055     } else if (strstart(uri, "rdma:", &p)) {
2056         rdma_start_outgoing_migration(s, p, &local_err);
2057 #endif
2058     } else if (strstart(uri, "exec:", &p)) {
2059         exec_start_outgoing_migration(s, p, &local_err);
2060     } else if (strstart(uri, "unix:", &p)) {
2061         unix_start_outgoing_migration(s, p, &local_err);
2062     } else if (strstart(uri, "fd:", &p)) {
2063         fd_start_outgoing_migration(s, p, &local_err);
2064     } else {
2065         error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
2066                    "a valid migration protocol");
2067         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
2068                           MIGRATION_STATUS_FAILED);
2069         block_cleanup_parameters(s);
2070         return;
2071     }
2072
2073     if (local_err) {
2074         migrate_fd_error(s, local_err);
2075         error_propagate(errp, local_err);
2076         return;
2077     }
2078 }
2079
2080 void qmp_migrate_cancel(Error **errp)
2081 {
2082     migrate_fd_cancel(migrate_get_current());
2083 }
2084
2085 void qmp_migrate_continue(MigrationStatus state, Error **errp)
2086 {
2087     MigrationState *s = migrate_get_current();
2088     if (s->state != state) {
2089         error_setg(errp,  "Migration not in expected state: %s",
2090                    MigrationStatus_str(s->state));
2091         return;
2092     }
2093     qemu_sem_post(&s->pause_sem);
2094 }
2095
2096 void qmp_migrate_set_cache_size(int64_t value, Error **errp)
2097 {
2098     MigrateSetParameters p = {
2099         .has_xbzrle_cache_size = true,
2100         .xbzrle_cache_size = value,
2101     };
2102
2103     qmp_migrate_set_parameters(&p, errp);
2104 }
2105
2106 int64_t qmp_query_migrate_cache_size(Error **errp)
2107 {
2108     return migrate_xbzrle_cache_size();
2109 }
2110
2111 void qmp_migrate_set_speed(int64_t value, Error **errp)
2112 {
2113     MigrateSetParameters p = {
2114         .has_max_bandwidth = true,
2115         .max_bandwidth = value,
2116     };
2117
2118     qmp_migrate_set_parameters(&p, errp);
2119 }
2120
2121 void qmp_migrate_set_downtime(double value, Error **errp)
2122 {
2123     if (value < 0 || value > MAX_MIGRATE_DOWNTIME_SECONDS) {
2124         error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
2125                    "downtime_limit",
2126                    "an integer in the range of 0 to "
2127                     stringify(MAX_MIGRATE_DOWNTIME_SECONDS)" seconds");
2128         return;
2129     }
2130
2131     value *= 1000; /* Convert to milliseconds */
2132
2133     MigrateSetParameters p = {
2134         .has_downtime_limit = true,
2135         .downtime_limit = (int64_t)value,
2136     };
2137
2138     qmp_migrate_set_parameters(&p, errp);
2139 }
2140
2141 bool migrate_release_ram(void)
2142 {
2143     MigrationState *s;
2144
2145     s = migrate_get_current();
2146
2147     return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM];
2148 }
2149
2150 bool migrate_postcopy_ram(void)
2151 {
2152     MigrationState *s;
2153
2154     s = migrate_get_current();
2155
2156     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
2157 }
2158
2159 bool migrate_postcopy(void)
2160 {
2161     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
2162 }
2163
2164 bool migrate_auto_converge(void)
2165 {
2166     MigrationState *s;
2167
2168     s = migrate_get_current();
2169
2170     return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
2171 }
2172
2173 bool migrate_zero_blocks(void)
2174 {
2175     MigrationState *s;
2176
2177     s = migrate_get_current();
2178
2179     return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
2180 }
2181
2182 bool migrate_postcopy_blocktime(void)
2183 {
2184     MigrationState *s;
2185
2186     s = migrate_get_current();
2187
2188     return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME];
2189 }
2190
2191 bool migrate_use_compression(void)
2192 {
2193     MigrationState *s;
2194
2195     s = migrate_get_current();
2196
2197     return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
2198 }
2199
2200 int migrate_compress_level(void)
2201 {
2202     MigrationState *s;
2203
2204     s = migrate_get_current();
2205
2206     return s->parameters.compress_level;
2207 }
2208
2209 int migrate_compress_threads(void)
2210 {
2211     MigrationState *s;
2212
2213     s = migrate_get_current();
2214
2215     return s->parameters.compress_threads;
2216 }
2217
2218 int migrate_compress_wait_thread(void)
2219 {
2220     MigrationState *s;
2221
2222     s = migrate_get_current();
2223
2224     return s->parameters.compress_wait_thread;
2225 }
2226
2227 int migrate_decompress_threads(void)
2228 {
2229     MigrationState *s;
2230
2231     s = migrate_get_current();
2232
2233     return s->parameters.decompress_threads;
2234 }
2235
2236 bool migrate_dirty_bitmaps(void)
2237 {
2238     MigrationState *s;
2239
2240     s = migrate_get_current();
2241
2242     return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS];
2243 }
2244
2245 bool migrate_ignore_shared(void)
2246 {
2247     MigrationState *s;
2248
2249     s = migrate_get_current();
2250
2251     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED];
2252 }
2253
2254 bool migrate_validate_uuid(void)
2255 {
2256     MigrationState *s;
2257
2258     s = migrate_get_current();
2259
2260     return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID];
2261 }
2262
2263 bool migrate_use_events(void)
2264 {
2265     MigrationState *s;
2266
2267     s = migrate_get_current();
2268
2269     return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
2270 }
2271
2272 bool migrate_use_multifd(void)
2273 {
2274     MigrationState *s;
2275
2276     s = migrate_get_current();
2277
2278     return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD];
2279 }
2280
2281 bool migrate_pause_before_switchover(void)
2282 {
2283     MigrationState *s;
2284
2285     s = migrate_get_current();
2286
2287     return s->enabled_capabilities[
2288         MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER];
2289 }
2290
2291 int migrate_multifd_channels(void)
2292 {
2293     MigrationState *s;
2294
2295     s = migrate_get_current();
2296
2297     return s->parameters.multifd_channels;
2298 }
2299
2300 MultiFDCompression migrate_multifd_compression(void)
2301 {
2302     MigrationState *s;
2303
2304     s = migrate_get_current();
2305
2306     return s->parameters.multifd_compression;
2307 }
2308
2309 int migrate_multifd_zlib_level(void)
2310 {
2311     MigrationState *s;
2312
2313     s = migrate_get_current();
2314
2315     return s->parameters.multifd_zlib_level;
2316 }
2317
2318 int migrate_multifd_zstd_level(void)
2319 {
2320     MigrationState *s;
2321
2322     s = migrate_get_current();
2323
2324     return s->parameters.multifd_zstd_level;
2325 }
2326
2327 int migrate_use_xbzrle(void)
2328 {
2329     MigrationState *s;
2330
2331     s = migrate_get_current();
2332
2333     return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
2334 }
2335
2336 int64_t migrate_xbzrle_cache_size(void)
2337 {
2338     MigrationState *s;
2339
2340     s = migrate_get_current();
2341
2342     return s->parameters.xbzrle_cache_size;
2343 }
2344
2345 static int64_t migrate_max_postcopy_bandwidth(void)
2346 {
2347     MigrationState *s;
2348
2349     s = migrate_get_current();
2350
2351     return s->parameters.max_postcopy_bandwidth;
2352 }
2353
2354 bool migrate_use_block(void)
2355 {
2356     MigrationState *s;
2357
2358     s = migrate_get_current();
2359
2360     return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK];
2361 }
2362
2363 bool migrate_use_return_path(void)
2364 {
2365     MigrationState *s;
2366
2367     s = migrate_get_current();
2368
2369     return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH];
2370 }
2371
2372 bool migrate_use_block_incremental(void)
2373 {
2374     MigrationState *s;
2375
2376     s = migrate_get_current();
2377
2378     return s->parameters.block_incremental;
2379 }
2380
2381 /* migration thread support */
2382 /*
2383  * Something bad happened to the RP stream, mark an error
2384  * The caller shall print or trace something to indicate why
2385  */
2386 static void mark_source_rp_bad(MigrationState *s)
2387 {
2388     s->rp_state.error = true;
2389 }
2390
2391 static struct rp_cmd_args {
2392     ssize_t     len; /* -1 = variable */
2393     const char *name;
2394 } rp_cmd_args[] = {
2395     [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
2396     [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
2397     [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
2398     [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
2399     [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
2400     [MIG_RP_MSG_RECV_BITMAP]    = { .len = -1, .name = "RECV_BITMAP" },
2401     [MIG_RP_MSG_RESUME_ACK]     = { .len =  4, .name = "RESUME_ACK" },
2402     [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
2403 };
2404
2405 /*
2406  * Process a request for pages received on the return path,
2407  * We're allowed to send more than requested (e.g. to round to our page size)
2408  * and we don't need to send pages that have already been sent.
2409  */
2410 static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
2411                                        ram_addr_t start, size_t len)
2412 {
2413     long our_host_ps = qemu_real_host_page_size;
2414
2415     trace_migrate_handle_rp_req_pages(rbname, start, len);
2416
2417     /*
2418      * Since we currently insist on matching page sizes, just sanity check
2419      * we're being asked for whole host pages.
2420      */
2421     if (start & (our_host_ps-1) ||
2422        (len & (our_host_ps-1))) {
2423         error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
2424                      " len: %zd", __func__, start, len);
2425         mark_source_rp_bad(ms);
2426         return;
2427     }
2428
2429     if (ram_save_queue_pages(rbname, start, len)) {
2430         mark_source_rp_bad(ms);
2431     }
2432 }
2433
2434 /* Return true to retry, false to quit */
2435 static bool postcopy_pause_return_path_thread(MigrationState *s)
2436 {
2437     trace_postcopy_pause_return_path();
2438
2439     qemu_sem_wait(&s->postcopy_pause_rp_sem);
2440
2441     trace_postcopy_pause_return_path_continued();
2442
2443     return true;
2444 }
2445
2446 static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name)
2447 {
2448     RAMBlock *block = qemu_ram_block_by_name(block_name);
2449
2450     if (!block) {
2451         error_report("%s: invalid block name '%s'", __func__, block_name);
2452         return -EINVAL;
2453     }
2454
2455     /* Fetch the received bitmap and refresh the dirty bitmap */
2456     return ram_dirty_bitmap_reload(s, block);
2457 }
2458
2459 static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value)
2460 {
2461     trace_source_return_path_thread_resume_ack(value);
2462
2463     if (value != MIGRATION_RESUME_ACK_VALUE) {
2464         error_report("%s: illegal resume_ack value %"PRIu32,
2465                      __func__, value);
2466         return -1;
2467     }
2468
2469     /* Now both sides are active. */
2470     migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER,
2471                       MIGRATION_STATUS_POSTCOPY_ACTIVE);
2472
2473     /* Notify send thread that time to continue send pages */
2474     qemu_sem_post(&s->rp_state.rp_sem);
2475
2476     return 0;
2477 }
2478
2479 /*
2480  * Handles messages sent on the return path towards the source VM
2481  *
2482  */
2483 static void *source_return_path_thread(void *opaque)
2484 {
2485     MigrationState *ms = opaque;
2486     QEMUFile *rp = ms->rp_state.from_dst_file;
2487     uint16_t header_len, header_type;
2488     uint8_t buf[512];
2489     uint32_t tmp32, sibling_error;
2490     ram_addr_t start = 0; /* =0 to silence warning */
2491     size_t  len = 0, expected_len;
2492     int res;
2493
2494     trace_source_return_path_thread_entry();
2495     rcu_register_thread();
2496
2497 retry:
2498     while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
2499            migration_is_setup_or_active(ms->state)) {
2500         trace_source_return_path_thread_loop_top();
2501         header_type = qemu_get_be16(rp);
2502         header_len = qemu_get_be16(rp);
2503
2504         if (qemu_file_get_error(rp)) {
2505             mark_source_rp_bad(ms);
2506             goto out;
2507         }
2508
2509         if (header_type >= MIG_RP_MSG_MAX ||
2510             header_type == MIG_RP_MSG_INVALID) {
2511             error_report("RP: Received invalid message 0x%04x length 0x%04x",
2512                          header_type, header_len);
2513             mark_source_rp_bad(ms);
2514             goto out;
2515         }
2516
2517         if ((rp_cmd_args[header_type].len != -1 &&
2518             header_len != rp_cmd_args[header_type].len) ||
2519             header_len > sizeof(buf)) {
2520             error_report("RP: Received '%s' message (0x%04x) with"
2521                          "incorrect length %d expecting %zu",
2522                          rp_cmd_args[header_type].name, header_type, header_len,
2523                          (size_t)rp_cmd_args[header_type].len);
2524             mark_source_rp_bad(ms);
2525             goto out;
2526         }
2527
2528         /* We know we've got a valid header by this point */
2529         res = qemu_get_buffer(rp, buf, header_len);
2530         if (res != header_len) {
2531             error_report("RP: Failed reading data for message 0x%04x"
2532                          " read %d expected %d",
2533                          header_type, res, header_len);
2534             mark_source_rp_bad(ms);
2535             goto out;
2536         }
2537
2538         /* OK, we have the message and the data */
2539         switch (header_type) {
2540         case MIG_RP_MSG_SHUT:
2541             sibling_error = ldl_be_p(buf);
2542             trace_source_return_path_thread_shut(sibling_error);
2543             if (sibling_error) {
2544                 error_report("RP: Sibling indicated error %d", sibling_error);
2545                 mark_source_rp_bad(ms);
2546             }
2547             /*
2548              * We'll let the main thread deal with closing the RP
2549              * we could do a shutdown(2) on it, but we're the only user
2550              * anyway, so there's nothing gained.
2551              */
2552             goto out;
2553
2554         case MIG_RP_MSG_PONG:
2555             tmp32 = ldl_be_p(buf);
2556             trace_source_return_path_thread_pong(tmp32);
2557             break;
2558
2559         case MIG_RP_MSG_REQ_PAGES:
2560             start = ldq_be_p(buf);
2561             len = ldl_be_p(buf + 8);
2562             migrate_handle_rp_req_pages(ms, NULL, start, len);
2563             break;
2564
2565         case MIG_RP_MSG_REQ_PAGES_ID:
2566             expected_len = 12 + 1; /* header + termination */
2567
2568             if (header_len >= expected_len) {
2569                 start = ldq_be_p(buf);
2570                 len = ldl_be_p(buf + 8);
2571                 /* Now we expect an idstr */
2572                 tmp32 = buf[12]; /* Length of the following idstr */
2573                 buf[13 + tmp32] = '\0';
2574                 expected_len += tmp32;
2575             }
2576             if (header_len != expected_len) {
2577                 error_report("RP: Req_Page_id with length %d expecting %zd",
2578                              header_len, expected_len);
2579                 mark_source_rp_bad(ms);
2580                 goto out;
2581             }
2582             migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
2583             break;
2584
2585         case MIG_RP_MSG_RECV_BITMAP:
2586             if (header_len < 1) {
2587                 error_report("%s: missing block name", __func__);
2588                 mark_source_rp_bad(ms);
2589                 goto out;
2590             }
2591             /* Format: len (1B) + idstr (<255B). This ends the idstr. */
2592             buf[buf[0] + 1] = '\0';
2593             if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) {
2594                 mark_source_rp_bad(ms);
2595                 goto out;
2596             }
2597             break;
2598
2599         case MIG_RP_MSG_RESUME_ACK:
2600             tmp32 = ldl_be_p(buf);
2601             if (migrate_handle_rp_resume_ack(ms, tmp32)) {
2602                 mark_source_rp_bad(ms);
2603                 goto out;
2604             }
2605             break;
2606
2607         default:
2608             break;
2609         }
2610     }
2611
2612 out:
2613     res = qemu_file_get_error(rp);
2614     if (res) {
2615         if (res == -EIO && migration_in_postcopy()) {
2616             /*
2617              * Maybe there is something we can do: it looks like a
2618              * network down issue, and we pause for a recovery.
2619              */
2620             if (postcopy_pause_return_path_thread(ms)) {
2621                 /* Reload rp, reset the rest */
2622                 if (rp != ms->rp_state.from_dst_file) {
2623                     qemu_fclose(rp);
2624                     rp = ms->rp_state.from_dst_file;
2625                 }
2626                 ms->rp_state.error = false;
2627                 goto retry;
2628             }
2629         }
2630
2631         trace_source_return_path_thread_bad_end();
2632         mark_source_rp_bad(ms);
2633     }
2634
2635     trace_source_return_path_thread_end();
2636     ms->rp_state.from_dst_file = NULL;
2637     qemu_fclose(rp);
2638     rcu_unregister_thread();
2639     return NULL;
2640 }
2641
2642 static int open_return_path_on_source(MigrationState *ms,
2643                                       bool create_thread)
2644 {
2645
2646     ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
2647     if (!ms->rp_state.from_dst_file) {
2648         return -1;
2649     }
2650
2651     trace_open_return_path_on_source();
2652
2653     if (!create_thread) {
2654         /* We're done */
2655         return 0;
2656     }
2657
2658     qemu_thread_create(&ms->rp_state.rp_thread, "return path",
2659                        source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
2660
2661     trace_open_return_path_on_source_continue();
2662
2663     return 0;
2664 }
2665
2666 /* Returns 0 if the RP was ok, otherwise there was an error on the RP */
2667 static int await_return_path_close_on_source(MigrationState *ms)
2668 {
2669     /*
2670      * If this is a normal exit then the destination will send a SHUT and the
2671      * rp_thread will exit, however if there's an error we need to cause
2672      * it to exit.
2673      */
2674     if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
2675         /*
2676          * shutdown(2), if we have it, will cause it to unblock if it's stuck
2677          * waiting for the destination.
2678          */
2679         qemu_file_shutdown(ms->rp_state.from_dst_file);
2680         mark_source_rp_bad(ms);
2681     }
2682     trace_await_return_path_close_on_source_joining();
2683     qemu_thread_join(&ms->rp_state.rp_thread);
2684     trace_await_return_path_close_on_source_close();
2685     return ms->rp_state.error;
2686 }
2687
2688 /*
2689  * Switch from normal iteration to postcopy
2690  * Returns non-0 on error
2691  */
2692 static int postcopy_start(MigrationState *ms)
2693 {
2694     int ret;
2695     QIOChannelBuffer *bioc;
2696     QEMUFile *fb;
2697     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2698     int64_t bandwidth = migrate_max_postcopy_bandwidth();
2699     bool restart_block = false;
2700     int cur_state = MIGRATION_STATUS_ACTIVE;
2701     if (!migrate_pause_before_switchover()) {
2702         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
2703                           MIGRATION_STATUS_POSTCOPY_ACTIVE);
2704     }
2705
2706     trace_postcopy_start();
2707     qemu_mutex_lock_iothread();
2708     trace_postcopy_start_set_run();
2709
2710     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2711     global_state_store();
2712     ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2713     if (ret < 0) {
2714         goto fail;
2715     }
2716
2717     ret = migration_maybe_pause(ms, &cur_state,
2718                                 MIGRATION_STATUS_POSTCOPY_ACTIVE);
2719     if (ret < 0) {
2720         goto fail;
2721     }
2722
2723     ret = bdrv_inactivate_all();
2724     if (ret < 0) {
2725         goto fail;
2726     }
2727     restart_block = true;
2728
2729     /*
2730      * Cause any non-postcopiable, but iterative devices to
2731      * send out their final data.
2732      */
2733     qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false);
2734
2735     /*
2736      * in Finish migrate and with the io-lock held everything should
2737      * be quiet, but we've potentially still got dirty pages and we
2738      * need to tell the destination to throw any pages it's already received
2739      * that are dirty
2740      */
2741     if (migrate_postcopy_ram()) {
2742         if (ram_postcopy_send_discard_bitmap(ms)) {
2743             error_report("postcopy send discard bitmap failed");
2744             goto fail;
2745         }
2746     }
2747
2748     /*
2749      * send rest of state - note things that are doing postcopy
2750      * will notice we're in POSTCOPY_ACTIVE and not actually
2751      * wrap their state up here
2752      */
2753     /* 0 max-postcopy-bandwidth means unlimited */
2754     if (!bandwidth) {
2755         qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
2756     } else {
2757         qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO);
2758     }
2759     if (migrate_postcopy_ram()) {
2760         /* Ping just for debugging, helps line traces up */
2761         qemu_savevm_send_ping(ms->to_dst_file, 2);
2762     }
2763
2764     /*
2765      * While loading the device state we may trigger page transfer
2766      * requests and the fd must be free to process those, and thus
2767      * the destination must read the whole device state off the fd before
2768      * it starts processing it.  Unfortunately the ad-hoc migration format
2769      * doesn't allow the destination to know the size to read without fully
2770      * parsing it through each devices load-state code (especially the open
2771      * coded devices that use get/put).
2772      * So we wrap the device state up in a package with a length at the start;
2773      * to do this we use a qemu_buf to hold the whole of the device state.
2774      */
2775     bioc = qio_channel_buffer_new(4096);
2776     qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer");
2777     fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
2778     object_unref(OBJECT(bioc));
2779
2780     /*
2781      * Make sure the receiver can get incoming pages before we send the rest
2782      * of the state
2783      */
2784     qemu_savevm_send_postcopy_listen(fb);
2785
2786     qemu_savevm_state_complete_precopy(fb, false, false);
2787     if (migrate_postcopy_ram()) {
2788         qemu_savevm_send_ping(fb, 3);
2789     }
2790
2791     qemu_savevm_send_postcopy_run(fb);
2792
2793     /* <><> end of stuff going into the package */
2794
2795     /* Last point of recovery; as soon as we send the package the destination
2796      * can open devices and potentially start running.
2797      * Lets just check again we've not got any errors.
2798      */
2799     ret = qemu_file_get_error(ms->to_dst_file);
2800     if (ret) {
2801         error_report("postcopy_start: Migration stream errored (pre package)");
2802         goto fail_closefb;
2803     }
2804
2805     restart_block = false;
2806
2807     /* Now send that blob */
2808     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
2809         goto fail_closefb;
2810     }
2811     qemu_fclose(fb);
2812
2813     /* Send a notify to give a chance for anything that needs to happen
2814      * at the transition to postcopy and after the device state; in particular
2815      * spice needs to trigger a transition now
2816      */
2817     ms->postcopy_after_devices = true;
2818     notifier_list_notify(&migration_state_notifiers, ms);
2819
2820     ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
2821
2822     qemu_mutex_unlock_iothread();
2823
2824     if (migrate_postcopy_ram()) {
2825         /*
2826          * Although this ping is just for debug, it could potentially be
2827          * used for getting a better measurement of downtime at the source.
2828          */
2829         qemu_savevm_send_ping(ms->to_dst_file, 4);
2830     }
2831
2832     if (migrate_release_ram()) {
2833         ram_postcopy_migrated_memory_release(ms);
2834     }
2835
2836     ret = qemu_file_get_error(ms->to_dst_file);
2837     if (ret) {
2838         error_report("postcopy_start: Migration stream errored");
2839         migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2840                               MIGRATION_STATUS_FAILED);
2841     }
2842
2843     return ret;
2844
2845 fail_closefb:
2846     qemu_fclose(fb);
2847 fail:
2848     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
2849                           MIGRATION_STATUS_FAILED);
2850     if (restart_block) {
2851         /* A failure happened early enough that we know the destination hasn't
2852          * accessed block devices, so we're safe to recover.
2853          */
2854         Error *local_err = NULL;
2855
2856         bdrv_invalidate_cache_all(&local_err);
2857         if (local_err) {
2858             error_report_err(local_err);
2859         }
2860     }
2861     qemu_mutex_unlock_iothread();
2862     return -1;
2863 }
2864
2865 /**
2866  * migration_maybe_pause: Pause if required to by
2867  * migrate_pause_before_switchover called with the iothread locked
2868  * Returns: 0 on success
2869  */
2870 static int migration_maybe_pause(MigrationState *s,
2871                                  int *current_active_state,
2872                                  int new_state)
2873 {
2874     if (!migrate_pause_before_switchover()) {
2875         return 0;
2876     }
2877
2878     /* Since leaving this state is not atomic with posting the semaphore
2879      * it's possible that someone could have issued multiple migrate_continue
2880      * and the semaphore is incorrectly positive at this point;
2881      * the docs say it's undefined to reinit a semaphore that's already
2882      * init'd, so use timedwait to eat up any existing posts.
2883      */
2884     while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) {
2885         /* This block intentionally left blank */
2886     }
2887
2888     /*
2889      * If the migration is cancelled when it is in the completion phase,
2890      * the migration state is set to MIGRATION_STATUS_CANCELLING.
2891      * So we don't need to wait a semaphore, otherwise we would always
2892      * wait for the 'pause_sem' semaphore.
2893      */
2894     if (s->state != MIGRATION_STATUS_CANCELLING) {
2895         qemu_mutex_unlock_iothread();
2896         migrate_set_state(&s->state, *current_active_state,
2897                           MIGRATION_STATUS_PRE_SWITCHOVER);
2898         qemu_sem_wait(&s->pause_sem);
2899         migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER,
2900                           new_state);
2901         *current_active_state = new_state;
2902         qemu_mutex_lock_iothread();
2903     }
2904
2905     return s->state == new_state ? 0 : -EINVAL;
2906 }
2907
2908 /**
2909  * migration_completion: Used by migration_thread when there's not much left.
2910  *   The caller 'breaks' the loop when this returns.
2911  *
2912  * @s: Current migration state
2913  */
2914 static void migration_completion(MigrationState *s)
2915 {
2916     int ret;
2917     int current_active_state = s->state;
2918
2919     if (s->state == MIGRATION_STATUS_ACTIVE) {
2920         qemu_mutex_lock_iothread();
2921         s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
2922         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL);
2923         s->vm_was_running = runstate_is_running();
2924         ret = global_state_store();
2925
2926         if (!ret) {
2927             bool inactivate = !migrate_colo_enabled();
2928             ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
2929             if (ret >= 0) {
2930                 ret = migration_maybe_pause(s, &current_active_state,
2931                                             MIGRATION_STATUS_DEVICE);
2932             }
2933             if (ret >= 0) {
2934                 qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
2935                 ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false,
2936                                                          inactivate);
2937             }
2938             if (inactivate && ret >= 0) {
2939                 s->block_inactive = true;
2940             }
2941         }
2942         qemu_mutex_unlock_iothread();
2943
2944         if (ret < 0) {
2945             goto fail;
2946         }
2947     } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
2948         trace_migration_completion_postcopy_end();
2949
2950         qemu_savevm_state_complete_postcopy(s->to_dst_file);
2951         trace_migration_completion_postcopy_end_after_complete();
2952     }
2953
2954     /*
2955      * If rp was opened we must clean up the thread before
2956      * cleaning everything else up (since if there are no failures
2957      * it will wait for the destination to send it's status in
2958      * a SHUT command).
2959      */
2960     if (s->rp_state.from_dst_file) {
2961         int rp_error;
2962         trace_migration_return_path_end_before();
2963         rp_error = await_return_path_close_on_source(s);
2964         trace_migration_return_path_end_after(rp_error);
2965         if (rp_error) {
2966             goto fail_invalidate;
2967         }
2968     }
2969
2970     if (qemu_file_get_error(s->to_dst_file)) {
2971         trace_migration_completion_file_err();
2972         goto fail_invalidate;
2973     }
2974
2975     if (!migrate_colo_enabled()) {
2976         migrate_set_state(&s->state, current_active_state,
2977                           MIGRATION_STATUS_COMPLETED);
2978     }
2979
2980     return;
2981
2982 fail_invalidate:
2983     /* If not doing postcopy, vm_start() will be called: let's regain
2984      * control on images.
2985      */
2986     if (s->state == MIGRATION_STATUS_ACTIVE ||
2987         s->state == MIGRATION_STATUS_DEVICE) {
2988         Error *local_err = NULL;
2989
2990         qemu_mutex_lock_iothread();
2991         bdrv_invalidate_cache_all(&local_err);
2992         if (local_err) {
2993             error_report_err(local_err);
2994         } else {
2995             s->block_inactive = false;
2996         }
2997         qemu_mutex_unlock_iothread();
2998     }
2999
3000 fail:
3001     migrate_set_state(&s->state, current_active_state,
3002                       MIGRATION_STATUS_FAILED);
3003 }
3004
3005 bool migrate_colo_enabled(void)
3006 {
3007     MigrationState *s = migrate_get_current();
3008     return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO];
3009 }
3010
3011 typedef enum MigThrError {
3012     /* No error detected */
3013     MIG_THR_ERR_NONE = 0,
3014     /* Detected error, but resumed successfully */
3015     MIG_THR_ERR_RECOVERED = 1,
3016     /* Detected fatal error, need to exit */
3017     MIG_THR_ERR_FATAL = 2,
3018 } MigThrError;
3019
3020 static int postcopy_resume_handshake(MigrationState *s)
3021 {
3022     qemu_savevm_send_postcopy_resume(s->to_dst_file);
3023
3024     while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3025         qemu_sem_wait(&s->rp_state.rp_sem);
3026     }
3027
3028     if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
3029         return 0;
3030     }
3031
3032     return -1;
3033 }
3034
3035 /* Return zero if success, or <0 for error */
3036 static int postcopy_do_resume(MigrationState *s)
3037 {
3038     int ret;
3039
3040     /*
3041      * Call all the resume_prepare() hooks, so that modules can be
3042      * ready for the migration resume.
3043      */
3044     ret = qemu_savevm_state_resume_prepare(s);
3045     if (ret) {
3046         error_report("%s: resume_prepare() failure detected: %d",
3047                      __func__, ret);
3048         return ret;
3049     }
3050
3051     /*
3052      * Last handshake with destination on the resume (destination will
3053      * switch to postcopy-active afterwards)
3054      */
3055     ret = postcopy_resume_handshake(s);
3056     if (ret) {
3057         error_report("%s: handshake failed: %d", __func__, ret);
3058         return ret;
3059     }
3060
3061     return 0;
3062 }
3063
3064 /*
3065  * We don't return until we are in a safe state to continue current
3066  * postcopy migration.  Returns MIG_THR_ERR_RECOVERED if recovered, or
3067  * MIG_THR_ERR_FATAL if unrecovery failure happened.
3068  */
3069 static MigThrError postcopy_pause(MigrationState *s)
3070 {
3071     assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
3072
3073     while (true) {
3074         QEMUFile *file;
3075
3076         migrate_set_state(&s->state, s->state,
3077                           MIGRATION_STATUS_POSTCOPY_PAUSED);
3078
3079         /* Current channel is possibly broken. Release it. */
3080         assert(s->to_dst_file);
3081         qemu_mutex_lock(&s->qemu_file_lock);
3082         file = s->to_dst_file;
3083         s->to_dst_file = NULL;
3084         qemu_mutex_unlock(&s->qemu_file_lock);
3085
3086         qemu_file_shutdown(file);
3087         qemu_fclose(file);
3088
3089         error_report("Detected IO failure for postcopy. "
3090                      "Migration paused.");
3091
3092         /*
3093          * We wait until things fixed up. Then someone will setup the
3094          * status back for us.
3095          */
3096         while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) {
3097             qemu_sem_wait(&s->postcopy_pause_sem);
3098         }
3099
3100         if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
3101             /* Woken up by a recover procedure. Give it a shot */
3102
3103             /*
3104              * Firstly, let's wake up the return path now, with a new
3105              * return path channel.
3106              */
3107             qemu_sem_post(&s->postcopy_pause_rp_sem);
3108
3109             /* Do the resume logic */
3110             if (postcopy_do_resume(s) == 0) {
3111                 /* Let's continue! */
3112                 trace_postcopy_pause_continued();
3113                 return MIG_THR_ERR_RECOVERED;
3114             } else {
3115                 /*
3116                  * Something wrong happened during the recovery, let's
3117                  * pause again. Pause is always better than throwing
3118                  * data away.
3119                  */
3120                 continue;
3121             }
3122         } else {
3123             /* This is not right... Time to quit. */
3124             return MIG_THR_ERR_FATAL;
3125         }
3126     }
3127 }
3128
3129 static MigThrError migration_detect_error(MigrationState *s)
3130 {
3131     int ret;
3132     int state = s->state;
3133     Error *local_error = NULL;
3134
3135     if (state == MIGRATION_STATUS_CANCELLING ||
3136         state == MIGRATION_STATUS_CANCELLED) {
3137         /* End the migration, but don't set the state to failed */
3138         return MIG_THR_ERR_FATAL;
3139     }
3140
3141     /* Try to detect any file errors */
3142     ret = qemu_file_get_error_obj(s->to_dst_file, &local_error);
3143     if (!ret) {
3144         /* Everything is fine */
3145         assert(!local_error);
3146         return MIG_THR_ERR_NONE;
3147     }
3148
3149     if (local_error) {
3150         migrate_set_error(s, local_error);
3151         error_free(local_error);
3152     }
3153
3154     if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) {
3155         /*
3156          * For postcopy, we allow the network to be down for a
3157          * while. After that, it can be continued by a
3158          * recovery phase.
3159          */
3160         return postcopy_pause(s);
3161     } else {
3162         /*
3163          * For precopy (or postcopy with error outside IO), we fail
3164          * with no time.
3165          */
3166         migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED);
3167         trace_migration_thread_file_err();
3168
3169         /* Time to stop the migration, now. */
3170         return MIG_THR_ERR_FATAL;
3171     }
3172 }
3173
3174 /* How many bytes have we transferred since the beginning of the migration */
3175 static uint64_t migration_total_bytes(MigrationState *s)
3176 {
3177     return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes;
3178 }
3179
3180 static void migration_calculate_complete(MigrationState *s)
3181 {
3182     uint64_t bytes = migration_total_bytes(s);
3183     int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3184     int64_t transfer_time;
3185
3186     s->total_time = end_time - s->start_time;
3187     if (!s->downtime) {
3188         /*
3189          * It's still not set, so we are precopy migration.  For
3190          * postcopy, downtime is calculated during postcopy_start().
3191          */
3192         s->downtime = end_time - s->downtime_start;
3193     }
3194
3195     transfer_time = s->total_time - s->setup_time;
3196     if (transfer_time) {
3197         s->mbps = ((double) bytes * 8.0) / transfer_time / 1000;
3198     }
3199 }
3200
3201 static void update_iteration_initial_status(MigrationState *s)
3202 {
3203     /*
3204      * Update these three fields at the same time to avoid mismatch info lead
3205      * wrong speed calculation.
3206      */
3207     s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3208     s->iteration_initial_bytes = migration_total_bytes(s);
3209     s->iteration_initial_pages = ram_get_total_transferred_pages();
3210 }
3211
3212 static void migration_update_counters(MigrationState *s,
3213                                       int64_t current_time)
3214 {
3215     uint64_t transferred, transferred_pages, time_spent;
3216     uint64_t current_bytes; /* bytes transferred since the beginning */
3217     double bandwidth;
3218
3219     if (current_time < s->iteration_start_time + BUFFER_DELAY) {
3220         return;
3221     }
3222
3223     current_bytes = migration_total_bytes(s);
3224     transferred = current_bytes - s->iteration_initial_bytes;
3225     time_spent = current_time - s->iteration_start_time;
3226     bandwidth = (double)transferred / time_spent;
3227     s->threshold_size = bandwidth * s->parameters.downtime_limit;
3228
3229     s->mbps = (((double) transferred * 8.0) /
3230                ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
3231
3232     transferred_pages = ram_get_total_transferred_pages() -
3233                             s->iteration_initial_pages;
3234     s->pages_per_second = (double) transferred_pages /
3235                              (((double) time_spent / 1000.0));
3236
3237     /*
3238      * if we haven't sent anything, we don't want to
3239      * recalculate. 10000 is a small enough number for our purposes
3240      */
3241     if (ram_counters.dirty_pages_rate && transferred > 10000) {
3242         s->expected_downtime = ram_counters.remaining / bandwidth;
3243     }
3244
3245     qemu_file_reset_rate_limit(s->to_dst_file);
3246
3247     update_iteration_initial_status(s);
3248
3249     trace_migrate_transferred(transferred, time_spent,
3250                               bandwidth, s->threshold_size);
3251 }
3252
3253 /* Migration thread iteration status */
3254 typedef enum {
3255     MIG_ITERATE_RESUME,         /* Resume current iteration */
3256     MIG_ITERATE_SKIP,           /* Skip current iteration */
3257     MIG_ITERATE_BREAK,          /* Break the loop */
3258 } MigIterateState;
3259
3260 /*
3261  * Return true if continue to the next iteration directly, false
3262  * otherwise.
3263  */
3264 static MigIterateState migration_iteration_run(MigrationState *s)
3265 {
3266     uint64_t pending_size, pend_pre, pend_compat, pend_post;
3267     bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE;
3268
3269     qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre,
3270                               &pend_compat, &pend_post);
3271     pending_size = pend_pre + pend_compat + pend_post;
3272
3273     trace_migrate_pending(pending_size, s->threshold_size,
3274                           pend_pre, pend_compat, pend_post);
3275
3276     if (pending_size && pending_size >= s->threshold_size) {
3277         /* Still a significant amount to transfer */
3278         if (!in_postcopy && pend_pre <= s->threshold_size &&
3279             atomic_read(&s->start_postcopy)) {
3280             if (postcopy_start(s)) {
3281                 error_report("%s: postcopy failed to start", __func__);
3282             }
3283             return MIG_ITERATE_SKIP;
3284         }
3285         /* Just another iteration step */
3286         qemu_savevm_state_iterate(s->to_dst_file, in_postcopy);
3287     } else {
3288         trace_migration_thread_low_pending(pending_size);
3289         migration_completion(s);
3290         return MIG_ITERATE_BREAK;
3291     }
3292
3293     return MIG_ITERATE_RESUME;
3294 }
3295
3296 static void migration_iteration_finish(MigrationState *s)
3297 {
3298     /* If we enabled cpu throttling for auto-converge, turn it off. */
3299     cpu_throttle_stop();
3300
3301     qemu_mutex_lock_iothread();
3302     switch (s->state) {
3303     case MIGRATION_STATUS_COMPLETED:
3304         migration_calculate_complete(s);
3305         runstate_set(RUN_STATE_POSTMIGRATE);
3306         break;
3307
3308     case MIGRATION_STATUS_ACTIVE:
3309         /*
3310          * We should really assert here, but since it's during
3311          * migration, let's try to reduce the usage of assertions.
3312          */
3313         if (!migrate_colo_enabled()) {
3314             error_report("%s: critical error: calling COLO code without "
3315                          "COLO enabled", __func__);
3316         }
3317         migrate_start_colo_process(s);
3318         /*
3319          * Fixme: we will run VM in COLO no matter its old running state.
3320          * After exited COLO, we will keep running.
3321          */
3322         s->vm_was_running = true;
3323         /* Fallthrough */
3324     case MIGRATION_STATUS_FAILED:
3325     case MIGRATION_STATUS_CANCELLED:
3326     case MIGRATION_STATUS_CANCELLING:
3327         if (s->vm_was_running) {
3328             vm_start();
3329         } else {
3330             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
3331                 runstate_set(RUN_STATE_POSTMIGRATE);
3332             }
3333         }
3334         break;
3335
3336     default:
3337         /* Should not reach here, but if so, forgive the VM. */
3338         error_report("%s: Unknown ending state %d", __func__, s->state);
3339         break;
3340     }
3341     migrate_fd_cleanup_schedule(s);
3342     qemu_mutex_unlock_iothread();
3343 }
3344
3345 void migration_make_urgent_request(void)
3346 {
3347     qemu_sem_post(&migrate_get_current()->rate_limit_sem);
3348 }
3349
3350 void migration_consume_urgent_request(void)
3351 {
3352     qemu_sem_wait(&migrate_get_current()->rate_limit_sem);
3353 }
3354
3355 /* Returns true if the rate limiting was broken by an urgent request */
3356 bool migration_rate_limit(void)
3357 {
3358     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
3359     MigrationState *s = migrate_get_current();
3360
3361     bool urgent = false;
3362     migration_update_counters(s, now);
3363     if (qemu_file_rate_limit(s->to_dst_file)) {
3364
3365         if (qemu_file_get_error(s->to_dst_file)) {
3366             return false;
3367         }
3368         /*
3369          * Wait for a delay to do rate limiting OR
3370          * something urgent to post the semaphore.
3371          */
3372         int ms = s->iteration_start_time + BUFFER_DELAY - now;
3373         trace_migration_rate_limit_pre(ms);
3374         if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) {
3375             /*
3376              * We were woken by one or more urgent things but
3377              * the timedwait will have consumed one of them.
3378              * The service routine for the urgent wake will dec
3379              * the semaphore itself for each item it consumes,
3380              * so add this one we just eat back.
3381              */
3382             qemu_sem_post(&s->rate_limit_sem);
3383             urgent = true;
3384         }
3385         trace_migration_rate_limit_post(urgent);
3386     }
3387     return urgent;
3388 }
3389
3390 /*
3391  * Master migration thread on the source VM.
3392  * It drives the migration and pumps the data down the outgoing channel.
3393  */
3394 static void *migration_thread(void *opaque)
3395 {
3396     MigrationState *s = opaque;
3397     int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
3398     MigThrError thr_error;
3399     bool urgent = false;
3400
3401     rcu_register_thread();
3402
3403     object_ref(OBJECT(s));
3404     update_iteration_initial_status(s);
3405
3406     qemu_savevm_state_header(s->to_dst_file);
3407
3408     /*
3409      * If we opened the return path, we need to make sure dst has it
3410      * opened as well.
3411      */
3412     if (s->rp_state.from_dst_file) {
3413         /* Now tell the dest that it should open its end so it can reply */
3414         qemu_savevm_send_open_return_path(s->to_dst_file);
3415
3416         /* And do a ping that will make stuff easier to debug */
3417         qemu_savevm_send_ping(s->to_dst_file, 1);
3418     }
3419
3420     if (migrate_postcopy()) {
3421         /*
3422          * Tell the destination that we *might* want to do postcopy later;
3423          * if the other end can't do postcopy it should fail now, nice and
3424          * early.
3425          */
3426         qemu_savevm_send_postcopy_advise(s->to_dst_file);
3427     }
3428
3429     if (migrate_colo_enabled()) {
3430         /* Notify migration destination that we enable COLO */
3431         qemu_savevm_send_colo_enable(s->to_dst_file);
3432     }
3433
3434     qemu_savevm_state_setup(s->to_dst_file);
3435
3436     if (qemu_savevm_state_guest_unplug_pending()) {
3437         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3438                           MIGRATION_STATUS_WAIT_UNPLUG);
3439
3440         while (s->state == MIGRATION_STATUS_WAIT_UNPLUG &&
3441                qemu_savevm_state_guest_unplug_pending()) {
3442             qemu_sem_timedwait(&s->wait_unplug_sem, 250);
3443         }
3444
3445         migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG,
3446                 MIGRATION_STATUS_ACTIVE);
3447     }
3448
3449     s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
3450     migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3451                       MIGRATION_STATUS_ACTIVE);
3452
3453     trace_migration_thread_setup_complete();
3454
3455     while (migration_is_active(s)) {
3456         if (urgent || !qemu_file_rate_limit(s->to_dst_file)) {
3457             MigIterateState iter_state = migration_iteration_run(s);
3458             if (iter_state == MIG_ITERATE_SKIP) {
3459                 continue;
3460             } else if (iter_state == MIG_ITERATE_BREAK) {
3461                 break;
3462             }
3463         }
3464
3465         /*
3466          * Try to detect any kind of failures, and see whether we
3467          * should stop the migration now.
3468          */
3469         thr_error = migration_detect_error(s);
3470         if (thr_error == MIG_THR_ERR_FATAL) {
3471             /* Stop migration */
3472             break;
3473         } else if (thr_error == MIG_THR_ERR_RECOVERED) {
3474             /*
3475              * Just recovered from a e.g. network failure, reset all
3476              * the local variables. This is important to avoid
3477              * breaking transferred_bytes and bandwidth calculation
3478              */
3479             update_iteration_initial_status(s);
3480         }
3481
3482         urgent = migration_rate_limit();
3483     }
3484
3485     trace_migration_thread_after_loop();
3486     migration_iteration_finish(s);
3487     object_unref(OBJECT(s));
3488     rcu_unregister_thread();
3489     return NULL;
3490 }
3491
3492 void migrate_fd_connect(MigrationState *s, Error *error_in)
3493 {
3494     Error *local_err = NULL;
3495     int64_t rate_limit;
3496     bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED;
3497
3498     s->expected_downtime = s->parameters.downtime_limit;
3499     if (resume) {
3500         assert(s->cleanup_bh);
3501     } else {
3502         assert(!s->cleanup_bh);
3503         s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s);
3504     }
3505     if (error_in) {
3506         migrate_fd_error(s, error_in);
3507         migrate_fd_cleanup(s);
3508         return;
3509     }
3510
3511     if (resume) {
3512         /* This is a resumed migration */
3513         rate_limit = s->parameters.max_postcopy_bandwidth /
3514             XFER_LIMIT_RATIO;
3515     } else {
3516         /* This is a fresh new migration */
3517         rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO;
3518
3519         /* Notify before starting migration thread */
3520         notifier_list_notify(&migration_state_notifiers, s);
3521     }
3522
3523     qemu_file_set_rate_limit(s->to_dst_file, rate_limit);
3524     qemu_file_set_blocking(s->to_dst_file, true);
3525
3526     /*
3527      * Open the return path. For postcopy, it is used exclusively. For
3528      * precopy, only if user specified "return-path" capability would
3529      * QEMU uses the return path.
3530      */
3531     if (migrate_postcopy_ram() || migrate_use_return_path()) {
3532         if (open_return_path_on_source(s, !resume)) {
3533             error_report("Unable to open return-path for postcopy");
3534             migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED);
3535             migrate_fd_cleanup(s);
3536             return;
3537         }
3538     }
3539
3540     if (resume) {
3541         /* Wakeup the main migration thread to do the recovery */
3542         migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED,
3543                           MIGRATION_STATUS_POSTCOPY_RECOVER);
3544         qemu_sem_post(&s->postcopy_pause_sem);
3545         return;
3546     }
3547
3548     if (multifd_save_setup(&local_err) != 0) {
3549         error_report_err(local_err);
3550         migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
3551                           MIGRATION_STATUS_FAILED);
3552         migrate_fd_cleanup(s);
3553         return;
3554     }
3555     qemu_thread_create(&s->thread, "live_migration", migration_thread, s,
3556                        QEMU_THREAD_JOINABLE);
3557     s->migration_thread_running = true;
3558 }
3559
3560 void migration_global_dump(Monitor *mon)
3561 {
3562     MigrationState *ms = migrate_get_current();
3563
3564     monitor_printf(mon, "globals:\n");
3565     monitor_printf(mon, "store-global-state: %s\n",
3566                    ms->store_global_state ? "on" : "off");
3567     monitor_printf(mon, "only-migratable: %s\n",
3568                    only_migratable ? "on" : "off");
3569     monitor_printf(mon, "send-configuration: %s\n",
3570                    ms->send_configuration ? "on" : "off");
3571     monitor_printf(mon, "send-section-footer: %s\n",
3572                    ms->send_section_footer ? "on" : "off");
3573     monitor_printf(mon, "decompress-error-check: %s\n",
3574                    ms->decompress_error_check ? "on" : "off");
3575     monitor_printf(mon, "clear-bitmap-shift: %u\n",
3576                    ms->clear_bitmap_shift);
3577 }
3578
3579 #define DEFINE_PROP_MIG_CAP(name, x)             \
3580     DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false)
3581
3582 static Property migration_properties[] = {
3583     DEFINE_PROP_BOOL("store-global-state", MigrationState,
3584                      store_global_state, true),
3585     DEFINE_PROP_BOOL("send-configuration", MigrationState,
3586                      send_configuration, true),
3587     DEFINE_PROP_BOOL("send-section-footer", MigrationState,
3588                      send_section_footer, true),
3589     DEFINE_PROP_BOOL("decompress-error-check", MigrationState,
3590                       decompress_error_check, true),
3591     DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState,
3592                       clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT),
3593
3594     /* Migration parameters */
3595     DEFINE_PROP_UINT8("x-compress-level", MigrationState,
3596                       parameters.compress_level,
3597                       DEFAULT_MIGRATE_COMPRESS_LEVEL),
3598     DEFINE_PROP_UINT8("x-compress-threads", MigrationState,
3599                       parameters.compress_threads,
3600                       DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT),
3601     DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState,
3602                       parameters.compress_wait_thread, true),
3603     DEFINE_PROP_UINT8("x-decompress-threads", MigrationState,
3604                       parameters.decompress_threads,
3605                       DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT),
3606     DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState,
3607                       parameters.throttle_trigger_threshold,
3608                       DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD),
3609     DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState,
3610                       parameters.cpu_throttle_initial,
3611                       DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL),
3612     DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState,
3613                       parameters.cpu_throttle_increment,
3614                       DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT),
3615     DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState,
3616                       parameters.cpu_throttle_tailslow, false),
3617     DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState,
3618                       parameters.max_bandwidth, MAX_THROTTLE),
3619     DEFINE_PROP_UINT64("x-downtime-limit", MigrationState,
3620                       parameters.downtime_limit,
3621                       DEFAULT_MIGRATE_SET_DOWNTIME),
3622     DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState,
3623                       parameters.x_checkpoint_delay,
3624                       DEFAULT_MIGRATE_X_CHECKPOINT_DELAY),
3625     DEFINE_PROP_UINT8("multifd-channels", MigrationState,
3626                       parameters.multifd_channels,
3627                       DEFAULT_MIGRATE_MULTIFD_CHANNELS),
3628     DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState,
3629                       parameters.multifd_compression,
3630                       DEFAULT_MIGRATE_MULTIFD_COMPRESSION),
3631     DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState,
3632                       parameters.multifd_zlib_level,
3633                       DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL),
3634     DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState,
3635                       parameters.multifd_zstd_level,
3636                       DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL),
3637     DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState,
3638                       parameters.xbzrle_cache_size,
3639                       DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE),
3640     DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState,
3641                       parameters.max_postcopy_bandwidth,
3642                       DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH),
3643     DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState,
3644                       parameters.max_cpu_throttle,
3645                       DEFAULT_MIGRATE_MAX_CPU_THROTTLE),
3646     DEFINE_PROP_SIZE("announce-initial", MigrationState,
3647                       parameters.announce_initial,
3648                       DEFAULT_MIGRATE_ANNOUNCE_INITIAL),
3649     DEFINE_PROP_SIZE("announce-max", MigrationState,
3650                       parameters.announce_max,
3651                       DEFAULT_MIGRATE_ANNOUNCE_MAX),
3652     DEFINE_PROP_SIZE("announce-rounds", MigrationState,
3653                       parameters.announce_rounds,
3654                       DEFAULT_MIGRATE_ANNOUNCE_ROUNDS),
3655     DEFINE_PROP_SIZE("announce-step", MigrationState,
3656                       parameters.announce_step,
3657                       DEFAULT_MIGRATE_ANNOUNCE_STEP),
3658
3659     /* Migration capabilities */
3660     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
3661     DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
3662     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
3663     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
3664     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
3665     DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS),
3666     DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM),
3667     DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO),
3668     DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM),
3669     DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK),
3670     DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
3671     DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
3672
3673     DEFINE_PROP_END_OF_LIST(),
3674 };
3675
3676 static void migration_class_init(ObjectClass *klass, void *data)
3677 {
3678     DeviceClass *dc = DEVICE_CLASS(klass);
3679
3680     dc->user_creatable = false;
3681     device_class_set_props(dc, migration_properties);
3682 }
3683
3684 static void migration_instance_finalize(Object *obj)
3685 {
3686     MigrationState *ms = MIGRATION_OBJ(obj);
3687     MigrationParameters *params = &ms->parameters;
3688
3689     qemu_mutex_destroy(&ms->error_mutex);
3690     qemu_mutex_destroy(&ms->qemu_file_lock);
3691     g_free(params->tls_hostname);
3692     g_free(params->tls_creds);
3693     qemu_sem_destroy(&ms->wait_unplug_sem);
3694     qemu_sem_destroy(&ms->rate_limit_sem);
3695     qemu_sem_destroy(&ms->pause_sem);
3696     qemu_sem_destroy(&ms->postcopy_pause_sem);
3697     qemu_sem_destroy(&ms->postcopy_pause_rp_sem);
3698     qemu_sem_destroy(&ms->rp_state.rp_sem);
3699     error_free(ms->error);
3700 }
3701
3702 static void migration_instance_init(Object *obj)
3703 {
3704     MigrationState *ms = MIGRATION_OBJ(obj);
3705     MigrationParameters *params = &ms->parameters;
3706
3707     ms->state = MIGRATION_STATUS_NONE;
3708     ms->mbps = -1;
3709     ms->pages_per_second = -1;
3710     qemu_sem_init(&ms->pause_sem, 0);
3711     qemu_mutex_init(&ms->error_mutex);
3712
3713     params->tls_hostname = g_strdup("");
3714     params->tls_creds = g_strdup("");
3715
3716     /* Set has_* up only for parameter checks */
3717     params->has_compress_level = true;
3718     params->has_compress_threads = true;
3719     params->has_decompress_threads = true;
3720     params->has_throttle_trigger_threshold = true;
3721     params->has_cpu_throttle_initial = true;
3722     params->has_cpu_throttle_increment = true;
3723     params->has_cpu_throttle_tailslow = true;
3724     params->has_max_bandwidth = true;
3725     params->has_downtime_limit = true;
3726     params->has_x_checkpoint_delay = true;
3727     params->has_block_incremental = true;
3728     params->has_multifd_channels = true;
3729     params->has_multifd_compression = true;
3730     params->has_multifd_zlib_level = true;
3731     params->has_multifd_zstd_level = true;
3732     params->has_xbzrle_cache_size = true;
3733     params->has_max_postcopy_bandwidth = true;
3734     params->has_max_cpu_throttle = true;
3735     params->has_announce_initial = true;
3736     params->has_announce_max = true;
3737     params->has_announce_rounds = true;
3738     params->has_announce_step = true;
3739
3740     qemu_sem_init(&ms->postcopy_pause_sem, 0);
3741     qemu_sem_init(&ms->postcopy_pause_rp_sem, 0);
3742     qemu_sem_init(&ms->rp_state.rp_sem, 0);
3743     qemu_sem_init(&ms->rate_limit_sem, 0);
3744     qemu_sem_init(&ms->wait_unplug_sem, 0);
3745     qemu_mutex_init(&ms->qemu_file_lock);
3746 }
3747
3748 /*
3749  * Return true if check pass, false otherwise. Error will be put
3750  * inside errp if provided.
3751  */
3752 static bool migration_object_check(MigrationState *ms, Error **errp)
3753 {
3754     MigrationCapabilityStatusList *head = NULL;
3755     /* Assuming all off */
3756     bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret;
3757     int i;
3758
3759     if (!migrate_params_check(&ms->parameters, errp)) {
3760         return false;
3761     }
3762
3763     for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
3764         if (ms->enabled_capabilities[i]) {
3765             head = migrate_cap_add(head, i, true);
3766         }
3767     }
3768
3769     ret = migrate_caps_check(cap_list, head, errp);
3770
3771     /* It works with head == NULL */
3772     qapi_free_MigrationCapabilityStatusList(head);
3773
3774     return ret;
3775 }
3776
3777 static const TypeInfo migration_type = {
3778     .name = TYPE_MIGRATION,
3779     /*
3780      * NOTE: TYPE_MIGRATION is not really a device, as the object is
3781      * not created using qdev_new(), it is not attached to the qdev
3782      * device tree, and it is never realized.
3783      *
3784      * TODO: Make this TYPE_OBJECT once QOM provides something like
3785      * TYPE_DEVICE's "-global" properties.
3786      */
3787     .parent = TYPE_DEVICE,
3788     .class_init = migration_class_init,
3789     .class_size = sizeof(MigrationClass),
3790     .instance_size = sizeof(MigrationState),
3791     .instance_init = migration_instance_init,
3792     .instance_finalize = migration_instance_finalize,
3793 };
3794
3795 static void register_migration_types(void)
3796 {
3797     type_register_static(&migration_type);
3798 }
3799
3800 type_init(register_migration_types);