tests/qemu-iotests/124

   1 #!/usr/bin/env python3
   2 # group: rw backing
   3 #
   4 # Tests for incremental drive-backup
   5 #
   6 # Copyright (C) 2015 John Snow for Red Hat, Inc.
   7 #
   8 # Based on 056.
   9 #
  10 # This program is free software; you can redistribute it and/or modify
  11 # it under the terms of the GNU General Public License as published by
  12 # the Free Software Foundation; either version 2 of the License, or
  13 # (at your option) any later version.
  14 #
  15 # This program is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 # GNU General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU General Public License
  21 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  22 #
  23
  24 import os
  25 import iotests
  26 from iotests import try_remove
  27
  28
  29 def io_write_patterns(img, patterns):
  30     for pattern in patterns:
  31         iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
  32
  33
  34 def transaction_action(action, **kwargs):
  35     return {
  36         'type': action,
  37         'data': dict((k.replace('_', '-'), v) for k, v in kwargs.items())
  38     }
  39
  40
  41 def transaction_bitmap_clear(node, name, **kwargs):
  42     return transaction_action('block-dirty-bitmap-clear',
  43                               node=node, name=name, **kwargs)
  44
  45
  46 def transaction_drive_backup(device, target, **kwargs):
  47     return transaction_action('drive-backup', job_id=device, device=device,
  48                               target=target, **kwargs)
  49
  50
  51 class Bitmap:
  52     def __init__(self, name, drive):
  53         self.name = name
  54         self.drive = drive
  55         self.num = 0
  56         self.backups = list()
  57
  58     def base_target(self):
  59         return (self.drive['backup'], None)
  60
  61     def new_target(self, num=None):
  62         if num is None:
  63             num = self.num
  64         self.num = num + 1
  65         base = os.path.join(iotests.test_dir,
  66                             "%s.%s." % (self.drive['id'], self.name))
  67         suff = "%i.%s" % (num, self.drive['fmt'])
  68         target = base + "inc" + suff
  69         reference = base + "ref" + suff
  70         self.backups.append((target, reference))
  71         return (target, reference)
  72
  73     def last_target(self):
  74         if self.backups:
  75             return self.backups[-1]
  76         return self.base_target()
  77
  78     def del_target(self):
  79         for image in self.backups.pop():
  80             try_remove(image)
  81         self.num -= 1
  82
  83     def cleanup(self):
  84         for backup in self.backups:
  85             for image in backup:
  86                 try_remove(image)
  87
  88
  89 class TestIncrementalBackupBase(iotests.QMPTestCase):
  90     def __init__(self, *args):
  91         super(TestIncrementalBackupBase, self).__init__(*args)
  92         self.bitmaps = list()
  93         self.files = list()
  94         self.drives = list()
  95         self.vm = iotests.VM()
  96         self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
  97
  98
  99     def setUp(self):
 100         # Create a base image with a distinctive patterning
 101         drive0 = self.add_node('drive0')
 102         self.img_create(drive0['file'], drive0['fmt'])
 103         self.vm.add_drive(drive0['file'], opts='node-name=node0')
 104         self.write_default_pattern(drive0['file'])
 105         self.vm.launch()
 106
 107
 108     def write_default_pattern(self, target):
 109         io_write_patterns(target, (('0x41', 0, 512),
 110                                    ('0xd5', '1M', '32k'),
 111                                    ('0xdc', '32M', '124k')))
 112
 113
 114     def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
 115         if path is None:
 116             path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
 117         if backup is None:
 118             backup = os.path.join(iotests.test_dir,
 119                                   '%s.full.backup.%s' % (node_id, fmt))
 120
 121         self.drives.append({
 122             'id': node_id,
 123             'file': path,
 124             'backup': backup,
 125             'fmt': fmt })
 126         return self.drives[-1]
 127
 128
 129     def img_create(self, img, fmt=iotests.imgfmt, size='64M',
 130                    parent=None, parentFormat=None, **kwargs):
 131         optargs = []
 132         for k,v in kwargs.items():
 133             optargs = optargs + ['-o', '%s=%s' % (k,v)]
 134         args = ['create', '-f', fmt] + optargs + [img, size]
 135         if parent:
 136             if parentFormat is None:
 137                 parentFormat = fmt
 138             args = args + ['-b', parent, '-F', parentFormat]
 139         iotests.qemu_img(*args)
 140         self.files.append(img)
 141
 142
 143     def do_qmp_backup(self, error='Input/output error', **kwargs):
 144         res = self.vm.qmp('drive-backup', **kwargs)
 145         self.assert_qmp(res, 'return', {})
 146         return self.wait_qmp_backup(kwargs['device'], error)
 147
 148
 149     def ignore_job_status_change_events(self):
 150         while True:
 151             e = self.vm.event_wait(name="JOB_STATUS_CHANGE")
 152             if e['data']['status'] == 'null':
 153                 break
 154
 155     def wait_qmp_backup(self, device, error='Input/output error'):
 156         event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
 157                                    match={'data': {'device': device}})
 158         self.assertNotEqual(event, None)
 159         self.ignore_job_status_change_events()
 160
 161         try:
 162             failure = self.dictpath(event, 'data/error')
 163         except AssertionError:
 164             # Backup succeeded.
 165             self.assert_qmp(event, 'data/offset', event['data']['len'])
 166             return True
 167         else:
 168             # Backup failed.
 169             self.assert_qmp(event, 'data/error', error)
 170             return False
 171
 172
 173     def wait_qmp_backup_cancelled(self, device):
 174         event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
 175                                    match={'data': {'device': device}})
 176         self.assertNotEqual(event, None)
 177         self.ignore_job_status_change_events()
 178
 179
 180     def create_anchor_backup(self, drive=None):
 181         if drive is None:
 182             drive = self.drives[-1]
 183         res = self.do_qmp_backup(job_id=drive['id'],
 184                                  device=drive['id'], sync='full',
 185                                  format=drive['fmt'], target=drive['backup'])
 186         self.assertTrue(res)
 187         self.files.append(drive['backup'])
 188         return drive['backup']
 189
 190
 191     def make_reference_backup(self, bitmap=None):
 192         if bitmap is None:
 193             bitmap = self.bitmaps[-1]
 194         _, reference = bitmap.last_target()
 195         res = self.do_qmp_backup(job_id=bitmap.drive['id'],
 196                                  device=bitmap.drive['id'], sync='full',
 197                                  format=bitmap.drive['fmt'], target=reference)
 198         self.assertTrue(res)
 199
 200
 201     def add_bitmap(self, name, drive, **kwargs):
 202         bitmap = Bitmap(name, drive)
 203         self.bitmaps.append(bitmap)
 204         result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
 205                              name=bitmap.name, **kwargs)
 206         self.assert_qmp(result, 'return', {})
 207         return bitmap
 208
 209
 210     def prepare_backup(self, bitmap=None, parent=None, **kwargs):
 211         if bitmap is None:
 212             bitmap = self.bitmaps[-1]
 213         if parent is None:
 214             parent, _ = bitmap.last_target()
 215
 216         target, _ = bitmap.new_target()
 217         self.img_create(target, bitmap.drive['fmt'], parent=parent,
 218                         **kwargs)
 219         return target
 220
 221
 222     def create_incremental(self, bitmap=None, parent=None,
 223                            parentFormat=None, validate=True,
 224                            target=None):
 225         if bitmap is None:
 226             bitmap = self.bitmaps[-1]
 227         if parent is None:
 228             parent, _ = bitmap.last_target()
 229
 230         if target is None:
 231             target = self.prepare_backup(bitmap, parent)
 232         res = self.do_qmp_backup(job_id=bitmap.drive['id'],
 233                                  device=bitmap.drive['id'],
 234                                  sync='incremental', bitmap=bitmap.name,
 235                                  format=bitmap.drive['fmt'], target=target,
 236                                  mode='existing')
 237         if not res:
 238             bitmap.del_target();
 239             self.assertFalse(validate)
 240         else:
 241             self.make_reference_backup(bitmap)
 242         return res
 243
 244
 245     def check_backups(self):
 246         for bitmap in self.bitmaps:
 247             for incremental, reference in bitmap.backups:
 248                 self.assertTrue(iotests.compare_images(incremental, reference))
 249             last = bitmap.last_target()[0]
 250             self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
 251
 252
 253     def hmp_io_writes(self, drive, patterns):
 254         for pattern in patterns:
 255             self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
 256         self.vm.hmp_qemu_io(drive, 'flush')
 257
 258
 259     def do_incremental_simple(self, **kwargs):
 260         self.create_anchor_backup()
 261         self.add_bitmap('bitmap0', self.drives[0], **kwargs)
 262
 263         # Sanity: Create a "hollow" incremental backup
 264         self.create_incremental()
 265         # Three writes: One complete overwrite, one new segment,
 266         # and one partial overlap.
 267         self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
 268                                                   ('0xfe', '16M', '256k'),
 269                                                   ('0x64', '32736k', '64k')))
 270         self.create_incremental()
 271         # Three more writes, one of each kind, like above
 272         self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
 273                                                   ('0x55', '8M', '352k'),
 274                                                   ('0x78', '15872k', '1M')))
 275         self.create_incremental()
 276         self.vm.shutdown()
 277         self.check_backups()
 278
 279
 280     def tearDown(self):
 281         self.vm.shutdown()
 282         for bitmap in self.bitmaps:
 283             bitmap.cleanup()
 284         for filename in self.files:
 285             try_remove(filename)
 286
 287
 288
 289 class TestIncrementalBackup(TestIncrementalBackupBase):
 290     def test_incremental_simple(self):
 291         '''
 292         Test: Create and verify three incremental backups.
 293
 294         Create a bitmap and a full backup before VM execution begins,
 295         then create a series of three incremental backups "during execution,"
 296         i.e.; after IO requests begin modifying the drive.
 297         '''
 298         return self.do_incremental_simple()
 299
 300
 301     def test_small_granularity(self):
 302         '''
 303         Test: Create and verify backups made with a small granularity bitmap.
 304
 305         Perform the same test as test_incremental_simple, but with a granularity
 306         of only 32KiB instead of the present default of 64KiB.
 307         '''
 308         return self.do_incremental_simple(granularity=32768)
 309
 310
 311     def test_large_granularity(self):
 312         '''
 313         Test: Create and verify backups made with a large granularity bitmap.
 314
 315         Perform the same test as test_incremental_simple, but with a granularity
 316         of 128KiB instead of the present default of 64KiB.
 317         '''
 318         return self.do_incremental_simple(granularity=131072)
 319
 320
 321     def test_larger_cluster_target(self):
 322         '''
 323         Test: Create and verify backups made to a larger cluster size target.
 324
 325         With a default granularity of 64KiB, verify that backups made to a
 326         larger cluster size target of 128KiB without a backing file works.
 327         '''
 328         drive0 = self.drives[0]
 329
 330         # Create a cluster_size=128k full backup / "anchor" backup
 331         self.img_create(drive0['backup'], cluster_size='128k')
 332         self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
 333                                            format=drive0['fmt'],
 334                                            target=drive0['backup'],
 335                                            mode='existing'))
 336
 337         # Create bitmap and dirty it with some new writes.
 338         # overwrite [32736, 32799] which will dirty bitmap clusters at
 339         # 32M-64K and 32M. 32M+64K will be left undirtied.
 340         bitmap0 = self.add_bitmap('bitmap0', drive0)
 341         self.hmp_io_writes(drive0['id'],
 342                            (('0xab', 0, 512),
 343                             ('0xfe', '16M', '256k'),
 344                             ('0x64', '32736k', '64k')))
 345         # Check the dirty bitmap stats
 346         self.assertTrue(self.vm.check_bitmap_status(
 347             'node0', bitmap0.name, {
 348                 'name': 'bitmap0',
 349                 'count': 458752,
 350                 'granularity': 65536,
 351                 'persistent': False
 352             }))
 353
 354         # Prepare a cluster_size=128k backup target without a backing file.
 355         (target, _) = bitmap0.new_target()
 356         self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
 357
 358         # Perform Incremental Backup
 359         self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
 360                                            sync='incremental',
 361                                            bitmap=bitmap0.name,
 362                                            format=bitmap0.drive['fmt'],
 363                                            target=target,
 364                                            mode='existing'))
 365         self.make_reference_backup(bitmap0)
 366
 367         # Add the backing file, then compare and exit.
 368         iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
 369                          drive0['backup'], '-F', drive0['fmt'], target)
 370         self.vm.shutdown()
 371         self.check_backups()
 372
 373
 374     def test_incremental_transaction(self):
 375         '''Test: Verify backups made from transactionally created bitmaps.
 376
 377         Create a bitmap "before" VM execution begins, then create a second
 378         bitmap AFTER writes have already occurred. Use transactions to create
 379         a full backup and synchronize both bitmaps to this backup.
 380         Create an incremental backup through both bitmaps and verify that
 381         both backups match the current drive0 image.
 382         '''
 383
 384         drive0 = self.drives[0]
 385         bitmap0 = self.add_bitmap('bitmap0', drive0)
 386         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
 387                                           ('0xfe', '16M', '256k'),
 388                                           ('0x64', '32736k', '64k')))
 389         bitmap1 = self.add_bitmap('bitmap1', drive0)
 390
 391         result = self.vm.qmp('transaction', actions=[
 392             transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
 393             transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
 394             transaction_drive_backup(drive0['id'], drive0['backup'],
 395                                      sync='full', format=drive0['fmt'])
 396         ])
 397         self.assert_qmp(result, 'return', {})
 398         self.wait_until_completed(drive0['id'])
 399         self.files.append(drive0['backup'])
 400
 401         self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
 402                                           ('0x55', '8M', '352k'),
 403                                           ('0x78', '15872k', '1M')))
 404         # Both bitmaps should be correctly in sync.
 405         self.create_incremental(bitmap0)
 406         self.create_incremental(bitmap1)
 407         self.vm.shutdown()
 408         self.check_backups()
 409
 410
 411     def do_transaction_failure_test(self, race=False):
 412         # Create a second drive, with pattern:
 413         drive1 = self.add_node('drive1')
 414         self.img_create(drive1['file'], drive1['fmt'])
 415         io_write_patterns(drive1['file'], (('0x14', 0, 512),
 416                                            ('0x5d', '1M', '32k'),
 417                                            ('0xcd', '32M', '124k')))
 418
 419         # Create a blkdebug interface to this img as 'drive1'
 420         result = self.vm.qmp('blockdev-add',
 421             node_name=drive1['id'],
 422             driver=drive1['fmt'],
 423             file={
 424                 'driver': 'blkdebug',
 425                 'image': {
 426                     'driver': 'file',
 427                     'filename': drive1['file']
 428                 },
 429                 'set-state': [{
 430                     'event': 'flush_to_disk',
 431                     'state': 1,
 432                     'new_state': 2
 433                 }],
 434                 'inject-error': [{
 435                     'event': 'read_aio',
 436                     'errno': 5,
 437                     'state': 2,
 438                     'immediately': False,
 439                     'once': True
 440                 }],
 441             }
 442         )
 443         self.assert_qmp(result, 'return', {})
 444
 445         # Create bitmaps and full backups for both drives
 446         drive0 = self.drives[0]
 447         dr0bm0 = self.add_bitmap('bitmap0', drive0)
 448         dr1bm0 = self.add_bitmap('bitmap0', drive1)
 449         self.create_anchor_backup(drive0)
 450         self.create_anchor_backup(drive1)
 451         self.assert_no_active_block_jobs()
 452         self.assertFalse(self.vm.get_qmp_events(wait=False))
 453
 454         # Emulate some writes
 455         if not race:
 456             self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
 457                                               ('0xfe', '16M', '256k'),
 458                                               ('0x64', '32736k', '64k')))
 459         self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
 460                                           ('0xef', '16M', '256k'),
 461                                           ('0x46', '32736k', '64k')))
 462
 463         # Create incremental backup targets
 464         target0 = self.prepare_backup(dr0bm0)
 465         target1 = self.prepare_backup(dr1bm0)
 466
 467         # Ask for a new incremental backup per-each drive,
 468         # expecting drive1's backup to fail. In the 'race' test,
 469         # we expect drive1 to attempt to cancel the empty drive0 job.
 470         transaction = [
 471             transaction_drive_backup(drive0['id'], target0, sync='incremental',
 472                                      format=drive0['fmt'], mode='existing',
 473                                      bitmap=dr0bm0.name),
 474             transaction_drive_backup(drive1['id'], target1, sync='incremental',
 475                                      format=drive1['fmt'], mode='existing',
 476                                      bitmap=dr1bm0.name)
 477         ]
 478         result = self.vm.qmp('transaction', actions=transaction,
 479                              properties={'completion-mode': 'grouped'} )
 480         self.assert_qmp(result, 'return', {})
 481
 482         # Observe that drive0's backup is cancelled and drive1 completes with
 483         # an error.
 484         self.wait_qmp_backup_cancelled(drive0['id'])
 485         self.assertFalse(self.wait_qmp_backup(drive1['id']))
 486         error = self.vm.event_wait('BLOCK_JOB_ERROR')
 487         self.assert_qmp(error, 'data', {'device': drive1['id'],
 488                                         'action': 'report',
 489                                         'operation': 'read'})
 490         self.assertFalse(self.vm.get_qmp_events(wait=False))
 491         self.assert_no_active_block_jobs()
 492
 493         # Delete drive0's successful target and eliminate our record of the
 494         # unsuccessful drive1 target.
 495         dr0bm0.del_target()
 496         dr1bm0.del_target()
 497         if race:
 498             # Don't re-run the transaction, we only wanted to test the race.
 499             self.vm.shutdown()
 500             return
 501
 502         # Re-run the same transaction:
 503         target0 = self.prepare_backup(dr0bm0)
 504         target1 = self.prepare_backup(dr1bm0)
 505
 506         # Re-run the exact same transaction.
 507         result = self.vm.qmp('transaction', actions=transaction,
 508                              properties={'completion-mode':'grouped'})
 509         self.assert_qmp(result, 'return', {})
 510
 511         # Both should complete successfully this time.
 512         self.assertTrue(self.wait_qmp_backup(drive0['id']))
 513         self.assertTrue(self.wait_qmp_backup(drive1['id']))
 514         self.make_reference_backup(dr0bm0)
 515         self.make_reference_backup(dr1bm0)
 516         self.assertFalse(self.vm.get_qmp_events(wait=False))
 517         self.assert_no_active_block_jobs()
 518
 519         # And the images should of course validate.
 520         self.vm.shutdown()
 521         self.check_backups()
 522
 523     def test_transaction_failure(self):
 524         '''Test: Verify backups made from a transaction that partially fails.
 525
 526         Add a second drive with its own unique pattern, and add a bitmap to each
 527         drive. Use blkdebug to interfere with the backup on just one drive and
 528         attempt to create a coherent incremental backup across both drives.
 529
 530         verify a failure in one but not both, then delete the failed stubs and
 531         re-run the same transaction.
 532
 533         verify that both incrementals are created successfully.
 534         '''
 535         self.do_transaction_failure_test()
 536
 537     def test_transaction_failure_race(self):
 538         '''Test: Verify that transactions with jobs that have no data to
 539         transfer do not cause race conditions in the cancellation of the entire
 540         transaction job group.
 541         '''
 542         self.do_transaction_failure_test(race=True)
 543
 544
 545     def test_sync_dirty_bitmap_missing(self):
 546         self.assert_no_active_block_jobs()
 547         self.files.append(self.err_img)
 548         result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
 549                              sync='incremental', format=self.drives[0]['fmt'],
 550                              target=self.err_img)
 551         self.assert_qmp(result, 'error/class', 'GenericError')
 552
 553
 554     def test_sync_dirty_bitmap_not_found(self):
 555         self.assert_no_active_block_jobs()
 556         self.files.append(self.err_img)
 557         result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
 558                              sync='incremental', bitmap='unknown',
 559                              format=self.drives[0]['fmt'], target=self.err_img)
 560         self.assert_qmp(result, 'error/class', 'GenericError')
 561
 562
 563     def test_sync_dirty_bitmap_bad_granularity(self):
 564         '''
 565         Test: Test what happens if we provide an improper granularity.
 566
 567         The granularity must always be a power of 2.
 568         '''
 569         self.assert_no_active_block_jobs()
 570         self.assertRaises(AssertionError, self.add_bitmap,
 571                           'bitmap0', self.drives[0],
 572                           granularity=64000)
 573
 574     def test_growing_before_backup(self):
 575         '''
 576         Test: Add a bitmap, truncate the image, write past the old
 577               end, do a backup.
 578
 579         Incremental backup should not ignore dirty bits past the old
 580         image end.
 581         '''
 582         self.assert_no_active_block_jobs()
 583
 584         self.create_anchor_backup()
 585
 586         self.add_bitmap('bitmap0', self.drives[0])
 587
 588         res = self.vm.qmp('block_resize', device=self.drives[0]['id'],
 589                           size=(65 * 1048576))
 590         self.assert_qmp(res, 'return', {})
 591
 592         # Dirty the image past the old end
 593         self.vm.hmp_qemu_io(self.drives[0]['id'], 'write 64M 64k')
 594
 595         target = self.prepare_backup(size='65M')
 596         self.create_incremental(target=target)
 597
 598         self.vm.shutdown()
 599         self.check_backups()
 600
 601
 602 class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
 603     '''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
 604
 605     def setUp(self):
 606         drive0 = self.add_node('drive0')
 607         self.img_create(drive0['file'], drive0['fmt'])
 608         self.write_default_pattern(drive0['file'])
 609         self.vm.launch()
 610
 611     def test_incremental_failure(self):
 612         '''Test: Verify backups made after a failure are correct.
 613
 614         Simulate a failure during an incremental backup block job,
 615         emulate additional writes, then create another incremental backup
 616         afterwards and verify that the backup created is correct.
 617         '''
 618
 619         drive0 = self.drives[0]
 620         result = self.vm.qmp('blockdev-add',
 621             node_name=drive0['id'],
 622             driver=drive0['fmt'],
 623             file={
 624                 'driver': 'blkdebug',
 625                 'image': {
 626                     'driver': 'file',
 627                     'filename': drive0['file']
 628                 },
 629                 'set-state': [{
 630                     'event': 'flush_to_disk',
 631                     'state': 1,
 632                     'new_state': 2
 633                 }],
 634                 'inject-error': [{
 635                     'event': 'read_aio',
 636                     'errno': 5,
 637                     'state': 2,
 638                     'immediately': False,
 639                     'once': True
 640                 }],
 641             }
 642         )
 643         self.assert_qmp(result, 'return', {})
 644
 645         self.create_anchor_backup(drive0)
 646         self.add_bitmap('bitmap0', drive0)
 647         # Note: at this point, during a normal execution,
 648         # Assume that the VM resumes and begins issuing IO requests here.
 649
 650         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
 651                                           ('0xfe', '16M', '256k'),
 652                                           ('0x64', '32736k', '64k')))
 653
 654         result = self.create_incremental(validate=False)
 655         self.assertFalse(result)
 656         self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
 657                                           ('0x55', '8M', '352k'),
 658                                           ('0x78', '15872k', '1M')))
 659         self.create_incremental()
 660         self.vm.shutdown()
 661         self.check_backups()
 662
 663     def test_incremental_pause(self):
 664         """
 665         Test an incremental backup that errors into a pause and is resumed.
 666         """
 667
 668         drive0 = self.drives[0]
 669         # NB: The blkdebug script here looks for a "flush, read" pattern.
 670         # The flush occurs in hmp_io_writes, and the read during the block job.
 671         result = self.vm.qmp('blockdev-add',
 672                              node_name=drive0['id'],
 673                              driver=drive0['fmt'],
 674                              file={
 675                                  'driver': 'blkdebug',
 676                                  'image': {
 677                                      'driver': 'file',
 678                                      'filename': drive0['file']
 679                                  },
 680                                  'set-state': [{
 681                                      'event': 'flush_to_disk',
 682                                      'state': 1,
 683                                      'new_state': 2
 684                                  }],
 685                                  'inject-error': [{
 686                                      'event': 'read_aio',
 687                                      'errno': 5,
 688                                      'state': 2,
 689                                      'immediately': False,
 690                                      'once': True
 691                                  }],
 692                              })
 693         self.assert_qmp(result, 'return', {})
 694         self.create_anchor_backup(drive0)
 695         bitmap = self.add_bitmap('bitmap0', drive0)
 696
 697         # Emulate guest activity
 698         self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
 699                                           ('0xfe', '16M', '256k'),
 700                                           ('0x64', '32736k', '64k')))
 701
 702         # Bitmap Status Check
 703         self.assertTrue(self.vm.check_bitmap_status(
 704             drive0['id'], bitmap.name, {
 705                 'count': 458752,
 706                 'granularity': 65536,
 707                 'busy': False,
 708                 'recording': True
 709             }))
 710
 711         # Start backup
 712         parent, _ = bitmap.last_target()
 713         target = self.prepare_backup(bitmap, parent)
 714         res = self.vm.qmp('drive-backup',
 715                           job_id=bitmap.drive['id'],
 716                           device=bitmap.drive['id'],
 717                           sync='incremental',
 718                           bitmap=bitmap.name,
 719                           format=bitmap.drive['fmt'],
 720                           target=target,
 721                           mode='existing',
 722                           on_source_error='stop')
 723         self.assert_qmp(res, 'return', {})
 724
 725         # Wait for the error
 726         event = self.vm.event_wait(name="BLOCK_JOB_ERROR",
 727                                    match={"data":{"device":bitmap.drive['id']}})
 728         self.assert_qmp(event, 'data', {'device': bitmap.drive['id'],
 729                                         'action': 'stop',
 730                                         'operation': 'read'})
 731
 732         # Bitmap Status Check
 733         self.assertTrue(self.vm.check_bitmap_status(
 734             drive0['id'], bitmap.name, {
 735                 'count': 458752,
 736                 'granularity': 65536,
 737                 'busy': True,
 738                 'recording': True
 739             }))
 740
 741         # Resume and check incremental backup for consistency
 742         res = self.vm.qmp('block-job-resume', device=bitmap.drive['id'])
 743         self.assert_qmp(res, 'return', {})
 744         self.wait_qmp_backup(bitmap.drive['id'])
 745
 746         # Bitmap Status Check
 747         self.assertTrue(self.vm.check_bitmap_status(
 748             drive0['id'], bitmap.name, {
 749                 'count': 0,
 750                 'granularity': 65536,
 751                 'busy': False,
 752                 'recording': True
 753             }))
 754
 755         # Finalize / Cleanup
 756         self.make_reference_backup(bitmap)
 757         self.vm.shutdown()
 758         self.check_backups()
 759
 760
 761 if __name__ == '__main__':
 762     iotests.main(supported_fmts=['qcow2'],
 763                  supported_protocols=['file'])