tests/qemu-iotests/257

   1 #!/usr/bin/env python
   2 #
   3 # Test bitmap-sync backups (incremental, differential, and partials)
   4 #
   5 # Copyright (c) 2019 John Snow for Red Hat, Inc.
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # owner=jsnow@redhat.com
  21
  22 import math
  23 import os
  24
  25 import iotests
  26 from iotests import log, qemu_img
  27
  28 SIZE = 64 * 1024 * 1024
  29 GRANULARITY = 64 * 1024
  30
  31
  32 class Pattern:
  33     def __init__(self, byte, offset, size=GRANULARITY):
  34         self.byte = byte
  35         self.offset = offset
  36         self.size = size
  37
  38     def bits(self, granularity):
  39         lower = self.offset // granularity
  40         upper = (self.offset + self.size - 1) // granularity
  41         return set(range(lower, upper + 1))
  42
  43
  44 class PatternGroup:
  45     """Grouping of Pattern objects. Initialize with an iterable of Patterns."""
  46     def __init__(self, patterns):
  47         self.patterns = patterns
  48
  49     def bits(self, granularity):
  50         """Calculate the unique bits dirtied by this pattern grouping"""
  51         res = set()
  52         for pattern in self.patterns:
  53             res |= pattern.bits(granularity)
  54         return res
  55
  56
  57 GROUPS = [
  58     PatternGroup([
  59         # Batch 0: 4 clusters
  60         Pattern('0x49', 0x0000000),
  61         Pattern('0x6c', 0x0100000),   # 1M
  62         Pattern('0x6f', 0x2000000),   # 32M
  63         Pattern('0x76', 0x3ff0000)]), # 64M - 64K
  64     PatternGroup([
  65         # Batch 1: 6 clusters (3 new)
  66         Pattern('0x65', 0x0000000),   # Full overwrite
  67         Pattern('0x77', 0x00f8000),   # Partial-left (1M-32K)
  68         Pattern('0x72', 0x2008000),   # Partial-right (32M+32K)
  69         Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
  70     PatternGroup([
  71         # Batch 2: 7 clusters (3 new)
  72         Pattern('0x74', 0x0010000),   # Adjacent-right
  73         Pattern('0x69', 0x00e8000),   # Partial-left  (1M-96K)
  74         Pattern('0x6e', 0x2018000),   # Partial-right (32M+96K)
  75         Pattern('0x67', 0x3fe0000,
  76                 2*GRANULARITY)]),     # Overwrite [(64M-128K)-64M)
  77     PatternGroup([
  78         # Batch 3: 8 clusters (5 new)
  79         # Carefully chosen such that nothing re-dirties the one cluster
  80         # that copies out successfully before failure in Group #1.
  81         Pattern('0xaa', 0x0010000,
  82                 3*GRANULARITY),       # Overwrite and 2x Adjacent-right
  83         Pattern('0xbb', 0x00d8000),   # Partial-left (1M-160K)
  84         Pattern('0xcc', 0x2028000),   # Partial-right (32M+160K)
  85         Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
  86 ]
  87
  88
  89 class EmulatedBitmap:
  90     def __init__(self, granularity=GRANULARITY):
  91         self._bits = set()
  92         self.granularity = granularity
  93
  94     def dirty_bits(self, bits):
  95         self._bits |= set(bits)
  96
  97     def dirty_group(self, n):
  98         self.dirty_bits(GROUPS[n].bits(self.granularity))
  99
 100     def clear(self):
 101         self._bits = set()
 102
 103     def clear_bits(self, bits):
 104         self._bits -= set(bits)
 105
 106     def clear_bit(self, bit):
 107         self.clear_bits({bit})
 108
 109     def clear_group(self, n):
 110         self.clear_bits(GROUPS[n].bits(self.granularity))
 111
 112     @property
 113     def first_bit(self):
 114         return sorted(self.bits)[0]
 115
 116     @property
 117     def bits(self):
 118         return self._bits
 119
 120     @property
 121     def count(self):
 122         return len(self.bits)
 123
 124     def compare(self, qmp_bitmap):
 125         """
 126         Print a nice human-readable message checking that a bitmap as reported
 127         by the QMP interface has as many bits set as we expect it to.
 128         """
 129
 130         name = qmp_bitmap.get('name', '(anonymous)')
 131         log("= Checking Bitmap {:s} =".format(name))
 132
 133         want = self.count
 134         have = qmp_bitmap['count'] // qmp_bitmap['granularity']
 135
 136         log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
 137             want, have, "OK!" if want == have else "ERROR!"))
 138         log('')
 139
 140
 141 class Drive:
 142     """Represents, vaguely, a drive attached to a VM.
 143     Includes format, graph, and device information."""
 144
 145     def __init__(self, path, vm=None):
 146         self.path = path
 147         self.vm = vm
 148         self.fmt = None
 149         self.size = None
 150         self.node = None
 151         self.device = None
 152
 153     @property
 154     def name(self):
 155         return self.node or self.device
 156
 157     def img_create(self, fmt, size):
 158         self.fmt = fmt
 159         self.size = size
 160         iotests.qemu_img_create('-f', self.fmt, self.path, str(self.size))
 161
 162     def create_target(self, name, fmt, size):
 163         basename = os.path.basename(self.path)
 164         file_node_name = "file_{}".format(basename)
 165         vm = self.vm
 166
 167         log(vm.command('blockdev-create', job_id='bdc-file-job',
 168                        options={
 169                            'driver': 'file',
 170                            'filename': self.path,
 171                            'size': 0,
 172                        }))
 173         vm.run_job('bdc-file-job')
 174         log(vm.command('blockdev-add', driver='file',
 175                        node_name=file_node_name, filename=self.path))
 176
 177         log(vm.command('blockdev-create', job_id='bdc-fmt-job',
 178                        options={
 179                            'driver': fmt,
 180                            'file': file_node_name,
 181                            'size': size,
 182                        }))
 183         vm.run_job('bdc-fmt-job')
 184         log(vm.command('blockdev-add', driver=fmt,
 185                        node_name=name,
 186                        file=file_node_name))
 187         self.fmt = fmt
 188         self.size = size
 189         self.node = name
 190
 191 def query_bitmaps(vm):
 192     res = vm.qmp("query-block")
 193     return {"bitmaps": {device['device'] or device['qdev']:
 194                         device.get('dirty-bitmaps', []) for
 195                         device in res['return']}}
 196
 197 def get_bitmap(bitmaps, drivename, name, recording=None):
 198     """
 199     get a specific bitmap from the object returned by query_bitmaps.
 200     :param recording: If specified, filter results by the specified value.
 201     """
 202     for bitmap in bitmaps['bitmaps'][drivename]:
 203         if bitmap.get('name', '') == name:
 204             if recording is None:
 205                 return bitmap
 206             elif bitmap.get('recording') == recording:
 207                 return bitmap
 208     return None
 209
 210 def blockdev_backup(vm, device, target, sync, **kwargs):
 211     # Strip any arguments explicitly nulled by the caller:
 212     kwargs = {key: val for key, val in kwargs.items() if val is not None}
 213     result = vm.qmp_log('blockdev-backup',
 214                         device=device,
 215                         target=target,
 216                         sync=sync,
 217                         **kwargs)
 218     return result
 219
 220 def blockdev_backup_mktarget(drive, target_id, filepath, sync, **kwargs):
 221     target_drive = Drive(filepath, vm=drive.vm)
 222     target_drive.create_target(target_id, drive.fmt, drive.size)
 223     blockdev_backup(drive.vm, drive.name, target_id, sync, **kwargs)
 224
 225 def reference_backup(drive, n, filepath):
 226     log("--- Reference Backup #{:d} ---\n".format(n))
 227     target_id = "ref_target_{:d}".format(n)
 228     job_id = "ref_backup_{:d}".format(n)
 229     blockdev_backup_mktarget(drive, target_id, filepath, "full",
 230                              job_id=job_id)
 231     drive.vm.run_job(job_id, auto_dismiss=True)
 232     log('')
 233
 234 def backup(drive, n, filepath, sync, **kwargs):
 235     log("--- Test Backup #{:d} ---\n".format(n))
 236     target_id = "backup_target_{:d}".format(n)
 237     job_id = "backup_{:d}".format(n)
 238     kwargs.setdefault('auto-finalize', False)
 239     blockdev_backup_mktarget(drive, target_id, filepath, sync,
 240                              job_id=job_id, **kwargs)
 241     return job_id
 242
 243 def perform_writes(drive, n):
 244     log("--- Write #{:d} ---\n".format(n))
 245     for pattern in GROUPS[n].patterns:
 246         cmd = "write -P{:s} 0x{:07x} 0x{:x}".format(
 247             pattern.byte,
 248             pattern.offset,
 249             pattern.size)
 250         log(cmd)
 251         log(drive.vm.hmp_qemu_io(drive.name, cmd))
 252     bitmaps = query_bitmaps(drive.vm)
 253     log(bitmaps, indent=2)
 254     log('')
 255     return bitmaps
 256
 257
 258 def compare_images(image, reference, baseimg=None, expected_match=True):
 259     """
 260     Print a nice human-readable message comparing these images.
 261     """
 262     expected_ret = 0 if expected_match else 1
 263     if baseimg:
 264         assert qemu_img("rebase", "-u", "-b", baseimg, image) == 0
 265     ret = qemu_img("compare", image, reference)
 266     log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
 267         image, reference,
 268         "Identical" if ret == 0 else "Mismatch",
 269         "OK!" if ret == expected_ret else "ERROR!"),
 270         filters=[iotests.filter_testfiles])
 271
 272 def test_bitmap_sync(bsync_mode, msync_mode='bitmap', failure=None):
 273     """
 274     Test bitmap backup routines.
 275
 276     :param bsync_mode: Is the Bitmap Sync mode, and can be any of:
 277         - on-success: This is the "incremental" style mode. Bitmaps are
 278                       synchronized to what was copied out only on success.
 279                       (Partial images must be discarded.)
 280         - never:      This is the "differential" style mode.
 281                       Bitmaps are never synchronized.
 282         - always:     This is a "best effort" style mode.
 283                       Bitmaps are always synchronized, regardless of failure.
 284                       (Partial images must be kept.)
 285
 286     :param msync_mode: The mirror sync mode to use for the first backup.
 287                        Can be any one of:
 288         - bitmap: Backups based on bitmap manifest.
 289         - full:   Full backups.
 290         - top:    Full backups of the top layer only.
 291
 292     :param failure: Is the (optional) failure mode, and can be any of:
 293         - None:         No failure. Test the normative path. Default.
 294         - simulated:    Cancel the job right before it completes.
 295                         This also tests writes "during" the job.
 296         - intermediate: This tests a job that fails mid-process and produces
 297                         an incomplete backup. Testing limitations prevent
 298                         testing competing writes.
 299     """
 300     with iotests.FilePaths(['img', 'bsync1', 'bsync2',
 301                             'fbackup0', 'fbackup1', 'fbackup2']) as \
 302                             (img_path, bsync1, bsync2,
 303                              fbackup0, fbackup1, fbackup2), \
 304          iotests.VM() as vm:
 305
 306         mode = "Mode {:s}; Bitmap Sync {:s}".format(msync_mode, bsync_mode)
 307         preposition = "with" if failure else "without"
 308         cond = "{:s} {:s}".format(preposition,
 309                                   "{:s} failure".format(failure) if failure
 310                                   else "failure")
 311         log("\n=== {:s} {:s} ===\n".format(mode, cond))
 312
 313         log('--- Preparing image & VM ---\n')
 314         drive0 = Drive(img_path, vm=vm)
 315         drive0.img_create(iotests.imgfmt, SIZE)
 316         vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
 317         vm.launch()
 318
 319         file_config = {
 320             'driver': 'file',
 321             'filename': drive0.path
 322         }
 323
 324         if failure == 'intermediate':
 325             file_config = {
 326                 'driver': 'blkdebug',
 327                 'image': file_config,
 328                 'set-state': [{
 329                     'event': 'flush_to_disk',
 330                     'state': 1,
 331                     'new_state': 2
 332                 }, {
 333                     'event': 'read_aio',
 334                     'state': 2,
 335                     'new_state': 3
 336                 }],
 337                 'inject-error': [{
 338                     'event': 'read_aio',
 339                     'errno': 5,
 340                     'state': 3,
 341                     'immediately': False,
 342                     'once': True
 343                 }]
 344             }
 345
 346         vm.qmp_log('blockdev-add',
 347                    filters=[iotests.filter_qmp_testfiles],
 348                    node_name="drive0",
 349                    driver=drive0.fmt,
 350                    file=file_config)
 351         drive0.node = 'drive0'
 352         drive0.device = 'device0'
 353         # Use share-rw to allow writes directly to the node;
 354         # The anonymous block-backend for this configuration prevents us
 355         # from using HMP's qemu-io commands to address the device.
 356         vm.qmp_log("device_add", id=drive0.device,
 357                    drive=drive0.name, driver="scsi-hd",
 358                    share_rw=True)
 359         log('')
 360
 361         # 0 - Writes and Reference Backup
 362         perform_writes(drive0, 0)
 363         reference_backup(drive0, 0, fbackup0)
 364         log('--- Add Bitmap ---\n')
 365         vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
 366                    name="bitmap0", granularity=GRANULARITY)
 367         log('')
 368         ebitmap = EmulatedBitmap()
 369
 370         # 1 - Writes and Reference Backup
 371         bitmaps = perform_writes(drive0, 1)
 372         ebitmap.dirty_group(1)
 373         bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
 374         ebitmap.compare(bitmap)
 375         reference_backup(drive0, 1, fbackup1)
 376
 377         # 1 - Test Backup (w/ Optional induced failure)
 378         if failure == 'intermediate':
 379             # Activate blkdebug induced failure for second-to-next read
 380             log(vm.hmp_qemu_io(drive0.name, 'flush'))
 381             log('')
 382         job = backup(drive0, 1, bsync1, msync_mode,
 383                      bitmap="bitmap0", bitmap_mode=bsync_mode)
 384
 385         def _callback():
 386             """Issue writes while the job is open to test bitmap divergence."""
 387             # Note: when `failure` is 'intermediate', this isn't called.
 388             log('')
 389             bitmaps = perform_writes(drive0, 2)
 390             # Named bitmap (static, should be unchanged)
 391             ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
 392             # Anonymous bitmap (dynamic, shows new writes)
 393             anonymous = EmulatedBitmap()
 394             anonymous.dirty_group(2)
 395             anonymous.compare(get_bitmap(bitmaps, drive0.device, '',
 396                                          recording=True))
 397
 398             # Simulate the order in which this will happen:
 399             # group 1 gets cleared first, then group two gets written.
 400             if ((bsync_mode == 'on-success' and not failure) or
 401                 (bsync_mode == 'always')):
 402                 ebitmap.clear()
 403             ebitmap.dirty_group(2)
 404
 405         vm.run_job(job, auto_dismiss=True, auto_finalize=False,
 406                    pre_finalize=_callback,
 407                    cancel=(failure == 'simulated'))
 408         bitmaps = query_bitmaps(vm)
 409         log(bitmaps, indent=2)
 410         log('')
 411
 412         if bsync_mode == 'always' and failure == 'intermediate':
 413             # TOP treats anything allocated as dirty, expect to see:
 414             if msync_mode == 'top':
 415                 ebitmap.dirty_group(0)
 416
 417             # We manage to copy one sector (one bit) before the error.
 418             ebitmap.clear_bit(ebitmap.first_bit)
 419
 420             # Full returns all bits set except what was copied/skipped
 421             if msync_mode == 'full':
 422                 fail_bit = ebitmap.first_bit
 423                 ebitmap.clear()
 424                 ebitmap.dirty_bits(range(fail_bit, SIZE // GRANULARITY))
 425
 426         ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
 427
 428         # 2 - Writes and Reference Backup
 429         bitmaps = perform_writes(drive0, 3)
 430         ebitmap.dirty_group(3)
 431         ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
 432         reference_backup(drive0, 2, fbackup2)
 433
 434         # 2 - Bitmap Backup (In failure modes, this is a recovery.)
 435         job = backup(drive0, 2, bsync2, "bitmap",
 436                      bitmap="bitmap0", bitmap_mode=bsync_mode)
 437         vm.run_job(job, auto_dismiss=True, auto_finalize=False)
 438         bitmaps = query_bitmaps(vm)
 439         log(bitmaps, indent=2)
 440         log('')
 441         if bsync_mode != 'never':
 442             ebitmap.clear()
 443         ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
 444
 445         log('--- Cleanup ---\n')
 446         vm.qmp_log("block-dirty-bitmap-remove",
 447                    node=drive0.name, name="bitmap0")
 448         log(query_bitmaps(vm), indent=2)
 449         vm.shutdown()
 450         log('')
 451
 452         log('--- Verification ---\n')
 453         # 'simulated' failures will actually all pass here because we canceled
 454         # while "pending". This is actually undefined behavior,
 455         # don't rely on this to be true!
 456         compare_images(bsync1, fbackup1, baseimg=fbackup0,
 457                        expected_match=failure != 'intermediate')
 458         if not failure or bsync_mode == 'always':
 459             # Always keep the last backup on success or when using 'always'
 460             base = bsync1
 461         else:
 462             base = fbackup0
 463         compare_images(bsync2, fbackup2, baseimg=base)
 464         compare_images(img_path, fbackup2)
 465         log('')
 466
 467 def test_backup_api():
 468     """
 469     Test malformed and prohibited invocations of the backup API.
 470     """
 471     with iotests.FilePaths(['img', 'bsync1']) as \
 472          (img_path, backup_path), \
 473          iotests.VM() as vm:
 474
 475         log("\n=== API failure tests ===\n")
 476         log('--- Preparing image & VM ---\n')
 477         drive0 = Drive(img_path, vm=vm)
 478         drive0.img_create(iotests.imgfmt, SIZE)
 479         vm.add_device("{},id=scsi0".format(iotests.get_virtio_scsi_device()))
 480         vm.launch()
 481
 482         file_config = {
 483             'driver': 'file',
 484             'filename': drive0.path
 485         }
 486
 487         vm.qmp_log('blockdev-add',
 488                    filters=[iotests.filter_qmp_testfiles],
 489                    node_name="drive0",
 490                    driver=drive0.fmt,
 491                    file=file_config)
 492         drive0.node = 'drive0'
 493         drive0.device = 'device0'
 494         vm.qmp_log("device_add", id=drive0.device,
 495                    drive=drive0.name, driver="scsi-hd")
 496         log('')
 497
 498         target0 = Drive(backup_path, vm=vm)
 499         target0.create_target("backup_target", drive0.fmt, drive0.size)
 500         log('')
 501
 502         vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
 503                    name="bitmap0", granularity=GRANULARITY)
 504         log('')
 505
 506         log('-- Testing invalid QMP commands --\n')
 507
 508         error_cases = {
 509             'incremental': {
 510                 None:        ['on-success', 'always', 'never', None],
 511                 'bitmap404': ['on-success', 'always', 'never', None],
 512                 'bitmap0':   ['always', 'never']
 513             },
 514             'bitmap': {
 515                 None:        ['on-success', 'always', 'never', None],
 516                 'bitmap404': ['on-success', 'always', 'never', None],
 517                 'bitmap0':   [None],
 518             },
 519             'full': {
 520                 None:        ['on-success', 'always', 'never'],
 521                 'bitmap404': ['on-success', 'always', 'never', None],
 522                 'bitmap0':   ['never', None],
 523             },
 524             'top': {
 525                 None:        ['on-success', 'always', 'never'],
 526                 'bitmap404': ['on-success', 'always', 'never', None],
 527                 'bitmap0':   ['never', None],
 528             },
 529             'none': {
 530                 None:        ['on-success', 'always', 'never'],
 531                 'bitmap404': ['on-success', 'always', 'never', None],
 532                 'bitmap0':   ['on-success', 'always', 'never', None],
 533             }
 534         }
 535
 536         # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
 537         for sync_mode in ('incremental', 'bitmap', 'full', 'top', 'none'):
 538             log("-- Sync mode {:s} tests --\n".format(sync_mode))
 539             for bitmap in (None, 'bitmap404', 'bitmap0'):
 540                 for policy in error_cases[sync_mode][bitmap]:
 541                     blockdev_backup(drive0.vm, drive0.name, "backup_target",
 542                                     sync_mode, job_id='api_job',
 543                                     bitmap=bitmap, bitmap_mode=policy)
 544                     log('')
 545
 546
 547 def main():
 548     for bsync_mode in ("never", "on-success", "always"):
 549         for failure in ("simulated", "intermediate", None):
 550             test_bitmap_sync(bsync_mode, "bitmap", failure)
 551
 552     for sync_mode in ('full', 'top'):
 553         for bsync_mode in ('on-success', 'always'):
 554             for failure in ('simulated', 'intermediate', None):
 555                 test_bitmap_sync(bsync_mode, sync_mode, failure)
 556
 557     test_backup_api()
 558
 559 if __name__ == '__main__':
 560     iotests.script_main(main, supported_fmts=['qcow2'],
 561                         supported_protocols=['file'])