tests/qemu-iotests/108

   1 #!/usr/bin/env bash
   2 # group: rw auto quick
   3 #
   4 # Test case for repairing qcow2 images which cannot be repaired using
   5 # the on-disk refcount structures
   6 #
   7 # Copyright (C) 2014 Red Hat, Inc.
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 2 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 # creator
  24 owner=hreitz@redhat.com
  25
  26 seq="$(basename $0)"
  27 echo "QA output created by $seq"
  28
  29 status=1        # failure is the default!
  30
  31 _cleanup()
  32 {
  33     _cleanup_test_img
  34     if [ -f "$TEST_DIR/qsd.pid" ]; then
  35         qsd_pid=$(cat "$TEST_DIR/qsd.pid")
  36         kill -KILL "$qsd_pid"
  37         fusermount -u "$TEST_DIR/fuse-export" &>/dev/null
  38     fi
  39     rm -f "$TEST_DIR/fuse-export"
  40 }
  41 trap "_cleanup; exit \$status" 0 1 2 3 15
  42
  43 # get standard environment, filters and checks
  44 . ./common.rc
  45 . ./common.filter
  46 . ./common.qemu
  47
  48 # This tests qcow2-specific low-level functionality
  49 _supported_fmt qcow2
  50 _supported_proto file fuse
  51 _supported_os Linux
  52 # This test directly modifies a refblock so it relies on refcount_bits being 16;
  53 # and the low-level modification it performs are not tuned for external data
  54 # files
  55 _unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)' data_file
  56
  57 # This test either needs sudo -n losetup or FUSE exports to work
  58 if sudo -n losetup &>/dev/null; then
  59     loopdev=true
  60 else
  61     loopdev=false
  62
  63     # Check for usable FUSE in the host environment:
  64     if test ! -c "/dev/fuse"; then
  65         _notrun 'No passwordless sudo nor usable /dev/fuse'
  66     fi
  67
  68     # QSD --export fuse will either yield "Parameter 'id' is missing"
  69     # or "Invalid parameter 'fuse'", depending on whether there is
  70     # FUSE support or not.
  71     error=$($QSD --export fuse 2>&1)
  72     if [[ $error = *"'fuse'"* ]]; then
  73         _notrun 'Passwordless sudo for losetup or FUSE support required, but' \
  74                 'neither is available'
  75     fi
  76 fi
  77
  78 echo
  79 echo '=== Repairing an image without any refcount table ==='
  80 echo
  81
  82 _make_test_img 64M
  83 # just write some data
  84 $QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
  85
  86 # refcount_table_offset
  87 poke_file "$TEST_IMG" $((0x30)) "\x00\x00\x00\x00\x00\x00\x00\x00"
  88 # refcount_table_clusters
  89 poke_file "$TEST_IMG" $((0x38)) "\x00\x00\x00\x00"
  90
  91 _check_test_img -r all
  92
  93 $QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
  94
  95 echo
  96 echo '=== Repairing unreferenced data cluster in new refblock area ==='
  97 echo
  98
  99 _make_test_img -o 'cluster_size=512' 64M
 100 # Allocate the first 128 kB in the image (first refblock)
 101 $QEMU_IO -c 'write 0 0x1b200' "$TEST_IMG" | _filter_qemu_io
 102 # should be 131072 == 0x20000
 103 stat -c '%s' "$TEST_IMG"
 104
 105 # Enter a cluster at 128 kB (0x20000)
 106 # XXX: This should be the first free entry in the last L2 table, but we cannot
 107 # be certain
 108 poke_file "$TEST_IMG" $((0x1ccc8)) "\x80\x00\x00\x00\x00\x02\x00\x00"
 109
 110 # Fill the cluster
 111 truncate -s $((0x20200)) "$TEST_IMG"
 112 $QEMU_IO -c "open -o driver=raw $TEST_IMG" -c 'write -P 42 128k 512' \
 113     | _filter_qemu_io
 114
 115 # The data should now appear at this guest offset
 116 $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
 117
 118 # This cluster is unallocated; fix it
 119 _check_test_img -r all
 120
 121 # This repair operation must have allocated a new refblock; and that refblock
 122 # should not overlap with the unallocated data cluster. If it does, the data
 123 # will be damaged, so check it.
 124 $QEMU_IO -c 'read -P 42 0x1b200 512' "$TEST_IMG" | _filter_qemu_io
 125
 126 echo
 127 echo '=== Repairing refblock beyond the image end ==='
 128 echo
 129
 130 echo
 131 echo '--- Otherwise clean ---'
 132 echo
 133
 134 _make_test_img 64M
 135 # Normally, qemu doesn't create empty refblocks, so we just have to do it by
 136 # hand
 137 # XXX: This should be the entry for the second refblock
 138 poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
 139 # Mark that refblock as used
 140 # XXX: This should be the 17th entry (cluster 16) of the first
 141 # refblock
 142 poke_file "$TEST_IMG" $((0x20020)) "\x00\x01"
 143 _check_test_img -r all
 144
 145 echo
 146 echo '--- Refblock is unallocated ---'
 147 echo
 148
 149 _make_test_img 64M
 150 poke_file "$TEST_IMG" $((0x10008)) "\x00\x00\x00\x00\x00\x10\x00\x00"
 151 _check_test_img -r all
 152
 153 echo
 154 echo '--- Signed overflow after the refblock ---'
 155 echo
 156
 157 _make_test_img 64M
 158 poke_file "$TEST_IMG" $((0x10008)) "\x7f\xff\xff\xff\xff\xff\x00\x00"
 159 _check_test_img -r all
 160
 161 echo
 162 echo '--- Unsigned overflow after the refblock ---'
 163 echo
 164
 165 _make_test_img 64M
 166 poke_file "$TEST_IMG" $((0x10008)) "\xff\xff\xff\xff\xff\xff\x00\x00"
 167 _check_test_img -r all
 168
 169 echo
 170 echo '=== Check rebuilt reftable location ==='
 171
 172 # In an earlier version of the refcount rebuild algorithm, the
 173 # reftable was generally placed at the image end (unless something was
 174 # allocated in the area covered by the refblock right before the image
 175 # file end, then we would try to place the reftable in that refblock).
 176 # This was later changed so the reftable would be placed in the
 177 # earliest possible location.  Test this.
 178
 179 echo
 180 echo '--- Does the image size increase? ---'
 181 echo
 182
 183 # First test: Just create some image, write some data to it, and
 184 # resize it so there is free space at the end of the image (enough
 185 # that it spans at least one full refblock, which for cluster_size=512
 186 # images, spans 128k).  With the old algorithm, the reftable would
 187 # have then been placed at the end of the image file, but with the new
 188 # one, it will be put in that free space.
 189 # We want to check whether the size of the image file increases due to
 190 # rebuilding the refcount structures (it should not).
 191
 192 _make_test_img -o 'cluster_size=512' 1M
 193 # Write something
 194 $QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
 195
 196 # Add free space
 197 file_len=$(stat -c '%s' "$TEST_IMG")
 198 truncate -s $((file_len + 256 * 1024)) "$TEST_IMG"
 199
 200 # Corrupt the image by saying the image header was not allocated
 201 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 202 rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
 203 poke_file "$TEST_IMG" $rb_offset "\x00\x00"
 204
 205 # Check whether rebuilding the refcount structures increases the image
 206 # file size
 207 file_len=$(stat -c '%s' "$TEST_IMG")
 208 echo
 209 # The only leaks there can be are the old refcount structures that are
 210 # leaked during rebuilding, no need to clutter the output with them
 211 _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
 212 echo
 213 post_repair_file_len=$(stat -c '%s' "$TEST_IMG")
 214
 215 if [[ $file_len -eq $post_repair_file_len ]]; then
 216     echo 'OK: Image size did not change'
 217 else
 218     echo 'ERROR: Image size differs' \
 219         "($file_len before, $post_repair_file_len after)"
 220 fi
 221
 222 echo
 223 echo '--- Will the reftable occupy a hole specifically left for it?  ---'
 224 echo
 225
 226 # Note: With cluster_size=512, every refblock covers 128k.
 227 # The reftable covers 8M per reftable cluster.
 228
 229 # Create an image that requires two reftable clusters (just because
 230 # this is more interesting than a single-clustered reftable).
 231 _make_test_img -o 'cluster_size=512' 9M
 232 $QEMU_IO -c 'write 0 8M' "$TEST_IMG" | _filter_qemu_io
 233
 234 # Writing 8M will have resized the reftable.  Unfortunately, doing so
 235 # will leave holes in the file, so we need to fill them up so we can
 236 # be sure the whole file is allocated.  Do that by writing
 237 # consecutively smaller chunks starting from 8 MB, until the file
 238 # length increases even with a chunk size of 512.  Then we must have
 239 # filled all holes.
 240 ofs=$((8 * 1024 * 1024))
 241 block_len=$((16 * 1024))
 242 while [[ $block_len -ge 512 ]]; do
 243     file_len=$(stat -c '%s' "$TEST_IMG")
 244     while [[ $(stat -c '%s' "$TEST_IMG") -eq $file_len ]]; do
 245         # Do not include this in the reference output, it does not
 246         # really matter which qemu-io calls we do here exactly
 247         $QEMU_IO -c "write $ofs $block_len" "$TEST_IMG" >/dev/null
 248         ofs=$((ofs + block_len))
 249     done
 250     block_len=$((block_len / 2))
 251 done
 252
 253 # Fill up to 9M (do not include this in the reference output either,
 254 # $ofs is random for all we know)
 255 $QEMU_IO -c "write $ofs $((9 * 1024 * 1024 - ofs))" "$TEST_IMG" >/dev/null
 256
 257 # Make space as follows:
 258 # - For the first refblock: Right at the beginning of the image (this
 259 #   refblock is placed in the first place possible),
 260 # - For the reftable somewhere soon afterwards, still near the
 261 #   beginning of the image (i.e. covered by the first refblock); the
 262 #   reftable too is placed in the first place possible, but only after
 263 #   all refblocks have been placed)
 264 # No space is needed for the other refblocks, because no refblock is
 265 # put before the space it covers.  In this test case, we do not mind
 266 # if they are placed at the image file's end.
 267
 268 # Before we make that space, we have to find out the host offset of
 269 # the area that belonged to the two data clusters at guest offset 4k,
 270 # because we expect the reftable to be placed there, and we will have
 271 # to verify that it is.
 272
 273 l1_offset=$(peek_file_be "$TEST_IMG" 40 8)
 274 l2_offset=$(peek_file_be "$TEST_IMG" $l1_offset 8)
 275 l2_offset=$((l2_offset & 0x00fffffffffffe00))
 276 data_4k_offset=$(peek_file_be "$TEST_IMG" \
 277                  $((l2_offset + 4096 / 512 * 8)) 8)
 278 data_4k_offset=$((data_4k_offset & 0x00fffffffffffe00))
 279
 280 $QEMU_IO -c "discard 0 512" -c "discard 4k 1k" "$TEST_IMG" | _filter_qemu_io
 281
 282 # Corrupt the image by saying the image header was not allocated
 283 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 284 rb_offset=$(peek_file_be "$TEST_IMG" $rt_offset 8)
 285 poke_file "$TEST_IMG" $rb_offset "\x00\x00"
 286
 287 echo
 288 # The only leaks there can be are the old refcount structures that are
 289 # leaked during rebuilding, no need to clutter the output with them
 290 _check_test_img -r all | grep -v '^Repairing cluster.*refcount=1 reference=0'
 291 echo
 292
 293 # Check whether the reftable was put where we expected
 294 rt_offset=$(peek_file_be "$TEST_IMG" 48 8)
 295 if [[ $rt_offset -eq $data_4k_offset ]]; then
 296     echo 'OK: Reftable is where we expect it'
 297 else
 298     echo "ERROR: Reftable is at $rt_offset, but was expected at $data_4k_offset"
 299 fi
 300
 301 echo
 302 echo '--- Rebuilding refcount structures on block devices ---'
 303 echo
 304
 305 # A block device cannot really grow, at least not during qemu-img
 306 # check.  As mentioned in the above cases, rebuilding the refcount
 307 # structure may lead to new refcount structures being written after
 308 # the end of the image, and in the past that happened even if there
 309 # was more than sufficient space in the image.  Such post-EOF writes
 310 # will not work on block devices, so test that the new algorithm
 311 # avoids it.
 312
 313 # If we have passwordless sudo and losetup, we can use those to create
 314 # a block device.  Otherwise, we can resort to qemu's FUSE export to
 315 # create a file that isn't growable, which effectively tests the same
 316 # thing.
 317
 318 _cleanup_test_img
 319 truncate -s $((64 * 1024 * 1024)) "$TEST_IMG"
 320
 321 if $loopdev; then
 322     export_mp=$(sudo -n losetup --show -f "$TEST_IMG")
 323     export_mp_driver=host_device
 324     sudo -n chmod go+rw "$export_mp"
 325 else
 326     # Create non-growable FUSE export that is a bit like an empty
 327     # block device
 328     export_mp="$TEST_DIR/fuse-export"
 329     export_mp_driver=file
 330     touch "$export_mp"
 331
 332     $QSD \
 333         --blockdev file,node-name=export-node,filename="$TEST_IMG" \
 334         --export fuse,id=fuse-export,node-name=export-node,mountpoint="$export_mp",writable=on,growable=off,allow-other=off \
 335         --pidfile "$TEST_DIR/qsd.pid" \
 336         --daemonize
 337 fi
 338
 339 # Now create a qcow2 image on the device -- unfortunately, qemu-img
 340 # create force-creates the file, so we have to resort to the
 341 # blockdev-create job.
 342 _launch_qemu \
 343     --blockdev $export_mp_driver,node-name=file,filename="$export_mp"
 344
 345 _send_qemu_cmd \
 346     $QEMU_HANDLE \
 347     '{ "execute": "qmp_capabilities" }' \
 348     'return'
 349
 350 # Small cluster size again, so the image needs multiple refblocks
 351 _send_qemu_cmd \
 352     $QEMU_HANDLE \
 353     '{ "execute": "blockdev-create",
 354        "arguments": {
 355            "job-id": "create",
 356            "options": {
 357                "driver": "qcow2",
 358                "file": "file",
 359                "size": '$((64 * 1024 * 1024))',
 360                "cluster-size": 512
 361            } } }' \
 362     '"concluded"'
 363
 364 _send_qemu_cmd \
 365     $QEMU_HANDLE \
 366     '{ "execute": "job-dismiss", "arguments": { "id": "create" } }' \
 367     'return'
 368
 369 _send_qemu_cmd \
 370     $QEMU_HANDLE \
 371     '{ "execute": "quit" }' \
 372     'return'
 373
 374 wait=y _cleanup_qemu
 375 echo
 376
 377 # Write some data
 378 $QEMU_IO -c 'write 0 64k' "$export_mp" | _filter_qemu_io
 379
 380 # Corrupt the image by saying the image header was not allocated
 381 rt_offset=$(peek_file_be "$export_mp" 48 8)
 382 rb_offset=$(peek_file_be "$export_mp" $rt_offset 8)
 383 poke_file "$export_mp" $rb_offset "\x00\x00"
 384
 385 # Repairing such a simple case should just work
 386 # (We used to put the reftable at the end of the image file, which can
 387 # never work for non-growable devices.)
 388 echo
 389 TEST_IMG="$export_mp" _check_test_img -r all \
 390     | grep -v '^Repairing cluster.*refcount=1 reference=0'
 391
 392 if $loopdev; then
 393     sudo -n losetup -d "$export_mp"
 394 else
 395     qsd_pid=$(cat "$TEST_DIR/qsd.pid")
 396     kill -TERM "$qsd_pid"
 397     # Wait for process to exit (cannot `wait` because the QSD is daemonized)
 398     while [ -f "$TEST_DIR/qsd.pid" ]; do
 399         true
 400     done
 401 fi
 402
 403 # success, all done
 404 echo '*** done'
 405 rm -f $seq.full
 406 status=0