t/t5326-multi-pack-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='exercise basic multi-pack bitmap functionality'
   4 . ./test-lib.sh
   5 . "${TEST_DIRECTORY}/lib-bitmap.sh"
   6
   7 # We'll be writing our own MIDX, so avoid getting confused by the
   8 # automatic ones.
   9 GIT_TEST_MULTI_PACK_INDEX=0
  10 GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
  11
  12 # This test exercise multi-pack bitmap functionality where the object order is
  13 # stored and read from a special chunk within the MIDX, so use the default
  14 # behavior here.
  15 sane_unset GIT_TEST_MIDX_WRITE_REV
  16 sane_unset GIT_TEST_MIDX_READ_RIDX
  17
  18 bitmap_reuse_tests() {
  19         from=$1
  20         to=$2
  21         writeLookupTable=false
  22
  23         for i in $3-${$#}
  24         do
  25                 case $i in
  26                 "pack.writeBitmapLookupTable") writeLookupTable=true;;
  27                 esac
  28         done
  29
  30         test_expect_success "setup pack reuse tests ($from -> $to)" '
  31                 rm -fr repo &&
  32                 git init repo &&
  33                 (
  34                         cd repo &&
  35                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  36                         test_commit_bulk 16 &&
  37                         git tag old-tip &&
  38
  39                         git config core.multiPackIndex true &&
  40                         if test "MIDX" = "$from"
  41                         then
  42                                 git repack -Ad &&
  43                                 git multi-pack-index write --bitmap
  44                         else
  45                                 git repack -Adb
  46                         fi
  47                 )
  48         '
  49
  50         test_expect_success "build bitmap from existing ($from -> $to)" '
  51                 (
  52                         cd repo &&
  53                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  54                         test_commit_bulk --id=further 16 &&
  55                         git tag new-tip &&
  56
  57                         if test "MIDX" = "$to"
  58                         then
  59                                 git repack -d &&
  60                                 git multi-pack-index write --bitmap
  61                         else
  62                                 git repack -Adb
  63                         fi
  64                 )
  65         '
  66
  67         test_expect_success "verify resulting bitmaps ($from -> $to)" '
  68                 (
  69                         cd repo &&
  70                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  71                         git for-each-ref &&
  72                         git rev-list --test-bitmap refs/tags/old-tip &&
  73                         git rev-list --test-bitmap refs/tags/new-tip
  74                 )
  75         '
  76 }
  77
  78 test_midx_bitmap_cases () {
  79         writeLookupTable=false
  80         writeBitmapLookupTable=
  81
  82         for i in "$@"
  83         do
  84                 case $i in
  85                 "pack.writeBitmapLookupTable")
  86                         writeLookupTable=true
  87                         writeBitmapLookupTable="$i"
  88                         ;;
  89                 esac
  90         done
  91
  92         test_expect_success 'setup test_repository' '
  93                 rm -rf * .git &&
  94                 git init &&
  95                 git config pack.writeBitmapLookupTable '"$writeLookupTable"'
  96         '
  97
  98         midx_bitmap_core
  99
 100         bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
 101         bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
 102         bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
 103
 104         test_expect_success 'missing object closure fails gracefully' '
 105                 rm -fr repo &&
 106                 git init repo &&
 107                 test_when_finished "rm -fr repo" &&
 108                 (
 109                         cd repo &&
 110                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 111
 112                         test_commit loose &&
 113                         test_commit packed &&
 114
 115                         # Do not pass "--revs"; we want a pack without the "loose"
 116                         # commit.
 117                         git pack-objects $objdir/pack/pack <<-EOF &&
 118                         $(git rev-parse packed)
 119                         EOF
 120
 121                         test_must_fail git multi-pack-index write --bitmap 2>err &&
 122                         grep "doesn.t have full closure" err &&
 123                         test_path_is_missing $midx
 124                 )
 125         '
 126
 127         midx_bitmap_partial_tests
 128
 129         test_expect_success 'removing a MIDX clears stale bitmaps' '
 130                 rm -fr repo &&
 131                 git init repo &&
 132                 test_when_finished "rm -fr repo" &&
 133                 (
 134                         cd repo &&
 135                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 136                         test_commit base &&
 137                         git repack &&
 138                         git multi-pack-index write --bitmap &&
 139
 140                         # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
 141                         stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
 142                         rm $midx &&
 143
 144                         # Then write a new MIDX.
 145                         test_commit new &&
 146                         git repack &&
 147                         git multi-pack-index write --bitmap &&
 148
 149                         test_path_is_file $midx &&
 150                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 151                         test_path_is_missing $stale_bitmap
 152                 )
 153         '
 154
 155         test_expect_success 'pack.preferBitmapTips' '
 156                 git init repo &&
 157                 test_when_finished "rm -fr repo" &&
 158                 (
 159                         cd repo &&
 160                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 161
 162                         test_commit_bulk --message="%s" 103 &&
 163
 164                         git log --format="%H" >commits.raw &&
 165                         sort <commits.raw >commits &&
 166
 167                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 168                         git update-ref --stdin <refs &&
 169
 170                         git multi-pack-index write --bitmap &&
 171                         test_path_is_file $midx &&
 172                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 173
 174                         test-tool bitmap list-commits | sort >bitmaps &&
 175                         comm -13 bitmaps commits >before &&
 176                         test_line_count = 1 before &&
 177
 178                         perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
 179                                 <before | git update-ref --stdin &&
 180
 181                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 182                         rm -fr $midx &&
 183
 184                         git -c pack.preferBitmapTips=refs/tags/include \
 185                                 multi-pack-index write --bitmap &&
 186                         test-tool bitmap list-commits | sort >bitmaps &&
 187                         comm -13 bitmaps commits >after &&
 188
 189                         ! test_cmp before after
 190                 )
 191         '
 192
 193         test_expect_success 'writing a bitmap with --refs-snapshot' '
 194                 git init repo &&
 195                 test_when_finished "rm -fr repo" &&
 196                 (
 197                         cd repo &&
 198                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 199
 200                         test_commit one &&
 201                         test_commit two &&
 202
 203                         git rev-parse one >snapshot &&
 204
 205                         git repack -ad &&
 206
 207                         # First, write a MIDX which see both refs/tags/one and
 208                         # refs/tags/two (causing both of those commits to receive
 209                         # bitmaps).
 210                         git multi-pack-index write --bitmap &&
 211
 212                         test_path_is_file $midx &&
 213                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 214
 215                         test-tool bitmap list-commits | sort >bitmaps &&
 216                         grep "$(git rev-parse one)" bitmaps &&
 217                         grep "$(git rev-parse two)" bitmaps &&
 218
 219                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 220                         rm -fr $midx &&
 221
 222                         # Then again, but with a refs snapshot which only sees
 223                         # refs/tags/one.
 224                         git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
 225
 226                         test_path_is_file $midx &&
 227                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 228
 229                         test-tool bitmap list-commits | sort >bitmaps &&
 230                         grep "$(git rev-parse one)" bitmaps &&
 231                         ! grep "$(git rev-parse two)" bitmaps
 232                 )
 233         '
 234
 235         test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
 236                 git init repo &&
 237                 test_when_finished "rm -fr repo" &&
 238                 (
 239                         cd repo &&
 240                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 241
 242                         test_commit_bulk --message="%s" 103 &&
 243
 244                         git log --format="%H" >commits.raw &&
 245                         sort <commits.raw >commits &&
 246
 247                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 248                         git update-ref --stdin <refs &&
 249
 250                         git multi-pack-index write --bitmap &&
 251                         test_path_is_file $midx &&
 252                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 253
 254                         test-tool bitmap list-commits | sort >bitmaps &&
 255                         comm -13 bitmaps commits >before &&
 256                         test_line_count = 1 before &&
 257
 258                         (
 259                                 grep -vf before commits.raw &&
 260                                 # mark missing commits as preferred
 261                                 sed "s/^/+/" before
 262                         ) >snapshot &&
 263
 264                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 265                         rm -fr $midx &&
 266
 267                         git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
 268                         test-tool bitmap list-commits | sort >bitmaps &&
 269                         comm -13 bitmaps commits >after &&
 270
 271                         ! test_cmp before after
 272                 )
 273         '
 274
 275         test_expect_success 'hash-cache values are propagated from pack bitmaps' '
 276                 rm -fr repo &&
 277                 git init repo &&
 278                 test_when_finished "rm -fr repo" &&
 279                 (
 280                         cd repo &&
 281                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 282
 283                         test_commit base &&
 284                         test_commit base2 &&
 285                         git repack -adb &&
 286
 287                         test-tool bitmap dump-hashes >pack.raw &&
 288                         test_file_not_empty pack.raw &&
 289                         sort pack.raw >pack.hashes &&
 290
 291                         test_commit new &&
 292                         git repack &&
 293                         git multi-pack-index write --bitmap &&
 294
 295                         test-tool bitmap dump-hashes >midx.raw &&
 296                         sort midx.raw >midx.hashes &&
 297
 298                         # ensure that every namehash in the pack bitmap can be found in
 299                         # the midx bitmap (i.e., that there are no oid-namehash pairs
 300                         # unique to the pack bitmap).
 301                         comm -23 pack.hashes midx.hashes >dropped.hashes &&
 302                         test_must_be_empty dropped.hashes
 303                 )
 304         '
 305
 306         test_expect_success 'no .bitmap is written without any objects' '
 307                 rm -fr repo &&
 308                 git init repo &&
 309                 test_when_finished "rm -fr repo" &&
 310                 (
 311                         cd repo &&
 312                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 313
 314                         empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
 315                         cat >packs <<-EOF &&
 316                         pack-$empty.idx
 317                         EOF
 318
 319                         git multi-pack-index write --bitmap --stdin-packs \
 320                                 <packs 2>err &&
 321
 322                         grep "bitmap without any objects" err &&
 323
 324                         test_path_is_file $midx &&
 325                         test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
 326                 )
 327         '
 328
 329         test_expect_success 'graceful fallback when missing reverse index' '
 330                 rm -fr repo &&
 331                 git init repo &&
 332                 test_when_finished "rm -fr repo" &&
 333                 (
 334                         cd repo &&
 335                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 336
 337                         test_commit base &&
 338
 339                         # write a pack and MIDX bitmap containing base
 340                         git repack -adb &&
 341                         git multi-pack-index write --bitmap &&
 342
 343                         GIT_TEST_MIDX_READ_RIDX=0 \
 344                                 git rev-list --use-bitmap-index HEAD 2>err &&
 345                         ! grep "ignoring extra bitmap file" err
 346                 )
 347         '
 348 }
 349
 350 test_midx_bitmap_cases
 351
 352 test_midx_bitmap_cases "pack.writeBitmapLookupTable"
 353
 354 test_expect_success 'multi-pack-index write writes lookup table if enabled' '
 355         rm -fr repo &&
 356         git init repo &&
 357         test_when_finished "rm -fr repo" &&
 358         (
 359                 cd repo &&
 360                 test_commit base &&
 361                 git config pack.writeBitmapLookupTable true &&
 362                 git repack -ad &&
 363                 GIT_TRACE2_EVENT="$(pwd)/trace" \
 364                         git multi-pack-index write --bitmap &&
 365                 grep "\"label\":\"writing_lookup_table\"" trace
 366         )
 367 '
 368
 369 test_expect_success 'preferred pack change with existing MIDX bitmap' '
 370         git init preferred-pack-with-existing &&
 371         (
 372                 cd preferred-pack-with-existing &&
 373
 374                 test_commit base &&
 375                 test_commit other &&
 376
 377                 git rev-list --objects --no-object-names base >p1.objects &&
 378                 git rev-list --objects --no-object-names other >p2.objects &&
 379
 380                 p1="$(git pack-objects "$objdir/pack/pack" \
 381                         --delta-base-offset <p1.objects)" &&
 382                 p2="$(git pack-objects "$objdir/pack/pack" \
 383                         --delta-base-offset <p2.objects)" &&
 384
 385                 # Generate a MIDX containing the first two packs,
 386                 # marking p1 as preferred, and ensure that it can be
 387                 # successfully cloned.
 388                 git multi-pack-index write --bitmap \
 389                         --preferred-pack="pack-$p1.pack" &&
 390                 test_path_is_file $midx &&
 391                 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 392                 git clone --no-local . clone1 &&
 393
 394                 # Then generate a new pack which sorts ahead of any
 395                 # existing pack (by tweaking the pack prefix).
 396                 test_commit foo &&
 397                 git pack-objects --all --unpacked $objdir/pack/pack0 &&
 398
 399                 # Generate a new MIDX which changes the preferred pack
 400                 # to a pack contained in the existing MIDX.
 401                 git multi-pack-index write --bitmap \
 402                         --preferred-pack="pack-$p2.pack" &&
 403                 test_path_is_file $midx &&
 404                 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 405
 406                 # When the above circumstances are met, the preferred
 407                 # pack should change appropriately and clones should
 408                 # (still) succeed.
 409                 git clone --no-local . clone2
 410         )
 411 '
 412
 413 test_expect_success 'tagged commits are selected for bitmapping' '
 414         rm -fr repo &&
 415         git init repo &&
 416         test_when_finished "rm -fr repo" &&
 417         (
 418                 cd repo &&
 419
 420                 test_commit --annotate base &&
 421                 git repack -d &&
 422
 423                 # Remove refs/heads/main which points at the commit directly,
 424                 # leaving only a reference to the annotated tag.
 425                 git branch -M main &&
 426                 git checkout base &&
 427                 git branch -d main &&
 428
 429                 git multi-pack-index write --bitmap &&
 430
 431                 git rev-parse HEAD >want &&
 432                 test-tool bitmap list-commits >actual &&
 433                 grep $(cat want) actual
 434         )
 435 '
 436
 437 test_expect_success 'do not follow replace objects for MIDX bitmap' '
 438         rm -fr repo &&
 439         git init repo &&
 440         test_when_finished "rm -fr repo" &&
 441         (
 442                 cd repo &&
 443
 444                 test_commit A &&
 445                 test_commit B &&
 446                 git checkout --orphan=orphan A &&
 447                 test_commit orphan &&
 448
 449                 git replace A HEAD &&
 450                 git repack -ad --write-midx --write-bitmap-index &&
 451
 452                 # generating reachability bitmaps with replace refs
 453                 # enabled will result in broken clones
 454                 git clone --no-local --bare . clone.git
 455         )
 456 '
 457
 458 corrupt_file () {
 459         chmod a+w "$1" &&
 460         printf "bogus" | dd of="$1" bs=1 seek="12" conv=notrunc
 461 }
 462
 463 test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
 464         git init valid &&
 465         test_when_finished rm -rf valid &&
 466
 467         test_commit_bulk 20 &&
 468         git repack -adbf &&
 469
 470         # Move pack-bitmap aside so it is not deleted
 471         # in next repack.
 472         packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
 473         mv "$packbitmap" "$packbitmap.bak" &&
 474
 475         test_commit_bulk 10 &&
 476         git repack -b --write-midx &&
 477         midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&
 478
 479         # Copy MIDX bitmap to backup. Copy pack bitmap from backup.
 480         cp "$midxbitmap" "$midxbitmap.bak" &&
 481         cp "$packbitmap.bak" "$packbitmap" &&
 482
 483         # fsck works at first
 484         git fsck 2>err &&
 485         test_must_be_empty err &&
 486
 487         corrupt_file "$packbitmap" &&
 488         test_must_fail git fsck 2>err &&
 489         grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&
 490
 491         cp "$packbitmap.bak" "$packbitmap" &&
 492         corrupt_file "$midxbitmap" &&
 493         test_must_fail git fsck 2>err &&
 494         grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
 495
 496         corrupt_file "$packbitmap" &&
 497         test_must_fail git fsck 2>err &&
 498         grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
 499         grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
 500 '
 501
 502 test_expect_success 'corrupt MIDX with bitmap causes fallback' '
 503         git init corrupt-midx-bitmap &&
 504         (
 505                 cd corrupt-midx-bitmap &&
 506
 507                 test_commit first &&
 508                 git repack -d &&
 509                 test_commit second &&
 510                 git repack -d &&
 511
 512                 git multi-pack-index write --bitmap &&
 513                 checksum=$(midx_checksum $objdir) &&
 514                 for f in $midx $midx-$checksum.bitmap
 515                 do
 516                         mv $f $f.bak || return 1
 517                 done &&
 518
 519                 # pack everything together, invalidating the MIDX
 520                 git repack -ad &&
 521                 # then restore the now-stale MIDX
 522                 for f in $midx $midx-$checksum.bitmap
 523                 do
 524                         mv $f.bak $f || return 1
 525                 done &&
 526
 527                 git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
 528                 # should attempt opening the broken pack twice (once
 529                 # from the attempt to load it via the stale bitmap, and
 530                 # again when attempting to load it from the stale MIDX)
 531                 # before falling back to the non-MIDX case
 532                 test 2 -eq $(grep -c "could not open pack" err) &&
 533                 test 6 -eq $(cat out)
 534         )
 535 '
 536
 537 for allow_pack_reuse in single multi
 538 do
 539         test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
 540                 test_when_finished "rm -rf midx-without-btmp" &&
 541                 git init midx-without-btmp &&
 542                 (
 543                         cd midx-without-btmp &&
 544                         test_commit initial &&
 545
 546                         git repack -Adbl --write-bitmap-index --write-midx &&
 547                         GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
 548                                 pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
 549                         test_must_be_empty err
 550                 )
 551         '
 552 done
 553
 554 test_expect_success 'remove one packfile between MIDX bitmap writes' '
 555         git init remove-pack-between-writes &&
 556         (
 557                 cd remove-pack-between-writes &&
 558
 559                 test_commit A &&
 560                 test_commit B &&
 561                 test_commit C &&
 562
 563                 # Create packs with the prefix "pack-A", "pack-B",
 564                 # "pack-C" to impose a lexicographic order on these
 565                 # packs so the pack being removed is always from the
 566                 # middle.
 567                 packdir=.git/objects/pack &&
 568                 A="$(echo A | git pack-objects $packdir/pack-A --revs)" &&
 569                 B="$(echo B | git pack-objects $packdir/pack-B --revs)" &&
 570                 C="$(echo C | git pack-objects $packdir/pack-C --revs)" &&
 571
 572                 git multi-pack-index write --bitmap &&
 573
 574                 cat >in <<-EOF &&
 575                 pack-A-$A.idx
 576                 pack-C-$C.idx
 577                 EOF
 578                 git multi-pack-index write --bitmap --stdin-packs <in &&
 579
 580                 git rev-list --test-bitmap HEAD
 581         )
 582 '
 583
 584 test_done