t/t5310-pack-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='exercise basic bitmap functionality'
   4
   5 TEST_PASSES_SANITIZE_LEAK=true
   6 . ./test-lib.sh
   7 . "$TEST_DIRECTORY"/lib-bitmap.sh
   8
   9 # Likewise, allow individual tests to control whether or not they use
  10 # the boundary-based traversal.
  11 sane_unset GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL
  12
  13 objpath () {
  14         echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
  15 }
  16
  17 # show objects present in pack ($1 should be associated *.idx)
  18 list_packed_objects () {
  19         git show-index <"$1" >object-list &&
  20         cut -d' ' -f2 object-list
  21 }
  22
  23 # has_any pattern-file content-file
  24 # tests whether content-file has any entry from pattern-file with entries being
  25 # whole lines.
  26 has_any () {
  27         grep -Ff "$1" "$2"
  28 }
  29
  30 test_bitmap_cases () {
  31         writeLookupTable=false
  32         for i in "$@"
  33         do
  34                 case "$i" in
  35                 "pack.writeBitmapLookupTable") writeLookupTable=true;;
  36                 esac
  37         done
  38
  39         test_expect_success 'setup test repository' '
  40                 rm -fr * .git &&
  41                 git init &&
  42                 git config pack.writeBitmapLookupTable '"$writeLookupTable"'
  43         '
  44         setup_bitmap_history
  45
  46         test_expect_success 'setup writing bitmaps during repack' '
  47                 git config repack.writeBitmaps true
  48         '
  49
  50         test_expect_success 'full repack creates bitmaps' '
  51                 GIT_TRACE2_EVENT="$(pwd)/trace" \
  52                         git repack -ad &&
  53                 ls .git/objects/pack/ | grep bitmap >output &&
  54                 test_line_count = 1 output &&
  55                 grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
  56                 grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
  57         '
  58
  59         basic_bitmap_tests
  60
  61         test_expect_success 'pack-objects respects --local (non-local loose)' '
  62                 git init --bare alt.git &&
  63                 echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
  64                 echo content1 >file1 &&
  65                 # non-local loose object which is not present in bitmapped pack
  66                 altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
  67                 # non-local loose object which is also present in bitmapped pack
  68                 git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
  69                 git add file1 &&
  70                 test_tick &&
  71                 git commit -m commit_file1 &&
  72                 echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
  73                 git index-pack 1.pack &&
  74                 list_packed_objects 1.idx >1.objects &&
  75                 printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
  76                 ! has_any nonlocal-loose 1.objects
  77         '
  78
  79         test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
  80                 echo content2 >file2 &&
  81                 blob2=$(git hash-object -w file2) &&
  82                 git add file2 &&
  83                 test_tick &&
  84                 git commit -m commit_file2 &&
  85                 printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
  86                 pack2=$(git pack-objects pack2 <keepobjects) &&
  87                 mv pack2-$pack2.* .git/objects/pack/ &&
  88                 >.git/objects/pack/pack2-$pack2.keep &&
  89                 rm $(objpath $blob2) &&
  90                 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
  91                 git index-pack 2a.pack &&
  92                 list_packed_objects 2a.idx >2a.objects &&
  93                 ! has_any keepobjects 2a.objects
  94         '
  95
  96         test_expect_success 'pack-objects respects --local (non-local pack)' '
  97                 mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
  98                 echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
  99                 git index-pack 2b.pack &&
 100                 list_packed_objects 2b.idx >2b.objects &&
 101                 ! has_any keepobjects 2b.objects
 102         '
 103
 104         test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
 105                 ls .git/objects/pack/ | grep bitmap >output &&
 106                 test_line_count = 1 output &&
 107                 packbitmap=$(basename $(cat output) .bitmap) &&
 108                 list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
 109                 test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
 110                 >.git/objects/pack/$packbitmap.keep &&
 111                 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
 112                 git index-pack 3a.pack &&
 113                 list_packed_objects 3a.idx >3a.objects &&
 114                 ! has_any packbitmap.objects 3a.objects
 115         '
 116
 117         test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
 118                 mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
 119                 rm -f .git/objects/pack/multi-pack-index &&
 120                 test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
 121                 echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
 122                 git index-pack 3b.pack &&
 123                 list_packed_objects 3b.idx >3b.objects &&
 124                 ! has_any packbitmap.objects 3b.objects
 125         '
 126
 127         test_expect_success 'pack-objects to file can use bitmap' '
 128                 # make sure we still have 1 bitmap index from previous tests
 129                 ls .git/objects/pack/ | grep bitmap >output &&
 130                 test_line_count = 1 output &&
 131                 # verify equivalent packs are generated with/without using bitmap index
 132                 packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
 133                 packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
 134                 list_packed_objects packa-$packasha1.idx >packa.objects &&
 135                 list_packed_objects packb-$packbsha1.idx >packb.objects &&
 136                 test_cmp packa.objects packb.objects
 137         '
 138
 139         test_expect_success 'full repack, reusing previous bitmaps' '
 140                 git repack -ad &&
 141                 ls .git/objects/pack/ | grep bitmap >output &&
 142                 test_line_count = 1 output
 143         '
 144
 145         test_expect_success 'fetch (full bitmap)' '
 146                 git --git-dir=clone.git fetch origin second:second &&
 147                 git rev-parse HEAD >expect &&
 148                 git --git-dir=clone.git rev-parse HEAD >actual &&
 149                 test_cmp expect actual
 150         '
 151
 152         test_expect_success 'create objects for missing-HAVE tests' '
 153                 blob=$(echo "missing have" | git hash-object -w --stdin) &&
 154                 tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
 155                 parent=$(echo parent | git commit-tree $tree) &&
 156                 commit=$(echo commit | git commit-tree $tree -p $parent) &&
 157                 cat >revs <<-EOF
 158                 HEAD
 159                 ^HEAD^
 160                 ^$commit
 161                 EOF
 162         '
 163
 164         test_expect_success 'pack-objects respects --incremental' '
 165                 cat >revs2 <<-EOF &&
 166                 HEAD
 167                 $commit
 168                 EOF
 169                 git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
 170                 git index-pack 4.pack &&
 171                 list_packed_objects 4.idx >4.objects &&
 172                 test_line_count = 4 4.objects &&
 173                 git rev-list --objects $commit >revlist &&
 174                 cut -d" " -f1 revlist |sort >objects &&
 175                 test_cmp 4.objects objects
 176         '
 177
 178         test_expect_success 'pack with missing blob' '
 179                 rm $(objpath $blob) &&
 180                 git pack-objects --stdout --revs <revs >/dev/null
 181         '
 182
 183         test_expect_success 'pack with missing tree' '
 184                 rm $(objpath $tree) &&
 185                 git pack-objects --stdout --revs <revs >/dev/null
 186         '
 187
 188         test_expect_success 'pack with missing parent' '
 189                 rm $(objpath $parent) &&
 190                 git pack-objects --stdout --revs <revs >/dev/null
 191         '
 192
 193         test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
 194                 git clone --bare . compat-jgit.git &&
 195                 (
 196                         cd compat-jgit.git &&
 197                         rm -f objects/pack/*.bitmap &&
 198                         jgit gc &&
 199                         git rev-list --test-bitmap HEAD
 200                 )
 201         '
 202
 203         test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
 204                 git clone --bare . compat-us.git &&
 205                 (
 206                         cd compat-us.git &&
 207                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 208                         git repack -adb &&
 209                         # jgit gc will barf if it does not like our bitmaps
 210                         jgit gc
 211                 )
 212         '
 213
 214         test_expect_success 'splitting packs does not generate bogus bitmaps' '
 215                 test-tool genrandom foo $((1024 * 1024)) >rand &&
 216                 git add rand &&
 217                 git commit -m "commit with big file" &&
 218                 git -c pack.packSizeLimit=500k repack -adb &&
 219                 git init --bare no-bitmaps.git &&
 220                 git -C no-bitmaps.git fetch .. HEAD
 221         '
 222
 223         test_expect_success 'set up reusable pack' '
 224                 rm -f .git/objects/pack/*.keep &&
 225                 git repack -adb &&
 226                 reusable_pack () {
 227                         git for-each-ref --format="%(objectname)" |
 228                         git pack-objects --delta-base-offset --revs --stdout "$@"
 229                 }
 230         '
 231
 232         test_expect_success 'pack reuse respects --honor-pack-keep' '
 233                 test_when_finished "rm -f .git/objects/pack/*.keep" &&
 234                 for i in .git/objects/pack/*.pack
 235                 do
 236                         >${i%.pack}.keep || return 1
 237                 done &&
 238                 reusable_pack --honor-pack-keep >empty.pack &&
 239                 git index-pack empty.pack &&
 240                 git show-index <empty.idx >actual &&
 241                 test_must_be_empty actual
 242         '
 243
 244         test_expect_success 'pack reuse respects --local' '
 245                 mv .git/objects/pack/* alt.git/objects/pack/ &&
 246                 test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
 247                 reusable_pack --local >empty.pack &&
 248                 git index-pack empty.pack &&
 249                 git show-index <empty.idx >actual &&
 250                 test_must_be_empty actual
 251         '
 252
 253         test_expect_success 'pack reuse respects --incremental' '
 254                 reusable_pack --incremental >empty.pack &&
 255                 git index-pack empty.pack &&
 256                 git show-index <empty.idx >actual &&
 257                 test_must_be_empty actual
 258         '
 259
 260         test_expect_success 'truncated bitmap fails gracefully (ewah)' '
 261                 test_config pack.writebitmaphashcache false &&
 262                 test_config pack.writebitmaplookuptable false &&
 263                 git repack -ad &&
 264                 git rev-list --use-bitmap-index --count --all >expect &&
 265                 bitmap=$(ls .git/objects/pack/*.bitmap) &&
 266                 test_when_finished "rm -f $bitmap" &&
 267                 test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
 268                 mv -f $bitmap.tmp $bitmap &&
 269                 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 270                 test_cmp expect actual &&
 271                 test_grep corrupt.ewah.bitmap stderr
 272         '
 273
 274         test_expect_success 'truncated bitmap fails gracefully (cache)' '
 275                 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 276                 git repack -ad &&
 277                 git rev-list --use-bitmap-index --count --all >expect &&
 278                 bitmap=$(ls .git/objects/pack/*.bitmap) &&
 279                 test_when_finished "rm -f $bitmap" &&
 280                 test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
 281                 mv -f $bitmap.tmp $bitmap &&
 282                 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 283                 test_cmp expect actual &&
 284                 test_grep corrupted.bitmap.index stderr
 285         '
 286
 287         # Create a state of history with these properties:
 288         #
 289         #  - refs that allow a client to fetch some new history, while sharing some old
 290         #    history with the server; we use branches delta-reuse-old and
 291         #    delta-reuse-new here
 292         #
 293         #  - the new history contains an object that is stored on the server as a delta
 294         #    against a base that is in the old history
 295         #
 296         #  - the base object is not immediately reachable from the tip of the old
 297         #    history; finding it would involve digging down through history we know the
 298         #    other side has
 299         #
 300         # This should result in a state where fetching from old->new would not
 301         # traditionally reuse the on-disk delta (because we'd have to dig to realize
 302         # that the client has it), but we will do so if bitmaps can tell us cheaply
 303         # that the other side has it.
 304         test_expect_success 'set up thin delta-reuse parent' '
 305                 # This first commit contains the buried base object.
 306                 test-tool genrandom delta 16384 >file &&
 307                 git add file &&
 308                 git commit -m "delta base" &&
 309                 base=$(git rev-parse --verify HEAD:file) &&
 310
 311                 # These intermediate commits bury the base back in history.
 312                 # This becomes the "old" state.
 313                 for i in 1 2 3 4 5
 314                 do
 315                         echo $i >file &&
 316                         git commit -am "intermediate $i" || return 1
 317                 done &&
 318                 git branch delta-reuse-old &&
 319
 320                 # And now our new history has a delta against the buried base. Note
 321                 # that this must be smaller than the original file, since pack-objects
 322                 # prefers to create deltas from smaller objects to larger.
 323                 test-tool genrandom delta 16300 >file &&
 324                 git commit -am "delta result" &&
 325                 delta=$(git rev-parse --verify HEAD:file) &&
 326                 git branch delta-reuse-new &&
 327
 328                 # Repack with bitmaps and double check that we have the expected delta
 329                 # relationship.
 330                 git repack -adb &&
 331                 have_delta $delta $base
 332         '
 333
 334         # Now we can sanity-check the non-bitmap behavior (that the server is not able
 335         # to reuse the delta). This isn't strictly something we care about, so this
 336         # test could be scrapped in the future. But it makes sure that the next test is
 337         # actually triggering the feature we want.
 338         #
 339         # Note that our tools for working with on-the-wire "thin" packs are limited. So
 340         # we actually perform the fetch, retain the resulting pack, and inspect the
 341         # result.
 342         test_expect_success 'fetch without bitmaps ignores delta against old base' '
 343                 test_config pack.usebitmaps false &&
 344                 test_when_finished "rm -rf client.git" &&
 345                 git init --bare client.git &&
 346                 (
 347                         cd client.git &&
 348                         git config transfer.unpackLimit 1 &&
 349                         git fetch .. delta-reuse-old:delta-reuse-old &&
 350                         git fetch .. delta-reuse-new:delta-reuse-new &&
 351                         have_delta $delta $ZERO_OID
 352                 )
 353         '
 354
 355         # And do the same for the bitmap case, where we do expect to find the delta.
 356         test_expect_success 'fetch with bitmaps can reuse old base' '
 357                 test_config pack.usebitmaps true &&
 358                 test_when_finished "rm -rf client.git" &&
 359                 git init --bare client.git &&
 360                 (
 361                         cd client.git &&
 362                         git config transfer.unpackLimit 1 &&
 363                         git fetch .. delta-reuse-old:delta-reuse-old &&
 364                         git fetch .. delta-reuse-new:delta-reuse-new &&
 365                         have_delta $delta $base
 366                 )
 367         '
 368
 369         test_expect_success 'pack.preferBitmapTips' '
 370                 git init repo &&
 371                 test_when_finished "rm -fr repo" &&
 372                 (
 373                         cd repo &&
 374                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 375
 376                         # create enough commits that not all are receive bitmap
 377                         # coverage even if they are all at the tip of some reference.
 378                         test_commit_bulk --message="%s" 103 &&
 379
 380                         git rev-list HEAD >commits.raw &&
 381                         sort <commits.raw >commits &&
 382
 383                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 384                         git update-ref --stdin <refs &&
 385
 386                         git repack -adb &&
 387                         test-tool bitmap list-commits | sort >bitmaps &&
 388
 389                         # remember which commits did not receive bitmaps
 390                         comm -13 bitmaps commits >before &&
 391                         test_file_not_empty before &&
 392
 393                         # mark the commits which did not receive bitmaps as preferred,
 394                         # and generate the bitmap again
 395                         perl -pe "s{^}{create refs/tags/include/$. }" <before |
 396                                 git update-ref --stdin &&
 397                         git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
 398
 399                         # finally, check that the commit(s) without bitmap coverage
 400                         # are not the same ones as before
 401                         test-tool bitmap list-commits | sort >bitmaps &&
 402                         comm -13 bitmaps commits >after &&
 403
 404                         ! test_cmp before after
 405                 )
 406         '
 407
 408         test_expect_success 'pack.preferBitmapTips' '
 409                 git init repo &&
 410                 test_when_finished "rm -rf repo" &&
 411                 (
 412                         cd repo &&
 413                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 414                         test_commit_bulk --message="%s" 103 &&
 415
 416                         cat >>.git/config <<-\EOF &&
 417                         [pack]
 418                                 preferBitmapTips
 419                         EOF
 420                         cat >expect <<-\EOF &&
 421                         error: missing value for '\''pack.preferbitmaptips'\''
 422                         EOF
 423                         git repack -adb 2>actual &&
 424                         test_cmp expect actual
 425                 )
 426         '
 427
 428         test_expect_success 'complains about multiple pack bitmaps' '
 429                 rm -fr repo &&
 430                 git init repo &&
 431                 test_when_finished "rm -fr repo" &&
 432                 (
 433                         cd repo &&
 434                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 435
 436                         test_commit base &&
 437
 438                         git repack -adb &&
 439                         bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
 440                         mv "$bitmap" "$bitmap.bak" &&
 441
 442                         test_commit other &&
 443                         git repack -ab &&
 444
 445                         mv "$bitmap.bak" "$bitmap" &&
 446
 447                         find .git/objects/pack -type f -name "*.pack" >packs &&
 448                         find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
 449                         test_line_count = 2 packs &&
 450                         test_line_count = 2 bitmaps &&
 451
 452                         GIT_TRACE2_EVENT=$(pwd)/trace2.txt git rev-list --use-bitmap-index HEAD &&
 453                         grep "opened bitmap" trace2.txt &&
 454                         grep "ignoring extra bitmap" trace2.txt
 455                 )
 456         '
 457 }
 458
 459 test_bitmap_cases
 460
 461 GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=1
 462 export GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL
 463
 464 test_bitmap_cases
 465
 466 sane_unset GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL
 467
 468 test_expect_success 'incremental repack fails when bitmaps are requested' '
 469         test_commit more-1 &&
 470         test_must_fail git repack -d 2>err &&
 471         test_grep "Incremental repacks are incompatible with bitmap" err
 472 '
 473
 474 test_expect_success 'incremental repack can disable bitmaps' '
 475         test_commit more-2 &&
 476         git repack -d --no-write-bitmap-index
 477 '
 478
 479 test_expect_success 'boundary-based traversal is used when requested' '
 480         git repack -a -d --write-bitmap-index &&
 481
 482         for argv in \
 483                 "git -c pack.useBitmapBoundaryTraversal=true" \
 484                 "git -c feature.experimental=true" \
 485                 "GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=1 git"
 486         do
 487                 eval "GIT_TRACE2_EVENT=1 $argv rev-list --objects \
 488                         --use-bitmap-index second..other 2>perf" &&
 489                 grep "\"region_enter\".*\"label\":\"haves/boundary\"" perf ||
 490                         return 1
 491         done &&
 492
 493         for argv in \
 494                 "git -c pack.useBitmapBoundaryTraversal=false" \
 495                 "git -c feature.experimental=true -c pack.useBitmapBoundaryTraversal=false" \
 496                 "GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=0 git -c pack.useBitmapBoundaryTraversal=true" \
 497                 "GIT_TEST_PACK_USE_BITMAP_BOUNDARY_TRAVERSAL=0 git -c feature.experimental=true"
 498         do
 499                 eval "GIT_TRACE2_EVENT=1 $argv rev-list --objects \
 500                         --use-bitmap-index second..other 2>perf" &&
 501                 grep "\"region_enter\".*\"label\":\"haves/classic\"" perf ||
 502                         return 1
 503         done
 504 '
 505
 506 test_bitmap_cases "pack.writeBitmapLookupTable"
 507
 508 test_expect_success 'verify writing bitmap lookup table when enabled' '
 509         GIT_TRACE2_EVENT="$(pwd)/trace2" \
 510                 git repack -ad &&
 511         grep "\"label\":\"writing_lookup_table\"" trace2
 512 '
 513
 514 test_expect_success 'truncated bitmap fails gracefully (lookup table)' '
 515         test_config pack.writebitmaphashcache false &&
 516         git repack -adb &&
 517         git rev-list --use-bitmap-index --count --all >expect &&
 518         bitmap=$(ls .git/objects/pack/*.bitmap) &&
 519         test_when_finished "rm -f $bitmap" &&
 520         test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
 521         mv -f $bitmap.tmp $bitmap &&
 522         git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 523         test_cmp expect actual &&
 524         test_grep corrupted.bitmap.index stderr
 525 '
 526
 527 test_done