t/t5333-pseudo-merge-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='pseudo-merge bitmaps'
   4
   5 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
   6
   7 . ./test-lib.sh
   8
   9 test_pseudo_merges () {
  10         test-tool bitmap dump-pseudo-merges
  11 }
  12
  13 test_pseudo_merge_commits () {
  14         test-tool bitmap dump-pseudo-merge-commits "$1"
  15 }
  16
  17 test_pseudo_merges_satisfied () {
  18         test_trace2_data bitmap pseudo_merges_satisfied "$1"
  19 }
  20
  21 test_pseudo_merges_cascades () {
  22         test_trace2_data bitmap pseudo_merges_cascades "$1"
  23 }
  24
  25 test_pseudo_merges_reused () {
  26         test_trace2_data pack-bitmap-write building_bitmaps_pseudo_merge_reused "$1"
  27 }
  28
  29 tag_everything () {
  30         git rev-list --all --no-object-names >in &&
  31         perl -lne '
  32                 print "create refs/tags/" . $. . " " . $1 if /([0-9a-f]+)/
  33         ' <in | git update-ref --stdin
  34 }
  35
  36 test_expect_success 'setup' '
  37         test_commit_bulk 512 &&
  38         tag_everything
  39 '
  40
  41 test_expect_success 'bitmap traversal without pseudo-merges' '
  42         git repack -adb &&
  43
  44         git rev-list --count --all --objects >expect &&
  45
  46         : >trace2.txt &&
  47         GIT_TRACE2_EVENT=$PWD/trace2.txt \
  48                 git rev-list --count --all --objects --use-bitmap-index >actual &&
  49
  50         test_pseudo_merges_satisfied 0 <trace2.txt &&
  51         test_pseudo_merges_cascades 0 <trace2.txt &&
  52         test_pseudo_merges >merges &&
  53         test_must_be_empty merges &&
  54         test_cmp expect actual
  55 '
  56
  57 test_expect_success 'pseudo-merges accurately represent their objects' '
  58         test_config bitmapPseudoMerge.test.pattern "refs/tags/" &&
  59         test_config bitmapPseudoMerge.test.maxMerges 8 &&
  60         test_config bitmapPseudoMerge.test.stableThreshold never &&
  61
  62         git repack -adb &&
  63
  64         test_pseudo_merges >merges &&
  65         test_line_count = 8 merges &&
  66
  67         for i in $(test_seq 0 $(($(wc -l <merges)-1)))
  68         do
  69                 test-tool bitmap dump-pseudo-merge-commits $i >commits &&
  70
  71                 git rev-list --objects --no-object-names --stdin <commits >expect.raw &&
  72                 test-tool bitmap dump-pseudo-merge-objects $i >actual.raw &&
  73
  74                 sort -u <expect.raw >expect &&
  75                 sort -u <actual.raw >actual &&
  76
  77                 test_cmp expect actual || return 1
  78         done
  79 '
  80
  81 test_expect_success 'bitmap traversal with pseudo-merges' '
  82         : >trace2.txt &&
  83         GIT_TRACE2_EVENT=$PWD/trace2.txt \
  84                 git rev-list --count --all --objects --use-bitmap-index >actual &&
  85         git rev-list --count --all --objects >expect &&
  86
  87         test_pseudo_merges_satisfied 8 <trace2.txt &&
  88         test_pseudo_merges_cascades 1 <trace2.txt &&
  89         test_cmp expect actual
  90 '
  91
  92 test_expect_success 'stale bitmap traversal with pseudo-merges' '
  93         test_commit other &&
  94
  95         : >trace2.txt &&
  96         GIT_TRACE2_EVENT=$PWD/trace2.txt \
  97                 git rev-list --count --all --objects --use-bitmap-index >actual &&
  98         git rev-list --count --all --objects >expect &&
  99
 100         test_pseudo_merges_satisfied 8 <trace2.txt &&
 101         test_pseudo_merges_cascades 1 <trace2.txt &&
 102         test_cmp expect actual
 103 '
 104
 105 test_expect_success 'bitmapPseudoMerge.sampleRate adjusts commit selection rate' '
 106         test_config bitmapPseudoMerge.test.pattern "refs/tags/" &&
 107         test_config bitmapPseudoMerge.test.maxMerges 1 &&
 108         test_config bitmapPseudoMerge.test.stableThreshold never &&
 109
 110         commits_nr=$(git rev-list --all --count) &&
 111
 112         for rate in 1.0 0.5 0.25
 113         do
 114                 git -c bitmapPseudoMerge.test.sampleRate=$rate repack -adb &&
 115
 116                 test_pseudo_merges >merges &&
 117                 test_line_count = 1 merges &&
 118                 test_pseudo_merge_commits 0 >commits &&
 119
 120                 test-tool bitmap list-commits >bitmaps &&
 121                 bitmaps_nr="$(wc -l <bitmaps)" &&
 122
 123                 perl -MPOSIX -e "print ceil(\$ARGV[0]*(\$ARGV[1]-\$ARGV[2]))" \
 124                         "$rate" "$commits_nr" "$bitmaps_nr" >expect &&
 125
 126                 test $(cat expect) -eq $(wc -l <commits) || return 1
 127         done
 128 '
 129
 130 test_expect_success 'bitmapPseudoMerge.threshold excludes newer commits' '
 131         git init pseudo-merge-threshold &&
 132         (
 133                 cd pseudo-merge-threshold &&
 134
 135                 new="1672549200" && # 2023-01-01
 136                 old="1641013200" && # 2022-01-01
 137
 138                 GIT_COMMITTER_DATE="$new +0000" &&
 139                 export GIT_COMMITTER_DATE &&
 140                 test_commit_bulk --message="new" --notick 128 &&
 141
 142                 GIT_COMMITTER_DATE="$old +0000" &&
 143                 export GIT_COMMITTER_DATE &&
 144                 test_commit_bulk --message="old" --notick 128 &&
 145
 146                 tag_everything &&
 147
 148                 git \
 149                         -c bitmapPseudoMerge.test.pattern="refs/tags/" \
 150                         -c bitmapPseudoMerge.test.maxMerges=1 \
 151                         -c bitmapPseudoMerge.test.threshold=$(($new - 1)) \
 152                         -c bitmapPseudoMerge.test.stableThreshold=never \
 153                         repack -adb &&
 154
 155                 test_pseudo_merges >merges &&
 156                 test_line_count = 1 merges &&
 157
 158                 test_pseudo_merge_commits 0 >oids &&
 159                 git cat-file --batch <oids >commits &&
 160
 161                 test $(wc -l <oids) = $(grep -c "^committer.*$old +0000$" commits)
 162         )
 163 '
 164
 165 test_expect_success 'bitmapPseudoMerge.stableThreshold creates stable groups' '
 166         (
 167                 cd pseudo-merge-threshold &&
 168
 169                 new="1672549200" && # 2023-01-01
 170                 mid="1654059600" && # 2022-06-01
 171                 old="1641013200" && # 2022-01-01
 172
 173                 GIT_COMMITTER_DATE="$mid +0000" &&
 174                 export GIT_COMMITTER_DATE &&
 175                 test_commit_bulk --message="mid" --notick 128 &&
 176
 177                 git for-each-ref --format="delete %(refname)" refs/tags >in &&
 178                 git update-ref --stdin <in &&
 179
 180                 tag_everything &&
 181
 182                 git \
 183                         -c bitmapPseudoMerge.test.pattern="refs/tags/" \
 184                         -c bitmapPseudoMerge.test.maxMerges=1 \
 185                         -c bitmapPseudoMerge.test.threshold=$(($new - 1)) \
 186                         -c bitmapPseudoMerge.test.stableThreshold=$(($mid - 1)) \
 187                         -c bitmapPseudoMerge.test.stableSize=10 \
 188                         repack -adb &&
 189
 190                 test_pseudo_merges >merges &&
 191                 merges_nr="$(wc -l <merges)" &&
 192
 193                 for i in $(test_seq $(($merges_nr - 1)))
 194                 do
 195                         test_pseudo_merge_commits 0 >oids &&
 196                         git cat-file --batch <oids >commits &&
 197
 198                         expect="$(grep -c "^committer.*$old +0000$" commits)" &&
 199                         actual="$(wc -l <oids)" &&
 200
 201                         test $expect = $actual || return 1
 202                 done &&
 203
 204                 test_pseudo_merge_commits $(($merges_nr - 1)) >oids &&
 205                 git cat-file --batch <oids >commits &&
 206                 test $(wc -l <oids) = $(grep -c "^committer.*$mid +0000$" commits)
 207         )
 208 '
 209
 210 test_expect_success 'out of order thresholds are rejected' '
 211         test_must_fail git \
 212                 -c bitmapPseudoMerge.test.pattern="refs/*" \
 213                 -c bitmapPseudoMerge.test.threshold=1.month.ago \
 214                 -c bitmapPseudoMerge.test.stableThreshold=1.week.ago \
 215                 repack -adb 2>err &&
 216
 217         cat >expect <<-EOF &&
 218         fatal: pseudo-merge group ${SQ}test${SQ} has unstable threshold before stable one
 219         EOF
 220
 221         test_cmp expect err
 222 '
 223
 224 test_expect_success 'pseudo-merge pattern with capture groups' '
 225         git init pseudo-merge-captures &&
 226         (
 227                 cd pseudo-merge-captures &&
 228
 229                 test_commit_bulk 128 &&
 230                 tag_everything &&
 231
 232                 for r in $(test_seq 8)
 233                 do
 234                         test_commit_bulk 16 &&
 235
 236                         git rev-list HEAD~16.. >in &&
 237
 238                         perl -lne "print \"create refs/remotes/$r/tags/\$. \$_\"" <in |
 239                         git update-ref --stdin || return 1
 240                 done &&
 241
 242                 git \
 243                         -c bitmapPseudoMerge.tags.pattern="refs/remotes/([0-9]+)/tags/" \
 244                         -c bitmapPseudoMerge.tags.maxMerges=1 \
 245                         repack -adb &&
 246
 247                 git for-each-ref --format="%(objectname) %(refname)" >refs &&
 248
 249                 test_pseudo_merges >merges &&
 250                 for m in $(test_seq 0 $(($(wc -l <merges) - 1)))
 251                 do
 252                         test_pseudo_merge_commits $m >oids &&
 253                         grep -f oids refs |
 254                         perl -lne "print \$1 if /refs\/remotes\/([0-9]+)/" |
 255                         sort -u || return 1
 256                 done >remotes &&
 257
 258                 test $(wc -l <remotes) -eq $(sort -u <remotes | wc -l)
 259         )
 260 '
 261
 262 test_expect_success 'pseudo-merge overlap setup' '
 263         git init pseudo-merge-overlap &&
 264         (
 265                 cd pseudo-merge-overlap &&
 266
 267                 test_commit_bulk 256 &&
 268                 tag_everything &&
 269
 270                 git \
 271                         -c bitmapPseudoMerge.all.pattern="refs/" \
 272                         -c bitmapPseudoMerge.all.maxMerges=1 \
 273                         -c bitmapPseudoMerge.all.stableThreshold=never \
 274                         -c bitmapPseudoMerge.tags.pattern="refs/tags/" \
 275                         -c bitmapPseudoMerge.tags.maxMerges=1 \
 276                         -c bitmapPseudoMerge.tags.stableThreshold=never \
 277                         repack -adb
 278         )
 279 '
 280
 281 test_expect_success 'pseudo-merge overlap generates overlapping groups' '
 282         (
 283                 cd pseudo-merge-overlap &&
 284
 285                 test_pseudo_merges >merges &&
 286                 test_line_count = 2 merges &&
 287
 288                 test_pseudo_merge_commits 0 >commits-0.raw &&
 289                 test_pseudo_merge_commits 1 >commits-1.raw &&
 290
 291                 sort commits-0.raw >commits-0 &&
 292                 sort commits-1.raw >commits-1 &&
 293
 294                 comm -12 commits-0 commits-1 >overlap &&
 295
 296                 test_line_count -gt 0 overlap
 297         )
 298 '
 299
 300 test_expect_success 'pseudo-merge overlap traversal' '
 301         (
 302                 cd pseudo-merge-overlap &&
 303
 304                 : >trace2.txt &&
 305                 GIT_TRACE2_EVENT=$PWD/trace2.txt \
 306                         git rev-list --count --all --objects --use-bitmap-index >actual &&
 307                 git rev-list --count --all --objects >expect &&
 308
 309                 test_pseudo_merges_satisfied 2 <trace2.txt &&
 310                 test_pseudo_merges_cascades 1 <trace2.txt &&
 311                 test_cmp expect actual
 312         )
 313 '
 314
 315 test_expect_success 'pseudo-merge overlap stale traversal' '
 316         (
 317                 cd pseudo-merge-overlap &&
 318
 319                 test_commit other &&
 320
 321                 : >trace2.txt &&
 322                 GIT_TRACE2_EVENT=$PWD/trace2.txt \
 323                         git rev-list --count --all --objects --use-bitmap-index >actual &&
 324                 git rev-list --count --all --objects >expect &&
 325
 326                 test_pseudo_merges_satisfied 2 <trace2.txt &&
 327                 test_pseudo_merges_cascades 1 <trace2.txt &&
 328                 test_cmp expect actual
 329         )
 330 '
 331
 332 test_expect_success 'pseudo-merge reuse' '
 333         git init pseudo-merge-reuse &&
 334         (
 335                 cd pseudo-merge-reuse &&
 336
 337                 stable="1641013200" && # 2022-01-01
 338                 unstable="1672549200" && # 2023-01-01
 339
 340                 GIT_COMMITTER_DATE="$stable +0000" &&
 341                 export GIT_COMMITTER_DATE &&
 342                 test_commit_bulk --notick 128 &&
 343                 GIT_COMMITTER_DATE="$unstable +0000" &&
 344                 export GIT_COMMITTER_DATE &&
 345                 test_commit_bulk --notick 128 &&
 346
 347                 tag_everything &&
 348
 349                 git \
 350                         -c bitmapPseudoMerge.test.pattern="refs/tags/" \
 351                         -c bitmapPseudoMerge.test.maxMerges=1 \
 352                         -c bitmapPseudoMerge.test.threshold=now \
 353                         -c bitmapPseudoMerge.test.stableThreshold=$(($unstable - 1)) \
 354                         -c bitmapPseudoMerge.test.stableSize=512 \
 355                         repack -adb &&
 356
 357                 test_pseudo_merges >merges &&
 358                 test_line_count = 2 merges &&
 359
 360                 test_pseudo_merge_commits 0 >stable-oids.before &&
 361                 test_pseudo_merge_commits 1 >unstable-oids.before &&
 362
 363                 : >trace2.txt &&
 364                 GIT_TRACE2_EVENT=$PWD/trace2.txt git \
 365                         -c bitmapPseudoMerge.test.pattern="refs/tags/" \
 366                         -c bitmapPseudoMerge.test.maxMerges=2 \
 367                         -c bitmapPseudoMerge.test.threshold=now \
 368                         -c bitmapPseudoMerge.test.stableThreshold=$(($unstable - 1)) \
 369                         -c bitmapPseudoMerge.test.stableSize=512 \
 370                         repack -adb &&
 371
 372                 test_pseudo_merges_reused 1 <trace2.txt &&
 373
 374                 test_pseudo_merges >merges &&
 375                 test_line_count = 3 merges &&
 376
 377                 test_pseudo_merge_commits 0 >stable-oids.after &&
 378                 for i in 1 2
 379                 do
 380                         test_pseudo_merge_commits $i || return 1
 381                 done >unstable-oids.after &&
 382
 383                 sort -u <stable-oids.before >expect &&
 384                 sort -u <stable-oids.after >actual &&
 385                 test_cmp expect actual &&
 386
 387                 sort -u <unstable-oids.before >expect &&
 388                 sort -u <unstable-oids.after >actual &&
 389                 test_cmp expect actual
 390         )
 391 '
 392
 393 test_expect_success 'empty pseudo-merge group' '
 394         git init pseudo-merge-empty-group &&
 395         (
 396                 cd pseudo-merge-empty-group &&
 397
 398                 # Ensure that a pseudo-merge group with no unstable
 399                 # commits does not generate an empty pseudo-merge
 400                 # bitmap.
 401                 git config bitmapPseudoMerge.empty.pattern refs/ &&
 402
 403                 test_commit base &&
 404                 git repack -adb &&
 405
 406                 test-tool bitmap dump-pseudo-merges >merges &&
 407                 test_line_count = 1 merges &&
 408
 409                 test 0 -eq "$(grep -c commits=0 <merges)"
 410         )
 411 '
 412
 413 test_expect_success 'pseudo-merge closure' '
 414         git init pseudo-merge-closure &&
 415         (
 416                 cd pseudo-merge-closure &&
 417
 418                 test_commit A &&
 419                 git repack -d &&
 420
 421                 test_commit B &&
 422
 423                 # Note that the contents of A is packed, but B is not. A
 424                 # (and the objects reachable from it) are thus visible
 425                 # to the MIDX, but the same is not true for B and its
 426                 # objects.
 427                 #
 428                 # Ensure that we do not attempt to create a pseudo-merge
 429                 # for B, depsite it matching the below pseudo-merge
 430                 # group pattern, as doing so would result in a failure
 431                 # to write a non-closed bitmap.
 432                 git config bitmapPseudoMerge.test.pattern refs/ &&
 433                 git config bitmapPseudoMerge.test.threshold now &&
 434
 435                 git multi-pack-index write --bitmap &&
 436
 437                 test-tool bitmap dump-pseudo-merges >pseudo-merges &&
 438                 test_line_count = 1 pseudo-merges &&
 439
 440                 git rev-parse A >expect &&
 441
 442                 test-tool bitmap list-commits >actual &&
 443                 test_cmp expect actual &&
 444                 test-tool bitmap dump-pseudo-merge-commits 0 >actual &&
 445                 test_cmp expect actual
 446         )
 447 '
 448
 449 test_done