t/t6421-merge-partial-clone.sh

   1 #!/bin/sh
   2
   3 test_description="limiting blob downloads when merging with partial clones"
   4 # Uses a methodology similar to
   5 #   t6042: corner cases with renames but not criss-cross merges
   6 #   t6036: corner cases with both renames and criss-cross merges
   7 #   t6423: directory rename detection
   8 #
   9 # The setup for all of them, pictorially, is:
  10 #
  11 #      A
  12 #      o
  13 #     / \
  14 #  O o   ?
  15 #     \ /
  16 #      o
  17 #      B
  18 #
  19 # To help make it easier to follow the flow of tests, they have been
  20 # divided into sections and each test will start with a quick explanation
  21 # of what commits O, A, and B contain.
  22 #
  23 # Notation:
  24 #    z/{b,c}   means  files z/b and z/c both exist
  25 #    x/d_1     means  file x/d exists with content d1.  (Purpose of the
  26 #                     underscore notation is to differentiate different
  27 #                     files that might be renamed into each other's paths.)
  28
  29 TEST_PASSES_SANITIZE_LEAK=true
  30 . ./test-lib.sh
  31 . "$TEST_DIRECTORY"/lib-merge.sh
  32
  33 test_setup_repo () {
  34         test -d server && return
  35         git init server &&
  36         (
  37                 cd server &&
  38
  39                 git config uploadpack.allowfilter 1 &&
  40                 git config uploadpack.allowanysha1inwant 1 &&
  41
  42                 mkdir -p general &&
  43                 test_seq 2 9 >general/leap1 &&
  44                 cp general/leap1 general/leap2 &&
  45                 echo leap2 >>general/leap2 &&
  46
  47                 mkdir -p basename &&
  48                 cp general/leap1 basename/numbers &&
  49                 cp general/leap1 basename/sequence &&
  50                 cp general/leap1 basename/values &&
  51                 echo numbers >>basename/numbers &&
  52                 echo sequence >>basename/sequence &&
  53                 echo values >>basename/values &&
  54
  55                 mkdir -p dir/unchanged &&
  56                 mkdir -p dir/subdir/tweaked &&
  57                 echo a >dir/subdir/a &&
  58                 echo b >dir/subdir/b &&
  59                 echo c >dir/subdir/c &&
  60                 echo d >dir/subdir/d &&
  61                 echo e >dir/subdir/e &&
  62                 cp general/leap1 dir/subdir/Makefile &&
  63                 echo toplevel makefile >>dir/subdir/Makefile &&
  64                 echo f >dir/subdir/tweaked/f &&
  65                 echo g >dir/subdir/tweaked/g &&
  66                 echo h >dir/subdir/tweaked/h &&
  67                 echo subdirectory makefile >dir/subdir/tweaked/Makefile &&
  68                 for i in $(test_seq 1 88)
  69                 do
  70                         echo content $i >dir/unchanged/file_$i
  71                 done &&
  72                 git add . &&
  73                 git commit -m "O" &&
  74
  75                 git branch O &&
  76                 git branch A &&
  77                 git branch B-single &&
  78                 git branch B-dir &&
  79                 git branch B-many &&
  80
  81                 git switch A &&
  82
  83                 git rm general/leap* &&
  84                 mkdir general/ &&
  85                 test_seq 1 9 >general/jump1 &&
  86                 cp general/jump1 general/jump2 &&
  87                 echo leap2 >>general/jump2 &&
  88
  89                 rm basename/numbers basename/sequence basename/values &&
  90                 mkdir -p basename/subdir/
  91                 cp general/jump1 basename/subdir/numbers &&
  92                 cp general/jump1 basename/subdir/sequence &&
  93                 cp general/jump1 basename/subdir/values &&
  94                 echo numbers >>basename/subdir/numbers &&
  95                 echo sequence >>basename/subdir/sequence &&
  96                 echo values >>basename/subdir/values &&
  97
  98                 git rm dir/subdir/tweaked/f &&
  99                 echo more >>dir/subdir/e &&
 100                 echo more >>dir/subdir/Makefile &&
 101                 echo more >>dir/subdir/tweaked/Makefile &&
 102                 mkdir dir/subdir/newsubdir &&
 103                 echo rust code >dir/subdir/newsubdir/newfile.rs &&
 104                 git mv dir/subdir/e dir/subdir/newsubdir/ &&
 105                 git mv dir folder &&
 106                 git add . &&
 107                 git commit -m "A" &&
 108
 109                 git switch B-single &&
 110                 echo new first line >dir/subdir/Makefile &&
 111                 cat general/leap1 >>dir/subdir/Makefile &&
 112                 echo toplevel makefile >>dir/subdir/Makefile &&
 113                 echo perl code >general/newfile.pl &&
 114                 git add . &&
 115                 git commit -m "B-single" &&
 116
 117                 git switch B-dir &&
 118                 echo java code >dir/subdir/newfile.java &&
 119                 echo scala code >dir/subdir/newfile.scala &&
 120                 echo groovy code >dir/subdir/newfile.groovy &&
 121                 git add . &&
 122                 git commit -m "B-dir" &&
 123
 124                 git switch B-many &&
 125                 test_seq 2 10 >general/leap1 &&
 126                 rm general/leap2 &&
 127                 cp general/leap1 general/leap2 &&
 128                 echo leap2 >>general/leap2 &&
 129
 130                 rm basename/numbers basename/sequence basename/values &&
 131                 mkdir -p basename/subdir/
 132                 cp general/leap1 basename/subdir/numbers &&
 133                 cp general/leap1 basename/subdir/sequence &&
 134                 cp general/leap1 basename/subdir/values &&
 135                 echo numbers >>basename/subdir/numbers &&
 136                 echo sequence >>basename/subdir/sequence &&
 137                 echo values >>basename/subdir/values &&
 138
 139                 mkdir dir/subdir/newsubdir/ &&
 140                 echo c code >dir/subdir/newfile.c &&
 141                 echo python code >dir/subdir/newsubdir/newfile.py &&
 142                 git add . &&
 143                 git commit -m "B-many" &&
 144
 145                 git switch A
 146         )
 147 }
 148
 149 # Testcase: Objects downloaded for single relevant rename
 150 #   Commit O:
 151 #              general/{leap1_O, leap2_O}
 152 #              basename/{numbers_O, sequence_O, values_O}
 153 #              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
 154 #              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
 155 #              dir/unchanged/<LOTS OF FILES>
 156 #   Commit A:
 157 #     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
 158 #      -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
 159 #      both Makefiles and jumps)
 160 #              general/{jump1_A, jump2_A}
 161 #              basename/subdir/{numbers_A, sequence_A, values_A}
 162 #              folder/subdir/{a,b,c,d,Makefile_TOP_A}
 163 #              folder/subdir/newsubdir/{e_A,newfile.rs}
 164 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
 165 #              folder/unchanged/<LOTS OF FILES>
 166 #   Commit B(-single):
 167 #     (add newfile.pl, tweak Makefile_TOP)
 168 #              general/{leap1_O, leap2_O,newfile.pl}
 169 #              basename/{numbers_O, sequence_O, values_O}
 170 #              dir/{a,b,c,d,e_O,Makefile_TOP_B}
 171 #              dir/tweaked/{f,g,h,Makefile_SUB_O}
 172 #              dir/unchanged/<LOTS OF FILES>
 173 #   Expected:
 174 #              general/{jump1_A, jump2_A,newfile.pl}
 175 #              basename/subdir/{numbers_A, sequence_A, values_A}
 176 #              folder/subdir/{a,b,c,d,Makefile_TOP_Merged}
 177 #              folder/subdir/newsubdir/{e_A,newfile.rs}
 178 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
 179 #              folder/unchanged/<LOTS OF FILES>
 180 #
 181 # Objects that need to be fetched:
 182 #   Rename detection:
 183 #     Side1 (O->A):
 184 #       Basename-matches rename detection only needs to fetch these objects:
 185 #         Makefile_TOP_O, Makefile_TOP_A
 186 #         (Despite many renames, all others are content irrelevant.  They
 187 #          are also location irrelevant because newfile.rs was added on
 188 #          the side doing the directory rename, and newfile.pl was added to
 189 #          a directory that was not renamed on either side.)
 190 #       General rename detection only needs to fetch these objects:
 191 #         <None>
 192 #          (Even though newfile.rs, jump[12], basename/subdir/*, and e
 193 #          could all be used as destinations in rename detection, the
 194 #          basename detection for Makefile matches up all relevant
 195 #          sources, so these other files never end up needing to be
 196 #          used)
 197 #     Side2 (O->B):
 198 #       Basename-matches rename detection only needs to fetch these objects:
 199 #         <None>
 200 #         (there are no deleted files, so no possible sources)
 201 #       General rename detection only needs to fetch these objects:
 202 #         <None>
 203 #         (there are no deleted files, so no possible sources)
 204 #   Merge:
 205 #     3-way content merge needs to grab these objects:
 206 #       Makefile_TOP_B
 207 #   Nothing else needs to fetch objects
 208 #
 209 #   Summary: 2 fetches (1 for 2 objects, 1 for 1 object)
 210 #
 211 test_expect_merge_algorithm failure success 'Objects downloaded for single relevant rename' '
 212         test_setup_repo &&
 213         git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-single &&
 214         (
 215                 cd objects-single &&
 216
 217                 git rev-list --objects --all --missing=print |
 218                         grep "^?" | sort >missing-objects-before &&
 219
 220                 git checkout -q origin/A &&
 221
 222                 GIT_TRACE2_PERF="$(pwd)/trace.output" git \
 223                         -c merge.directoryRenames=true merge --no-stat \
 224                         --no-progress origin/B-single &&
 225
 226                 # Check the number of objects we reported we would fetch
 227                 cat >expect <<-EOF &&
 228                 fetch_count:2
 229                 fetch_count:1
 230                 EOF
 231                 grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
 232                 test_cmp expect actual &&
 233
 234                 # Check the number of fetch commands exec-ed by filtering trace to
 235                 # child_start events by the top-level program (2nd field == d0)
 236                 grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches &&
 237                 test_line_count = 2 fetches &&
 238
 239                 git rev-list --objects --all --missing=print |
 240                         grep "^?" | sort >missing-objects-after &&
 241                 comm -2 -3 missing-objects-before missing-objects-after >old &&
 242                 comm -1 -3 missing-objects-before missing-objects-after >new &&
 243                 # No new missing objects
 244                 test_must_be_empty new &&
 245                 # Fetched 2 + 1 = 3 objects
 246                 test_line_count = 3 old
 247         )
 248 '
 249
 250 # Testcase: Objects downloaded for directory rename
 251 #   Commit O:
 252 #              general/{leap1_O, leap2_O}
 253 #              basename/{numbers_O, sequence_O, values_O}
 254 #              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
 255 #              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
 256 #              dir/unchanged/<LOTS OF FILES>
 257 #   Commit A:
 258 #     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ ->
 259 #      folder/, move e into newsubdir, add newfile.rs, remove f, modify
 260 #      both Makefiles and jumps)
 261 #              general/{jump1_A, jump2_A}
 262 #              basename/subdir/{numbers_A, sequence_A, values_A}
 263 #              folder/subdir/{a,b,c,d,Makefile_TOP_A}
 264 #              folder/subdir/newsubdir/{e_A,newfile.rs}
 265 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
 266 #              folder/unchanged/<LOTS OF FILES>
 267 #   Commit B(-dir):
 268 #     (add dir/subdir/newfile.{java,scala,groovy}
 269 #              general/{leap1_O, leap2_O}
 270 #              basename/{numbers_O, sequence_O, values_O}
 271 #              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O,
 272 #                          newfile.java,newfile.scala,newfile.groovy}
 273 #              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
 274 #              dir/unchanged/<LOTS OF FILES>
 275 #   Expected:
 276 #              general/{jump1_A, jump2_A}
 277 #              basename/subdir/{numbers_A, sequence_A, values_A}
 278 #              folder/subdir/{a,b,c,d,Makefile_TOP_A,
 279 #                             newfile.java,newfile.scala,newfile.groovy}
 280 #              folder/subdir/newsubdir/{e_A,newfile.rs}
 281 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
 282 #              folder/unchanged/<LOTS OF FILES>
 283 #
 284 # Objects that need to be fetched:
 285 #   Makefile_TOP_O, Makefile_TOP_A
 286 #   Makefile_SUB_O, Makefile_SUB_A
 287 #   e_O, e_A
 288 #   * Despite A's rename of jump->leap, those renames are irrelevant.
 289 #   * Despite A's rename of basename/ -> basename/subdir/, those renames are
 290 #     irrelevant.
 291 #   * Because of A's rename of dir/ -> folder/ and B-dir's addition of
 292 #     newfile.* into dir/subdir/, we need to determine directory renames.
 293 #     (Technically, there are enough exact renames to determine directory
 294 #      rename detection, but the current implementation always does
 295 #      basename searching before directory rename detection.  Running it
 296 #      also before basename searching would mean doing directory rename
 297 #      detection twice, but it's a bit expensive to do that and cases like
 298 #      this are not all that common.)
 299 #   Summary: 1 fetches for 6 objects
 300 #
 301 test_expect_merge_algorithm failure success 'Objects downloaded when a directory rename triggered' '
 302         test_setup_repo &&
 303         git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-dir &&
 304         (
 305                 cd objects-dir &&
 306
 307                 git rev-list --objects --all --missing=print |
 308                         grep "^?" | sort >missing-objects-before &&
 309
 310                 git checkout -q origin/A &&
 311
 312                 GIT_TRACE2_PERF="$(pwd)/trace.output" git \
 313                         -c merge.directoryRenames=true merge --no-stat \
 314                         --no-progress origin/B-dir &&
 315
 316                 # Check the number of objects we reported we would fetch
 317                 cat >expect <<-EOF &&
 318                 fetch_count:6
 319                 EOF
 320                 grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
 321                 test_cmp expect actual &&
 322
 323                 # Check the number of fetch commands exec-ed by filtering trace to
 324                 # child_start events by the top-level program (2nd field == d0)
 325                 grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches &&
 326                 test_line_count = 1 fetches &&
 327
 328                 git rev-list --objects --all --missing=print |
 329                         grep "^?" | sort >missing-objects-after &&
 330                 comm -2 -3 missing-objects-before missing-objects-after >old &&
 331                 comm -1 -3 missing-objects-before missing-objects-after >new &&
 332                 # No new missing objects
 333                 test_must_be_empty new &&
 334                 # Fetched 6 objects
 335                 test_line_count = 6 old
 336         )
 337 '
 338
 339 # Testcase: Objects downloaded with lots of renames and modifications
 340 #   Commit O:
 341 #              general/{leap1_O, leap2_O}
 342 #              basename/{numbers_O, sequence_O, values_O}
 343 #              dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
 344 #              dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
 345 #              dir/unchanged/<LOTS OF FILES>
 346 #   Commit A:
 347 #     (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
 348 #      -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
 349 #      both Makefiles and jumps)
 350 #              general/{jump1_A, jump2_A}
 351 #              basename/subdir/{numbers_A, sequence_A, values_A}
 352 #              folder/subdir/{a,b,c,d,Makefile_TOP_A}
 353 #              folder/subdir/newsubdir/{e_A,newfile.rs}
 354 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A}
 355 #              folder/unchanged/<LOTS OF FILES>
 356 #   Commit B(-minimal):
 357 #     (modify both leaps, rename basename/ -> basename/subdir/, add
 358 #      newfile.{c,py})
 359 #              general/{leap1_B, leap2_B}
 360 #              basename/subdir/{numbers_B, sequence_B, values_B}
 361 #              dir/{a,b,c,d,e_O,Makefile_TOP_O,newfile.c}
 362 #              dir/tweaked/{f,g,h,Makefile_SUB_O,newfile.py}
 363 #              dir/unchanged/<LOTS OF FILES>
 364 #   Expected:
 365 #              general/{jump1_Merged, jump2_Merged}
 366 #              basename/subdir/{numbers_Merged, sequence_Merged, values_Merged}
 367 #              folder/subdir/{a,b,c,d,Makefile_TOP_A,newfile.c}
 368 #              folder/subdir/newsubdir/e_A
 369 #              folder/subdir/tweaked/{g,h,Makefile_SUB_A,newfile.py}
 370 #              folder/unchanged/<LOTS OF FILES>
 371 #
 372 # Objects that need to be fetched:
 373 #   Rename detection:
 374 #     Side1 (O->A):
 375 #       Basename-matches rename detection only needs to fetch these objects:
 376 #         numbers_O, numbers_A
 377 #         sequence_O, sequence_A
 378 #         values_O, values_A
 379 #         Makefile_TOP_O, Makefile_TOP_A
 380 #         Makefile_SUB_O, Makefile_SUB_A
 381 #         e_O, e_A
 382 #       General rename detection only needs to fetch these objects:
 383 #         leap1_O, leap2_O
 384 #         jump1_A, jump2_A, newfile.rs
 385 #         (only need remaining relevant sources, but any relevant sources need
 386 #          to be matched against all possible unpaired destinations)
 387 #     Side2 (O->B):
 388 #       Basename-matches rename detection only needs to fetch these objects:
 389 #         numbers_B
 390 #         sequence_B
 391 #         values_B
 392 #       (because numbers_O, sequence_O, and values_O already fetched above)
 393 #       General rename detection only needs to fetch these objects:
 394 #         <None>
 395 #   Merge:
 396 #     3-way content merge needs to grab these objects:
 397 #       leap1_B
 398 #       leap2_B
 399 #   Nothing else needs to fetch objects
 400 #
 401 #   Summary: 4 fetches (1 for 6 objects, 1 for 8, 1 for 3, 1 for 2)
 402 #
 403 test_expect_merge_algorithm failure success 'Objects downloaded with lots of renames and modifications' '
 404         test_setup_repo &&
 405         git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-many &&
 406         (
 407                 cd objects-many &&
 408
 409                 git rev-list --objects --all --missing=print |
 410                         grep "^?" | sort >missing-objects-before &&
 411
 412                 git checkout -q origin/A &&
 413
 414                 GIT_TRACE2_PERF="$(pwd)/trace.output" git \
 415                         -c merge.directoryRenames=true merge --no-stat \
 416                         --no-progress origin/B-many &&
 417
 418                 # Check the number of objects we reported we would fetch
 419                 cat >expect <<-EOF &&
 420                 fetch_count:12
 421                 fetch_count:5
 422                 fetch_count:3
 423                 fetch_count:2
 424                 EOF
 425                 grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
 426                 test_cmp expect actual &&
 427
 428                 # Check the number of fetch commands exec-ed by filtering trace to
 429                 # child_start events by the top-level program (2nd field == d0)
 430                 grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches &&
 431                 test_line_count = 4 fetches &&
 432
 433                 git rev-list --objects --all --missing=print |
 434                         grep "^?" | sort >missing-objects-after &&
 435                 comm -2 -3 missing-objects-before missing-objects-after >old &&
 436                 comm -1 -3 missing-objects-before missing-objects-after >new &&
 437                 # No new missing objects
 438                 test_must_be_empty new &&
 439                 # Fetched 12 + 5 + 3 + 2 = 22 objects
 440                 test_line_count = 22 old
 441         )
 442 '
 443
 444 test_done