bundle-uri.c

   1 #define USE_THE_REPOSITORY_VARIABLE
   2
   3 #include "git-compat-util.h"
   4 #include "bundle-uri.h"
   5 #include "bundle.h"
   6 #include "copy.h"
   7 #include "environment.h"
   8 #include "gettext.h"
   9 #include "refs.h"
  10 #include "run-command.h"
  11 #include "hashmap.h"
  12 #include "pkt-line.h"
  13 #include "config.h"
  14 #include "fetch-pack.h"
  15 #include "remote.h"
  16
  17 static struct {
  18         enum bundle_list_heuristic heuristic;
  19         const char *name;
  20 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  21         { BUNDLE_HEURISTIC_NONE, ""},
  22         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  23 };
  24
  25 static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED,
  26                            const struct hashmap_entry *he1,
  27                            const struct hashmap_entry *he2,
  28                            const void *id)
  29 {
  30         const struct remote_bundle_info *e1 =
  31                 container_of(he1, const struct remote_bundle_info, ent);
  32         const struct remote_bundle_info *e2 =
  33                 container_of(he2, const struct remote_bundle_info, ent);
  34
  35         return strcmp(e1->id, id ? (const char *)id : e2->id);
  36 }
  37
  38 void init_bundle_list(struct bundle_list *list)
  39 {
  40         memset(list, 0, sizeof(*list));
  41
  42         /* Implied defaults. */
  43         list->mode = BUNDLE_MODE_ALL;
  44         list->version = 1;
  45
  46         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  47 }
  48
  49 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  50                                     void *data UNUSED)
  51 {
  52         FREE_AND_NULL(bundle->id);
  53         FREE_AND_NULL(bundle->uri);
  54         FREE_AND_NULL(bundle->file);
  55         bundle->unbundled = 0;
  56         return 0;
  57 }
  58
  59 void clear_bundle_list(struct bundle_list *list)
  60 {
  61         if (!list)
  62                 return;
  63
  64         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  65         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  66         free(list->baseURI);
  67 }
  68
  69 int for_all_bundles_in_list(struct bundle_list *list,
  70                             bundle_iterator iter,
  71                             void *data)
  72 {
  73         struct remote_bundle_info *info;
  74         struct hashmap_iter i;
  75
  76         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  77                 int result = iter(info, data);
  78
  79                 if (result)
  80                         return result;
  81         }
  82
  83         return 0;
  84 }
  85
  86 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  87 {
  88         FILE *fp = data;
  89         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  90         fprintf(fp, "\turi = %s\n", info->uri);
  91
  92         if (info->creationToken)
  93                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  94         return 0;
  95 }
  96
  97 void print_bundle_list(FILE *fp, struct bundle_list *list)
  98 {
  99         const char *mode;
 100
 101         switch (list->mode) {
 102         case BUNDLE_MODE_ALL:
 103                 mode = "all";
 104                 break;
 105
 106         case BUNDLE_MODE_ANY:
 107                 mode = "any";
 108                 break;
 109
 110         case BUNDLE_MODE_NONE:
 111         default:
 112                 mode = "<unknown>";
 113         }
 114
 115         fprintf(fp, "[bundle]\n");
 116         fprintf(fp, "\tversion = %d\n", list->version);
 117         fprintf(fp, "\tmode = %s\n", mode);
 118
 119         if (list->heuristic) {
 120                 int i;
 121                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 122                         if (heuristics[i].heuristic == list->heuristic) {
 123                                 printf("\theuristic = %s\n",
 124                                        heuristics[list->heuristic].name);
 125                                 break;
 126                         }
 127                 }
 128         }
 129
 130         for_all_bundles_in_list(list, summarize_bundle, fp);
 131 }
 132
 133 /**
 134  * Given a key-value pair, update the state of the given bundle list.
 135  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 136  * is not understood or the value is malformed.
 137  */
 138 static int bundle_list_update(const char *key, const char *value,
 139                               struct bundle_list *list)
 140 {
 141         struct strbuf id = STRBUF_INIT;
 142         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 143         struct remote_bundle_info *bundle;
 144         const char *subsection, *subkey;
 145         size_t subsection_len;
 146
 147         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 148                 return -1;
 149
 150         if (!subsection_len) {
 151                 if (!strcmp(subkey, "version")) {
 152                         int version;
 153                         if (!git_parse_int(value, &version))
 154                                 return -1;
 155                         if (version != 1)
 156                                 return -1;
 157
 158                         list->version = version;
 159                         return 0;
 160                 }
 161
 162                 if (!strcmp(subkey, "mode")) {
 163                         if (!strcmp(value, "all"))
 164                                 list->mode = BUNDLE_MODE_ALL;
 165                         else if (!strcmp(value, "any"))
 166                                 list->mode = BUNDLE_MODE_ANY;
 167                         else
 168                                 return -1;
 169                         return 0;
 170                 }
 171
 172                 if (!strcmp(subkey, "heuristic")) {
 173                         int i;
 174                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 175                                 if (heuristics[i].heuristic &&
 176                                     heuristics[i].name &&
 177                                     !strcmp(value, heuristics[i].name)) {
 178                                         list->heuristic = heuristics[i].heuristic;
 179                                         return 0;
 180                                 }
 181                         }
 182
 183                         /* Ignore unknown heuristics. */
 184                         return 0;
 185                 }
 186
 187                 /* Ignore other unknown global keys. */
 188                 return 0;
 189         }
 190
 191         strbuf_add(&id, subsection, subsection_len);
 192
 193         /*
 194          * Check for an existing bundle with this <id>, or create one
 195          * if necessary.
 196          */
 197         lookup.id = id.buf;
 198         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 199         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 200                 CALLOC_ARRAY(bundle, 1);
 201                 bundle->id = strbuf_detach(&id, NULL);
 202                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 203                 hashmap_add(&list->bundles, &bundle->ent);
 204         }
 205         strbuf_release(&id);
 206
 207         if (!strcmp(subkey, "uri")) {
 208                 if (bundle->uri)
 209                         return -1;
 210                 bundle->uri = relative_url(list->baseURI, value, NULL);
 211                 return 0;
 212         }
 213
 214         if (!strcmp(subkey, "creationtoken")) {
 215                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 216                         warning(_("could not parse bundle list key %s with value '%s'"),
 217                                 "creationToken", value);
 218                 return 0;
 219         }
 220
 221         /*
 222          * At this point, we ignore any information that we don't
 223          * understand, assuming it to be hints for a heuristic the client
 224          * does not currently understand.
 225          */
 226         return 0;
 227 }
 228
 229 static int config_to_bundle_list(const char *key, const char *value,
 230                                  const struct config_context *ctx UNUSED,
 231                                  void *data)
 232 {
 233         struct bundle_list *list = data;
 234         return bundle_list_update(key, value, list);
 235 }
 236
 237 int bundle_uri_parse_config_format(const char *uri,
 238                                    const char *filename,
 239                                    struct bundle_list *list)
 240 {
 241         int result;
 242         struct config_options opts = {
 243                 .error_action = CONFIG_ERROR_ERROR,
 244         };
 245
 246         if (!list->baseURI) {
 247                 struct strbuf baseURI = STRBUF_INIT;
 248                 strbuf_addstr(&baseURI, uri);
 249
 250                 /*
 251                  * If the URI does not end with a trailing slash, then
 252                  * remove the filename portion of the path. This is
 253                  * important for relative URIs.
 254                  */
 255                 strbuf_strip_file_from_path(&baseURI);
 256                 list->baseURI = strbuf_detach(&baseURI, NULL);
 257         }
 258         result = git_config_from_file_with_options(config_to_bundle_list,
 259                                                    filename, list,
 260                                                    CONFIG_SCOPE_UNKNOWN,
 261                                                    &opts);
 262
 263         if (!result && list->mode == BUNDLE_MODE_NONE) {
 264                 warning(_("bundle list at '%s' has no mode"), uri);
 265                 result = 1;
 266         }
 267
 268         return result;
 269 }
 270
 271 static char *find_temp_filename(void)
 272 {
 273         int fd;
 274         struct strbuf name = STRBUF_INIT;
 275         /*
 276          * Find a temporary filename that is available. This is briefly
 277          * racy, but unlikely to collide.
 278          */
 279         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 280         if (fd < 0) {
 281                 warning(_("failed to create temporary file"));
 282                 return NULL;
 283         }
 284
 285         close(fd);
 286         unlink(name.buf);
 287         return strbuf_detach(&name, NULL);
 288 }
 289
 290 static int download_https_uri_to_file(const char *file, const char *uri)
 291 {
 292         int result = 0;
 293         struct child_process cp = CHILD_PROCESS_INIT;
 294         FILE *child_in = NULL, *child_out = NULL;
 295         struct strbuf line = STRBUF_INIT;
 296         int found_get = 0;
 297
 298         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 299         cp.err = -1;
 300         cp.in = -1;
 301         cp.out = -1;
 302
 303         if (start_command(&cp))
 304                 return 1;
 305
 306         child_in = fdopen(cp.in, "w");
 307         if (!child_in) {
 308                 result = 1;
 309                 goto cleanup;
 310         }
 311
 312         child_out = fdopen(cp.out, "r");
 313         if (!child_out) {
 314                 result = 1;
 315                 goto cleanup;
 316         }
 317
 318         fprintf(child_in, "capabilities\n");
 319         fflush(child_in);
 320
 321         while (!strbuf_getline(&line, child_out)) {
 322                 if (!line.len)
 323                         break;
 324                 if (!strcmp(line.buf, "get"))
 325                         found_get = 1;
 326         }
 327         strbuf_release(&line);
 328
 329         if (!found_get) {
 330                 result = error(_("insufficient capabilities"));
 331                 goto cleanup;
 332         }
 333
 334         fprintf(child_in, "get %s %s\n\n", uri, file);
 335
 336 cleanup:
 337         if (child_in)
 338                 fclose(child_in);
 339         if (finish_command(&cp))
 340                 return 1;
 341         if (child_out)
 342                 fclose(child_out);
 343         return result;
 344 }
 345
 346 static int copy_uri_to_file(const char *filename, const char *uri)
 347 {
 348         const char *out;
 349
 350         if (starts_with(uri, "https:") ||
 351             starts_with(uri, "http:"))
 352                 return download_https_uri_to_file(filename, uri);
 353
 354         if (skip_prefix(uri, "file://", &out))
 355                 uri = out;
 356
 357         /* Copy as a file */
 358         return copy_file(filename, uri, 0);
 359 }
 360
 361 static int unbundle_from_file(struct repository *r, const char *file)
 362 {
 363         int result = 0;
 364         int bundle_fd;
 365         struct bundle_header header = BUNDLE_HEADER_INIT;
 366         struct string_list_item *refname;
 367         struct strbuf bundle_ref = STRBUF_INIT;
 368         size_t bundle_prefix_len;
 369
 370         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 371                 return 1;
 372
 373         /*
 374          * Skip the reachability walk here, since we will be adding
 375          * a reachable ref pointing to the new tips, which will reach
 376          * the prerequisite commits.
 377          */
 378         if ((result = unbundle(r, &header, bundle_fd, NULL,
 379                                VERIFY_BUNDLE_QUIET | (fetch_pack_fsck_objects() ? VERIFY_BUNDLE_FSCK : 0))))
 380                 return 1;
 381
 382         /*
 383          * Convert all refs/heads/ from the bundle into refs/bundles/
 384          * in the local repository.
 385          */
 386         strbuf_addstr(&bundle_ref, "refs/bundles/");
 387         bundle_prefix_len = bundle_ref.len;
 388
 389         for_each_string_list_item(refname, &header.references) {
 390                 struct object_id *oid = refname->util;
 391                 struct object_id old_oid;
 392                 const char *branch_name;
 393                 int has_old;
 394
 395                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 396                         continue;
 397
 398                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 399                 strbuf_addstr(&bundle_ref, branch_name);
 400
 401                 has_old = !refs_read_ref(get_main_ref_store(the_repository),
 402                                          bundle_ref.buf, &old_oid);
 403                 refs_update_ref(get_main_ref_store(the_repository),
 404                                 "fetched bundle", bundle_ref.buf, oid,
 405                                 has_old ? &old_oid : NULL,
 406                                 0, UPDATE_REFS_MSG_ON_ERR);
 407         }
 408
 409         bundle_header_release(&header);
 410         return result;
 411 }
 412
 413 struct bundle_list_context {
 414         struct repository *r;
 415         struct bundle_list *list;
 416         enum bundle_list_mode mode;
 417         int count;
 418         int depth;
 419 };
 420
 421 /*
 422  * This early definition is necessary because we use indirect recursion:
 423  *
 424  * While iterating through a bundle list that was downloaded as part
 425  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 426  * again, but with depth + 1.
 427  */
 428 static int fetch_bundle_uri_internal(struct repository *r,
 429                                      struct remote_bundle_info *bundle,
 430                                      int depth,
 431                                      struct bundle_list *list);
 432
 433 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 434 {
 435         int res;
 436         struct bundle_list_context *ctx = data;
 437
 438         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 439                 return 0;
 440
 441         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 442
 443         /*
 444          * Only increment count if the download succeeded. If our mode is
 445          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 446          * list in case they work instead.
 447          */
 448         if (!res)
 449                 ctx->count++;
 450
 451         /*
 452          * To be opportunistic as possible, we continue iterating and
 453          * download as many bundles as we can, so we can apply the ones
 454          * that work, even in BUNDLE_MODE_ALL mode.
 455          */
 456         return 0;
 457 }
 458
 459 struct bundles_for_sorting {
 460         struct remote_bundle_info **items;
 461         size_t alloc;
 462         size_t nr;
 463 };
 464
 465 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 466 {
 467         struct bundles_for_sorting *list = data;
 468         list->items[list->nr++] = bundle;
 469         return 0;
 470 }
 471
 472 /**
 473  * For use in QSORT() to get a list sorted by creationToken
 474  * in decreasing order.
 475  */
 476 static int compare_creation_token_decreasing(const void *va, const void *vb)
 477 {
 478         const struct remote_bundle_info * const *a = va;
 479         const struct remote_bundle_info * const *b = vb;
 480
 481         if ((*a)->creationToken > (*b)->creationToken)
 482                 return -1;
 483         if ((*a)->creationToken < (*b)->creationToken)
 484                 return 1;
 485         return 0;
 486 }
 487
 488 static int fetch_bundles_by_token(struct repository *r,
 489                                   struct bundle_list *list)
 490 {
 491         int cur;
 492         int move_direction = 0;
 493         const char *creationTokenStr;
 494         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 495         struct bundle_list_context ctx = {
 496                 .r = r,
 497                 .list = list,
 498                 .mode = list->mode,
 499         };
 500         struct bundles_for_sorting bundles = {
 501                 .alloc = hashmap_get_size(&list->bundles),
 502         };
 503
 504         ALLOC_ARRAY(bundles.items, bundles.alloc);
 505
 506         for_all_bundles_in_list(list, append_bundle, &bundles);
 507
 508         if (!bundles.nr) {
 509                 free(bundles.items);
 510                 return 0;
 511         }
 512
 513         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 514
 515         /*
 516          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 517          * is not strictly smaller than the maximum creation token in the
 518          * bundle list, then do not download any bundles.
 519          */
 520         if (!repo_config_get_value(r,
 521                                    "fetch.bundlecreationtoken",
 522                                    &creationTokenStr) &&
 523             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 524             bundles.items[0]->creationToken <= maxCreationToken) {
 525                 free(bundles.items);
 526                 return 0;
 527         }
 528
 529         /*
 530          * Attempt to download and unbundle the minimum number of bundles by
 531          * creationToken in decreasing order. If we fail to unbundle (after
 532          * a successful download) then move to the next non-downloaded bundle
 533          * and attempt downloading. Once we succeed in applying a bundle,
 534          * move to the previous unapplied bundle and attempt to unbundle it
 535          * again.
 536          *
 537          * In the case of a fresh clone, we will likely download all of the
 538          * bundles before successfully unbundling the oldest one, then the
 539          * rest of the bundles unbundle successfully in increasing order
 540          * of creationToken.
 541          *
 542          * If there are existing objects, then this process may terminate
 543          * early when all required commits from "new" bundles exist in the
 544          * repo's object store.
 545          */
 546         cur = 0;
 547         while (cur >= 0 && cur < bundles.nr) {
 548                 struct remote_bundle_info *bundle = bundles.items[cur];
 549
 550                 /*
 551                  * If we need to dig into bundles below the previous
 552                  * creation token value, then likely we are in an erroneous
 553                  * state due to missing or invalid bundles. Halt the process
 554                  * instead of continuing to download extra data.
 555                  */
 556                 if (bundle->creationToken <= maxCreationToken)
 557                         break;
 558
 559                 if (!bundle->file) {
 560                         /*
 561                          * Not downloaded yet. Try downloading.
 562                          *
 563                          * Note that bundle->file is non-NULL if a download
 564                          * was attempted, even if it failed to download.
 565                          */
 566                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 567                                 /* Mark as unbundled so we do not retry. */
 568                                 bundle->unbundled = 1;
 569
 570                                 /* Try looking deeper in the list. */
 571                                 move_direction = 1;
 572                                 goto move;
 573                         }
 574
 575                         /* We expect bundles when using creationTokens. */
 576                         if (!is_bundle(bundle->file, 1)) {
 577                                 warning(_("file downloaded from '%s' is not a bundle"),
 578                                         bundle->uri);
 579                                 break;
 580                         }
 581                 }
 582
 583                 if (bundle->file && !bundle->unbundled) {
 584                         /*
 585                          * This was downloaded, but not successfully
 586                          * unbundled. Try unbundling again.
 587                          */
 588                         if (unbundle_from_file(ctx.r, bundle->file)) {
 589                                 /* Try looking deeper in the list. */
 590                                 move_direction = 1;
 591                         } else {
 592                                 /*
 593                                  * Succeeded in unbundle. Retry bundles
 594                                  * that previously failed to unbundle.
 595                                  */
 596                                 move_direction = -1;
 597                                 bundle->unbundled = 1;
 598
 599                                 if (bundle->creationToken > newMaxCreationToken)
 600                                         newMaxCreationToken = bundle->creationToken;
 601                         }
 602                 }
 603
 604                 /*
 605                  * Else case: downloaded and unbundled successfully.
 606                  * Skip this by moving in the same direction as the
 607                  * previous step.
 608                  */
 609
 610 move:
 611                 /* Move in the specified direction and repeat. */
 612                 cur += move_direction;
 613         }
 614
 615         /*
 616          * We succeed if the loop terminates because 'cur' drops below
 617          * zero. The other case is that we terminate because 'cur'
 618          * reaches the end of the list, so we have a failure no matter
 619          * which bundles we apply from the list.
 620          */
 621         if (cur < 0) {
 622                 struct strbuf value = STRBUF_INIT;
 623                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 624                 if (repo_config_set_multivar_gently(ctx.r,
 625                                                     "fetch.bundleCreationToken",
 626                                                     value.buf, NULL, 0))
 627                         warning(_("failed to store maximum creation token"));
 628
 629                 strbuf_release(&value);
 630         }
 631
 632         free(bundles.items);
 633         return cur >= 0;
 634 }
 635
 636 static int download_bundle_list(struct repository *r,
 637                                 struct bundle_list *local_list,
 638                                 struct bundle_list *global_list,
 639                                 int depth)
 640 {
 641         struct bundle_list_context ctx = {
 642                 .r = r,
 643                 .list = global_list,
 644                 .depth = depth + 1,
 645                 .mode = local_list->mode,
 646         };
 647
 648         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 649 }
 650
 651 static int fetch_bundle_list_in_config_format(struct repository *r,
 652                                               struct bundle_list *global_list,
 653                                               struct remote_bundle_info *bundle,
 654                                               int depth)
 655 {
 656         int result;
 657         struct bundle_list list_from_bundle;
 658
 659         init_bundle_list(&list_from_bundle);
 660
 661         if ((result = bundle_uri_parse_config_format(bundle->uri,
 662                                                      bundle->file,
 663                                                      &list_from_bundle)))
 664                 goto cleanup;
 665
 666         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 667                 warning(_("unrecognized bundle mode from URI '%s'"),
 668                         bundle->uri);
 669                 result = -1;
 670                 goto cleanup;
 671         }
 672
 673         /*
 674          * If this list uses the creationToken heuristic, then the URIs
 675          * it advertises are expected to be bundles, not nested lists.
 676          * We can drop 'global_list' and 'depth'.
 677          */
 678         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 679                 result = fetch_bundles_by_token(r, &list_from_bundle);
 680                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 681         } else if ((result = download_bundle_list(r, &list_from_bundle,
 682                                            global_list, depth)))
 683                 goto cleanup;
 684
 685 cleanup:
 686         clear_bundle_list(&list_from_bundle);
 687         return result;
 688 }
 689
 690 /**
 691  * This limits the recursion on fetch_bundle_uri_internal() when following
 692  * bundle lists.
 693  */
 694 static int max_bundle_uri_depth = 4;
 695
 696 /**
 697  * Recursively download all bundles advertised at the given URI
 698  * to files. If the file is a bundle, then add it to the given
 699  * 'list'. Otherwise, expect a bundle list and recurse on the
 700  * URIs in that list according to the list mode (ANY or ALL).
 701  */
 702 static int fetch_bundle_uri_internal(struct repository *r,
 703                                      struct remote_bundle_info *bundle,
 704                                      int depth,
 705                                      struct bundle_list *list)
 706 {
 707         int result = 0;
 708         struct remote_bundle_info *bcopy;
 709
 710         if (depth >= max_bundle_uri_depth) {
 711                 warning(_("exceeded bundle URI recursion limit (%d)"),
 712                         max_bundle_uri_depth);
 713                 return -1;
 714         }
 715
 716         if (!bundle->file &&
 717             !(bundle->file = find_temp_filename())) {
 718                 result = -1;
 719                 goto cleanup;
 720         }
 721
 722         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 723                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 724                 goto cleanup;
 725         }
 726
 727         if ((result = !is_bundle(bundle->file, 1))) {
 728                 result = fetch_bundle_list_in_config_format(
 729                                 r, list, bundle, depth);
 730                 if (result)
 731                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 732                                 bundle->uri);
 733                 goto cleanup;
 734         }
 735
 736         /* Copy the bundle and insert it into the global list. */
 737         CALLOC_ARRAY(bcopy, 1);
 738         bcopy->id = xstrdup(bundle->id);
 739         bcopy->file = xstrdup(bundle->file);
 740         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 741         hashmap_add(&list->bundles, &bcopy->ent);
 742
 743 cleanup:
 744         if (result && bundle->file)
 745                 unlink(bundle->file);
 746         return result;
 747 }
 748
 749 /**
 750  * This loop iterator breaks the loop with nonzero return code on the
 751  * first successful unbundling of a bundle.
 752  */
 753 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 754 {
 755         struct repository *r = data;
 756
 757         if (!info->file || info->unbundled)
 758                 return 0;
 759
 760         if (!unbundle_from_file(r, info->file)) {
 761                 info->unbundled = 1;
 762                 return 1;
 763         }
 764
 765         return 0;
 766 }
 767
 768 static int unbundle_all_bundles(struct repository *r,
 769                                 struct bundle_list *list)
 770 {
 771         /*
 772          * Iterate through all bundles looking for ones that can
 773          * successfully unbundle. If any succeed, then perhaps another
 774          * will succeed in the next attempt.
 775          *
 776          * Keep in mind that a non-zero result for the loop here means
 777          * the loop terminated early on a successful unbundling, which
 778          * signals that we can try again.
 779          */
 780         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 781
 782         return 0;
 783 }
 784
 785 static int unlink_bundle(struct remote_bundle_info *info, void *data UNUSED)
 786 {
 787         if (info->file)
 788                 unlink_or_warn(info->file);
 789         return 0;
 790 }
 791
 792 int fetch_bundle_uri(struct repository *r, const char *uri,
 793                      int *has_heuristic)
 794 {
 795         int result;
 796         struct bundle_list list;
 797         struct remote_bundle_info bundle = {
 798                 .uri = xstrdup(uri),
 799                 .id = xstrdup(""),
 800         };
 801
 802         init_bundle_list(&list);
 803
 804         /*
 805          * Do not fetch an empty bundle URI. An empty bundle URI
 806          * could signal that a configured bundle URI has been disabled.
 807          */
 808         if (!*uri) {
 809                 result = 0;
 810                 goto cleanup;
 811         }
 812
 813         /* If a bundle is added to this global list, then it is required. */
 814         list.mode = BUNDLE_MODE_ALL;
 815
 816         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 817                 goto cleanup;
 818
 819         result = unbundle_all_bundles(r, &list);
 820
 821 cleanup:
 822         if (has_heuristic)
 823                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 824         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 825         clear_bundle_list(&list);
 826         clear_remote_bundle_info(&bundle, NULL);
 827         return result;
 828 }
 829
 830 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 831 {
 832         int result;
 833         struct bundle_list global_list;
 834
 835         /*
 836          * If the creationToken heuristic is used, then the URIs
 837          * advertised by 'list' are not nested lists and instead
 838          * direct bundles. We do not need to use global_list.
 839          */
 840         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 841                 return fetch_bundles_by_token(r, list);
 842
 843         init_bundle_list(&global_list);
 844
 845         /* If a bundle is added to this global list, then it is required. */
 846         global_list.mode = BUNDLE_MODE_ALL;
 847
 848         if ((result = download_bundle_list(r, list, &global_list, 0)))
 849                 goto cleanup;
 850
 851         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 852                 result = fetch_bundles_by_token(r, list);
 853         else
 854                 result = unbundle_all_bundles(r, &global_list);
 855
 856 cleanup:
 857         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 858         clear_bundle_list(&global_list);
 859         return result;
 860 }
 861
 862 /**
 863  * API for serve.c.
 864  */
 865
 866 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 867 {
 868         static int advertise_bundle_uri = -1;
 869
 870         if (advertise_bundle_uri != -1)
 871                 goto cached;
 872
 873         advertise_bundle_uri = 0;
 874         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 875
 876 cached:
 877         return advertise_bundle_uri;
 878 }
 879
 880 static int config_to_packet_line(const char *key, const char *value,
 881                                  const struct config_context *ctx UNUSED,
 882                                  void *data)
 883 {
 884         struct packet_reader *writer = data;
 885
 886         if (starts_with(key, "bundle."))
 887                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 888
 889         return 0;
 890 }
 891
 892 int bundle_uri_command(struct repository *r,
 893                        struct packet_reader *request)
 894 {
 895         struct packet_writer writer;
 896         packet_writer_init(&writer, 1);
 897
 898         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 899                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 900         if (request->status != PACKET_READ_FLUSH)
 901                 die(_("bundle-uri: expected flush after arguments"));
 902
 903         /*
 904          * Read all "bundle.*" config lines to the client as key=value
 905          * packet lines.
 906          */
 907         repo_config(r, config_to_packet_line, &writer);
 908
 909         packet_writer_flush(&writer);
 910
 911         return 0;
 912 }
 913
 914 /**
 915  * General API for {transport,connect}.c etc.
 916  */
 917 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 918 {
 919         int result;
 920         const char *equals;
 921         struct strbuf key = STRBUF_INIT;
 922
 923         if (!strlen(line))
 924                 return error(_("bundle-uri: got an empty line"));
 925
 926         equals = strchr(line, '=');
 927
 928         if (!equals)
 929                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 930         if (line == equals || !*(equals + 1))
 931                 return error(_("bundle-uri: line has empty key or value"));
 932
 933         strbuf_add(&key, line, equals - line);
 934         result = bundle_list_update(key.buf, equals + 1, list);
 935         strbuf_release(&key);
 936
 937         return result;
 938 }