bundle-uri.c

   1 #define USE_THE_REPOSITORY_VARIABLE
   2
   3 #include "git-compat-util.h"
   4 #include "bundle-uri.h"
   5 #include "bundle.h"
   6 #include "copy.h"
   7 #include "environment.h"
   8 #include "gettext.h"
   9 #include "refs.h"
  10 #include "run-command.h"
  11 #include "hashmap.h"
  12 #include "pkt-line.h"
  13 #include "config.h"
  14 #include "fetch-pack.h"
  15 #include "remote.h"
  16 #include "trace2.h"
  17
  18 static struct {
  19         enum bundle_list_heuristic heuristic;
  20         const char *name;
  21 } heuristics[BUNDLE_HEURISTIC__COUNT] = {
  22         { BUNDLE_HEURISTIC_NONE, ""},
  23         { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" },
  24 };
  25
  26 static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED,
  27                            const struct hashmap_entry *he1,
  28                            const struct hashmap_entry *he2,
  29                            const void *id)
  30 {
  31         const struct remote_bundle_info *e1 =
  32                 container_of(he1, const struct remote_bundle_info, ent);
  33         const struct remote_bundle_info *e2 =
  34                 container_of(he2, const struct remote_bundle_info, ent);
  35
  36         return strcmp(e1->id, id ? (const char *)id : e2->id);
  37 }
  38
  39 void init_bundle_list(struct bundle_list *list)
  40 {
  41         memset(list, 0, sizeof(*list));
  42
  43         /* Implied defaults. */
  44         list->mode = BUNDLE_MODE_ALL;
  45         list->version = 1;
  46
  47         hashmap_init(&list->bundles, compare_bundles, NULL, 0);
  48 }
  49
  50 static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
  51                                     void *data UNUSED)
  52 {
  53         FREE_AND_NULL(bundle->id);
  54         FREE_AND_NULL(bundle->uri);
  55         FREE_AND_NULL(bundle->file);
  56         bundle->unbundled = 0;
  57         return 0;
  58 }
  59
  60 void clear_bundle_list(struct bundle_list *list)
  61 {
  62         if (!list)
  63                 return;
  64
  65         for_all_bundles_in_list(list, clear_remote_bundle_info, NULL);
  66         hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent);
  67         free(list->baseURI);
  68 }
  69
  70 int for_all_bundles_in_list(struct bundle_list *list,
  71                             bundle_iterator iter,
  72                             void *data)
  73 {
  74         struct remote_bundle_info *info;
  75         struct hashmap_iter i;
  76
  77         hashmap_for_each_entry(&list->bundles, &i, info, ent) {
  78                 int result = iter(info, data);
  79
  80                 if (result)
  81                         return result;
  82         }
  83
  84         return 0;
  85 }
  86
  87 static int summarize_bundle(struct remote_bundle_info *info, void *data)
  88 {
  89         FILE *fp = data;
  90         fprintf(fp, "[bundle \"%s\"]\n", info->id);
  91         fprintf(fp, "\turi = %s\n", info->uri);
  92
  93         if (info->creationToken)
  94                 fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken);
  95         return 0;
  96 }
  97
  98 void print_bundle_list(FILE *fp, struct bundle_list *list)
  99 {
 100         const char *mode;
 101
 102         switch (list->mode) {
 103         case BUNDLE_MODE_ALL:
 104                 mode = "all";
 105                 break;
 106
 107         case BUNDLE_MODE_ANY:
 108                 mode = "any";
 109                 break;
 110
 111         case BUNDLE_MODE_NONE:
 112         default:
 113                 mode = "<unknown>";
 114         }
 115
 116         fprintf(fp, "[bundle]\n");
 117         fprintf(fp, "\tversion = %d\n", list->version);
 118         fprintf(fp, "\tmode = %s\n", mode);
 119
 120         if (list->heuristic) {
 121                 int i;
 122                 for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 123                         if (heuristics[i].heuristic == list->heuristic) {
 124                                 printf("\theuristic = %s\n",
 125                                        heuristics[list->heuristic].name);
 126                                 break;
 127                         }
 128                 }
 129         }
 130
 131         for_all_bundles_in_list(list, summarize_bundle, fp);
 132 }
 133
 134 /**
 135  * Given a key-value pair, update the state of the given bundle list.
 136  * Returns 0 if the key-value pair is understood. Returns -1 if the key
 137  * is not understood or the value is malformed.
 138  */
 139 static int bundle_list_update(const char *key, const char *value,
 140                               struct bundle_list *list)
 141 {
 142         struct strbuf id = STRBUF_INIT;
 143         struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT;
 144         struct remote_bundle_info *bundle;
 145         const char *subsection, *subkey;
 146         size_t subsection_len;
 147
 148         if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey))
 149                 return -1;
 150
 151         if (!subsection_len) {
 152                 if (!strcmp(subkey, "version")) {
 153                         int version;
 154                         if (!git_parse_int(value, &version))
 155                                 return -1;
 156                         if (version != 1)
 157                                 return -1;
 158
 159                         list->version = version;
 160                         return 0;
 161                 }
 162
 163                 if (!strcmp(subkey, "mode")) {
 164                         if (!strcmp(value, "all"))
 165                                 list->mode = BUNDLE_MODE_ALL;
 166                         else if (!strcmp(value, "any"))
 167                                 list->mode = BUNDLE_MODE_ANY;
 168                         else
 169                                 return -1;
 170                         return 0;
 171                 }
 172
 173                 if (!strcmp(subkey, "heuristic")) {
 174                         int i;
 175                         for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) {
 176                                 if (heuristics[i].heuristic &&
 177                                     heuristics[i].name &&
 178                                     !strcmp(value, heuristics[i].name)) {
 179                                         list->heuristic = heuristics[i].heuristic;
 180                                         return 0;
 181                                 }
 182                         }
 183
 184                         /* Ignore unknown heuristics. */
 185                         return 0;
 186                 }
 187
 188                 /* Ignore other unknown global keys. */
 189                 return 0;
 190         }
 191
 192         strbuf_add(&id, subsection, subsection_len);
 193
 194         /*
 195          * Check for an existing bundle with this <id>, or create one
 196          * if necessary.
 197          */
 198         lookup.id = id.buf;
 199         hashmap_entry_init(&lookup.ent, strhash(lookup.id));
 200         if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) {
 201                 CALLOC_ARRAY(bundle, 1);
 202                 bundle->id = strbuf_detach(&id, NULL);
 203                 hashmap_entry_init(&bundle->ent, strhash(bundle->id));
 204                 hashmap_add(&list->bundles, &bundle->ent);
 205         }
 206         strbuf_release(&id);
 207
 208         if (!strcmp(subkey, "uri")) {
 209                 if (bundle->uri)
 210                         return -1;
 211                 bundle->uri = relative_url(list->baseURI, value, NULL);
 212                 return 0;
 213         }
 214
 215         if (!strcmp(subkey, "creationtoken")) {
 216                 if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1)
 217                         warning(_("could not parse bundle list key %s with value '%s'"),
 218                                 "creationToken", value);
 219                 return 0;
 220         }
 221
 222         /*
 223          * At this point, we ignore any information that we don't
 224          * understand, assuming it to be hints for a heuristic the client
 225          * does not currently understand.
 226          */
 227         return 0;
 228 }
 229
 230 static int config_to_bundle_list(const char *key, const char *value,
 231                                  const struct config_context *ctx UNUSED,
 232                                  void *data)
 233 {
 234         struct bundle_list *list = data;
 235         return bundle_list_update(key, value, list);
 236 }
 237
 238 int bundle_uri_parse_config_format(const char *uri,
 239                                    const char *filename,
 240                                    struct bundle_list *list)
 241 {
 242         int result;
 243         struct config_options opts = {
 244                 .error_action = CONFIG_ERROR_ERROR,
 245         };
 246
 247         if (!list->baseURI) {
 248                 struct strbuf baseURI = STRBUF_INIT;
 249                 strbuf_addstr(&baseURI, uri);
 250
 251                 /*
 252                  * If the URI does not end with a trailing slash, then
 253                  * remove the filename portion of the path. This is
 254                  * important for relative URIs.
 255                  */
 256                 strbuf_strip_file_from_path(&baseURI);
 257                 list->baseURI = strbuf_detach(&baseURI, NULL);
 258         }
 259         result = git_config_from_file_with_options(config_to_bundle_list,
 260                                                    filename, list,
 261                                                    CONFIG_SCOPE_UNKNOWN,
 262                                                    &opts);
 263
 264         if (!result && list->mode == BUNDLE_MODE_NONE) {
 265                 warning(_("bundle list at '%s' has no mode"), uri);
 266                 result = 1;
 267         }
 268
 269         return result;
 270 }
 271
 272 static char *find_temp_filename(void)
 273 {
 274         int fd;
 275         struct strbuf name = STRBUF_INIT;
 276         /*
 277          * Find a temporary filename that is available. This is briefly
 278          * racy, but unlikely to collide.
 279          */
 280         fd = odb_mkstemp(&name, "bundles/tmp_uri_XXXXXX");
 281         if (fd < 0) {
 282                 warning(_("failed to create temporary file"));
 283                 return NULL;
 284         }
 285
 286         close(fd);
 287         unlink(name.buf);
 288         return strbuf_detach(&name, NULL);
 289 }
 290
 291 static int download_https_uri_to_file(const char *file, const char *uri)
 292 {
 293         int result = 0;
 294         struct child_process cp = CHILD_PROCESS_INIT;
 295         FILE *child_in = NULL, *child_out = NULL;
 296         struct strbuf line = STRBUF_INIT;
 297         int found_get = 0;
 298
 299         strvec_pushl(&cp.args, "git-remote-https", uri, NULL);
 300         cp.err = -1;
 301         cp.in = -1;
 302         cp.out = -1;
 303
 304         if (start_command(&cp))
 305                 return 1;
 306
 307         child_in = fdopen(cp.in, "w");
 308         if (!child_in) {
 309                 result = 1;
 310                 goto cleanup;
 311         }
 312
 313         child_out = fdopen(cp.out, "r");
 314         if (!child_out) {
 315                 result = 1;
 316                 goto cleanup;
 317         }
 318
 319         fprintf(child_in, "capabilities\n");
 320         fflush(child_in);
 321
 322         while (!strbuf_getline(&line, child_out)) {
 323                 if (!line.len)
 324                         break;
 325                 if (!strcmp(line.buf, "get"))
 326                         found_get = 1;
 327         }
 328         strbuf_release(&line);
 329
 330         if (!found_get) {
 331                 result = error(_("insufficient capabilities"));
 332                 goto cleanup;
 333         }
 334
 335         fprintf(child_in, "get %s %s\n\n", uri, file);
 336
 337 cleanup:
 338         if (child_in)
 339                 fclose(child_in);
 340         if (finish_command(&cp))
 341                 return 1;
 342         if (child_out)
 343                 fclose(child_out);
 344         return result;
 345 }
 346
 347 static int copy_uri_to_file(const char *filename, const char *uri)
 348 {
 349         const char *out;
 350
 351         if (starts_with(uri, "https:") ||
 352             starts_with(uri, "http:"))
 353                 return download_https_uri_to_file(filename, uri);
 354
 355         if (skip_prefix(uri, "file://", &out))
 356                 uri = out;
 357
 358         /* Copy as a file */
 359         return copy_file(filename, uri, 0);
 360 }
 361
 362 static int unbundle_from_file(struct repository *r, const char *file)
 363 {
 364         int result = 0;
 365         int bundle_fd;
 366         struct bundle_header header = BUNDLE_HEADER_INIT;
 367         struct string_list_item *refname;
 368         struct strbuf bundle_ref = STRBUF_INIT;
 369         size_t bundle_prefix_len;
 370
 371         if ((bundle_fd = read_bundle_header(file, &header)) < 0)
 372                 return 1;
 373
 374         /*
 375          * Skip the reachability walk here, since we will be adding
 376          * a reachable ref pointing to the new tips, which will reach
 377          * the prerequisite commits.
 378          */
 379         if ((result = unbundle(r, &header, bundle_fd, NULL,
 380                                VERIFY_BUNDLE_QUIET | (fetch_pack_fsck_objects() ? VERIFY_BUNDLE_FSCK : 0))))
 381                 return 1;
 382
 383         /*
 384          * Convert all refs/heads/ from the bundle into refs/bundles/
 385          * in the local repository.
 386          */
 387         strbuf_addstr(&bundle_ref, "refs/bundles/");
 388         bundle_prefix_len = bundle_ref.len;
 389
 390         for_each_string_list_item(refname, &header.references) {
 391                 struct object_id *oid = refname->util;
 392                 struct object_id old_oid;
 393                 const char *branch_name;
 394                 int has_old;
 395
 396                 if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
 397                         continue;
 398
 399                 strbuf_setlen(&bundle_ref, bundle_prefix_len);
 400                 strbuf_addstr(&bundle_ref, branch_name);
 401
 402                 has_old = !refs_read_ref(get_main_ref_store(the_repository),
 403                                          bundle_ref.buf, &old_oid);
 404                 refs_update_ref(get_main_ref_store(the_repository),
 405                                 "fetched bundle", bundle_ref.buf, oid,
 406                                 has_old ? &old_oid : NULL,
 407                                 0, UPDATE_REFS_MSG_ON_ERR);
 408         }
 409
 410         bundle_header_release(&header);
 411         return result;
 412 }
 413
 414 struct bundle_list_context {
 415         struct repository *r;
 416         struct bundle_list *list;
 417         enum bundle_list_mode mode;
 418         int count;
 419         int depth;
 420 };
 421
 422 /*
 423  * This early definition is necessary because we use indirect recursion:
 424  *
 425  * While iterating through a bundle list that was downloaded as part
 426  * of fetch_bundle_uri_internal(), iterator methods eventually call it
 427  * again, but with depth + 1.
 428  */
 429 static int fetch_bundle_uri_internal(struct repository *r,
 430                                      struct remote_bundle_info *bundle,
 431                                      int depth,
 432                                      struct bundle_list *list);
 433
 434 static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
 435 {
 436         int res;
 437         struct bundle_list_context *ctx = data;
 438
 439         if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
 440                 return 0;
 441
 442         res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
 443
 444         /*
 445          * Only increment count if the download succeeded. If our mode is
 446          * BUNDLE_MODE_ANY, then we will want to try other URIs in the
 447          * list in case they work instead.
 448          */
 449         if (!res)
 450                 ctx->count++;
 451
 452         /*
 453          * To be opportunistic as possible, we continue iterating and
 454          * download as many bundles as we can, so we can apply the ones
 455          * that work, even in BUNDLE_MODE_ALL mode.
 456          */
 457         return 0;
 458 }
 459
 460 struct bundles_for_sorting {
 461         struct remote_bundle_info **items;
 462         size_t alloc;
 463         size_t nr;
 464 };
 465
 466 static int append_bundle(struct remote_bundle_info *bundle, void *data)
 467 {
 468         struct bundles_for_sorting *list = data;
 469         list->items[list->nr++] = bundle;
 470         return 0;
 471 }
 472
 473 /**
 474  * For use in QSORT() to get a list sorted by creationToken
 475  * in decreasing order.
 476  */
 477 static int compare_creation_token_decreasing(const void *va, const void *vb)
 478 {
 479         const struct remote_bundle_info * const *a = va;
 480         const struct remote_bundle_info * const *b = vb;
 481
 482         if ((*a)->creationToken > (*b)->creationToken)
 483                 return -1;
 484         if ((*a)->creationToken < (*b)->creationToken)
 485                 return 1;
 486         return 0;
 487 }
 488
 489 static int fetch_bundles_by_token(struct repository *r,
 490                                   struct bundle_list *list)
 491 {
 492         int cur;
 493         int move_direction = 0;
 494         const char *creationTokenStr;
 495         uint64_t maxCreationToken = 0, newMaxCreationToken = 0;
 496         struct bundle_list_context ctx = {
 497                 .r = r,
 498                 .list = list,
 499                 .mode = list->mode,
 500         };
 501         struct bundles_for_sorting bundles = {
 502                 .alloc = hashmap_get_size(&list->bundles),
 503         };
 504
 505         ALLOC_ARRAY(bundles.items, bundles.alloc);
 506
 507         for_all_bundles_in_list(list, append_bundle, &bundles);
 508
 509         if (!bundles.nr) {
 510                 free(bundles.items);
 511                 return 0;
 512         }
 513
 514         QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
 515
 516         /*
 517          * If fetch.bundleCreationToken exists, parses to a uint64t, and
 518          * is not strictly smaller than the maximum creation token in the
 519          * bundle list, then do not download any bundles.
 520          */
 521         if (!repo_config_get_value(r,
 522                                    "fetch.bundlecreationtoken",
 523                                    &creationTokenStr) &&
 524             sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) == 1 &&
 525             bundles.items[0]->creationToken <= maxCreationToken) {
 526                 free(bundles.items);
 527                 return 0;
 528         }
 529
 530         /*
 531          * Attempt to download and unbundle the minimum number of bundles by
 532          * creationToken in decreasing order. If we fail to unbundle (after
 533          * a successful download) then move to the next non-downloaded bundle
 534          * and attempt downloading. Once we succeed in applying a bundle,
 535          * move to the previous unapplied bundle and attempt to unbundle it
 536          * again.
 537          *
 538          * In the case of a fresh clone, we will likely download all of the
 539          * bundles before successfully unbundling the oldest one, then the
 540          * rest of the bundles unbundle successfully in increasing order
 541          * of creationToken.
 542          *
 543          * If there are existing objects, then this process may terminate
 544          * early when all required commits from "new" bundles exist in the
 545          * repo's object store.
 546          */
 547         cur = 0;
 548         while (cur >= 0 && cur < bundles.nr) {
 549                 struct remote_bundle_info *bundle = bundles.items[cur];
 550
 551                 /*
 552                  * If we need to dig into bundles below the previous
 553                  * creation token value, then likely we are in an erroneous
 554                  * state due to missing or invalid bundles. Halt the process
 555                  * instead of continuing to download extra data.
 556                  */
 557                 if (bundle->creationToken <= maxCreationToken)
 558                         break;
 559
 560                 if (!bundle->file) {
 561                         /*
 562                          * Not downloaded yet. Try downloading.
 563                          *
 564                          * Note that bundle->file is non-NULL if a download
 565                          * was attempted, even if it failed to download.
 566                          */
 567                         if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
 568                                 /* Mark as unbundled so we do not retry. */
 569                                 bundle->unbundled = 1;
 570
 571                                 /* Try looking deeper in the list. */
 572                                 move_direction = 1;
 573                                 goto move;
 574                         }
 575
 576                         /* We expect bundles when using creationTokens. */
 577                         if (!is_bundle(bundle->file, 1)) {
 578                                 warning(_("file downloaded from '%s' is not a bundle"),
 579                                         bundle->uri);
 580                                 break;
 581                         }
 582                 }
 583
 584                 if (bundle->file && !bundle->unbundled) {
 585                         /*
 586                          * This was downloaded, but not successfully
 587                          * unbundled. Try unbundling again.
 588                          */
 589                         if (unbundle_from_file(ctx.r, bundle->file)) {
 590                                 /* Try looking deeper in the list. */
 591                                 move_direction = 1;
 592                         } else {
 593                                 /*
 594                                  * Succeeded in unbundle. Retry bundles
 595                                  * that previously failed to unbundle.
 596                                  */
 597                                 move_direction = -1;
 598                                 bundle->unbundled = 1;
 599
 600                                 if (bundle->creationToken > newMaxCreationToken)
 601                                         newMaxCreationToken = bundle->creationToken;
 602                         }
 603                 }
 604
 605                 /*
 606                  * Else case: downloaded and unbundled successfully.
 607                  * Skip this by moving in the same direction as the
 608                  * previous step.
 609                  */
 610
 611 move:
 612                 /* Move in the specified direction and repeat. */
 613                 cur += move_direction;
 614         }
 615
 616         /*
 617          * We succeed if the loop terminates because 'cur' drops below
 618          * zero. The other case is that we terminate because 'cur'
 619          * reaches the end of the list, so we have a failure no matter
 620          * which bundles we apply from the list.
 621          */
 622         if (cur < 0) {
 623                 struct strbuf value = STRBUF_INIT;
 624                 strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken);
 625                 if (repo_config_set_multivar_gently(ctx.r,
 626                                                     "fetch.bundleCreationToken",
 627                                                     value.buf, NULL, 0))
 628                         warning(_("failed to store maximum creation token"));
 629
 630                 strbuf_release(&value);
 631         }
 632
 633         free(bundles.items);
 634         return cur >= 0;
 635 }
 636
 637 static int download_bundle_list(struct repository *r,
 638                                 struct bundle_list *local_list,
 639                                 struct bundle_list *global_list,
 640                                 int depth)
 641 {
 642         struct bundle_list_context ctx = {
 643                 .r = r,
 644                 .list = global_list,
 645                 .depth = depth + 1,
 646                 .mode = local_list->mode,
 647         };
 648
 649         return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
 650 }
 651
 652 static int fetch_bundle_list_in_config_format(struct repository *r,
 653                                               struct bundle_list *global_list,
 654                                               struct remote_bundle_info *bundle,
 655                                               int depth)
 656 {
 657         int result;
 658         struct bundle_list list_from_bundle;
 659
 660         init_bundle_list(&list_from_bundle);
 661
 662         if ((result = bundle_uri_parse_config_format(bundle->uri,
 663                                                      bundle->file,
 664                                                      &list_from_bundle)))
 665                 goto cleanup;
 666
 667         if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
 668                 warning(_("unrecognized bundle mode from URI '%s'"),
 669                         bundle->uri);
 670                 result = -1;
 671                 goto cleanup;
 672         }
 673
 674         /*
 675          * If this list uses the creationToken heuristic, then the URIs
 676          * it advertises are expected to be bundles, not nested lists.
 677          * We can drop 'global_list' and 'depth'.
 678          */
 679         if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
 680                 result = fetch_bundles_by_token(r, &list_from_bundle);
 681                 global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
 682         } else if ((result = download_bundle_list(r, &list_from_bundle,
 683                                            global_list, depth)))
 684                 goto cleanup;
 685
 686 cleanup:
 687         clear_bundle_list(&list_from_bundle);
 688         return result;
 689 }
 690
 691 /**
 692  * This limits the recursion on fetch_bundle_uri_internal() when following
 693  * bundle lists.
 694  */
 695 static int max_bundle_uri_depth = 4;
 696
 697 /**
 698  * Recursively download all bundles advertised at the given URI
 699  * to files. If the file is a bundle, then add it to the given
 700  * 'list'. Otherwise, expect a bundle list and recurse on the
 701  * URIs in that list according to the list mode (ANY or ALL).
 702  */
 703 static int fetch_bundle_uri_internal(struct repository *r,
 704                                      struct remote_bundle_info *bundle,
 705                                      int depth,
 706                                      struct bundle_list *list)
 707 {
 708         int result = 0;
 709         struct remote_bundle_info *bcopy;
 710
 711         if (depth >= max_bundle_uri_depth) {
 712                 warning(_("exceeded bundle URI recursion limit (%d)"),
 713                         max_bundle_uri_depth);
 714                 return -1;
 715         }
 716
 717         if (!bundle->file &&
 718             !(bundle->file = find_temp_filename())) {
 719                 result = -1;
 720                 goto cleanup;
 721         }
 722
 723         if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
 724                 warning(_("failed to download bundle from URI '%s'"), bundle->uri);
 725                 goto cleanup;
 726         }
 727
 728         if ((result = !is_bundle(bundle->file, 1))) {
 729                 result = fetch_bundle_list_in_config_format(
 730                                 r, list, bundle, depth);
 731                 if (result)
 732                         warning(_("file at URI '%s' is not a bundle or bundle list"),
 733                                 bundle->uri);
 734                 goto cleanup;
 735         }
 736
 737         /* Copy the bundle and insert it into the global list. */
 738         CALLOC_ARRAY(bcopy, 1);
 739         bcopy->id = xstrdup(bundle->id);
 740         bcopy->file = xstrdup(bundle->file);
 741         hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
 742         hashmap_add(&list->bundles, &bcopy->ent);
 743
 744 cleanup:
 745         if (result && bundle->file)
 746                 unlink(bundle->file);
 747         return result;
 748 }
 749
 750 /**
 751  * This loop iterator breaks the loop with nonzero return code on the
 752  * first successful unbundling of a bundle.
 753  */
 754 static int attempt_unbundle(struct remote_bundle_info *info, void *data)
 755 {
 756         struct repository *r = data;
 757
 758         if (!info->file || info->unbundled)
 759                 return 0;
 760
 761         if (!unbundle_from_file(r, info->file)) {
 762                 info->unbundled = 1;
 763                 return 1;
 764         }
 765
 766         return 0;
 767 }
 768
 769 static int unbundle_all_bundles(struct repository *r,
 770                                 struct bundle_list *list)
 771 {
 772         /*
 773          * Iterate through all bundles looking for ones that can
 774          * successfully unbundle. If any succeed, then perhaps another
 775          * will succeed in the next attempt.
 776          *
 777          * Keep in mind that a non-zero result for the loop here means
 778          * the loop terminated early on a successful unbundling, which
 779          * signals that we can try again.
 780          */
 781         while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
 782
 783         return 0;
 784 }
 785
 786 static int unlink_bundle(struct remote_bundle_info *info, void *data UNUSED)
 787 {
 788         if (info->file)
 789                 unlink_or_warn(info->file);
 790         return 0;
 791 }
 792
 793 int fetch_bundle_uri(struct repository *r, const char *uri,
 794                      int *has_heuristic)
 795 {
 796         int result;
 797         struct bundle_list list;
 798         struct remote_bundle_info bundle = {
 799                 .uri = xstrdup(uri),
 800                 .id = xstrdup(""),
 801         };
 802
 803         trace2_region_enter("fetch", "fetch-bundle-uri", the_repository);
 804
 805         init_bundle_list(&list);
 806
 807         /*
 808          * Do not fetch an empty bundle URI. An empty bundle URI
 809          * could signal that a configured bundle URI has been disabled.
 810          */
 811         if (!*uri) {
 812                 result = 0;
 813                 goto cleanup;
 814         }
 815
 816         /* If a bundle is added to this global list, then it is required. */
 817         list.mode = BUNDLE_MODE_ALL;
 818
 819         if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
 820                 goto cleanup;
 821
 822         result = unbundle_all_bundles(r, &list);
 823
 824 cleanup:
 825         if (has_heuristic)
 826                 *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE);
 827         for_all_bundles_in_list(&list, unlink_bundle, NULL);
 828         clear_bundle_list(&list);
 829         clear_remote_bundle_info(&bundle, NULL);
 830         trace2_region_leave("fetch", "fetch-bundle-uri", the_repository);
 831         return result;
 832 }
 833
 834 int fetch_bundle_list(struct repository *r, struct bundle_list *list)
 835 {
 836         int result;
 837         struct bundle_list global_list;
 838
 839         /*
 840          * If the creationToken heuristic is used, then the URIs
 841          * advertised by 'list' are not nested lists and instead
 842          * direct bundles. We do not need to use global_list.
 843          */
 844         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 845                 return fetch_bundles_by_token(r, list);
 846
 847         init_bundle_list(&global_list);
 848
 849         /* If a bundle is added to this global list, then it is required. */
 850         global_list.mode = BUNDLE_MODE_ALL;
 851
 852         if ((result = download_bundle_list(r, list, &global_list, 0)))
 853                 goto cleanup;
 854
 855         if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
 856                 result = fetch_bundles_by_token(r, list);
 857         else
 858                 result = unbundle_all_bundles(r, &global_list);
 859
 860 cleanup:
 861         for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
 862         clear_bundle_list(&global_list);
 863         return result;
 864 }
 865
 866 /**
 867  * API for serve.c.
 868  */
 869
 870 int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED)
 871 {
 872         static int advertise_bundle_uri = -1;
 873
 874         if (advertise_bundle_uri != -1)
 875                 goto cached;
 876
 877         advertise_bundle_uri = 0;
 878         repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri);
 879
 880 cached:
 881         return advertise_bundle_uri;
 882 }
 883
 884 static int config_to_packet_line(const char *key, const char *value,
 885                                  const struct config_context *ctx UNUSED,
 886                                  void *data)
 887 {
 888         struct packet_reader *writer = data;
 889
 890         if (starts_with(key, "bundle."))
 891                 packet_write_fmt(writer->fd, "%s=%s", key, value);
 892
 893         return 0;
 894 }
 895
 896 int bundle_uri_command(struct repository *r,
 897                        struct packet_reader *request)
 898 {
 899         struct packet_writer writer;
 900         packet_writer_init(&writer, 1);
 901
 902         while (packet_reader_read(request) == PACKET_READ_NORMAL)
 903                 die(_("bundle-uri: unexpected argument: '%s'"), request->line);
 904         if (request->status != PACKET_READ_FLUSH)
 905                 die(_("bundle-uri: expected flush after arguments"));
 906
 907         /*
 908          * Read all "bundle.*" config lines to the client as key=value
 909          * packet lines.
 910          */
 911         repo_config(r, config_to_packet_line, &writer);
 912
 913         packet_writer_flush(&writer);
 914
 915         return 0;
 916 }
 917
 918 /**
 919  * General API for {transport,connect}.c etc.
 920  */
 921 int bundle_uri_parse_line(struct bundle_list *list, const char *line)
 922 {
 923         int result;
 924         const char *equals;
 925         struct strbuf key = STRBUF_INIT;
 926
 927         if (!strlen(line))
 928                 return error(_("bundle-uri: got an empty line"));
 929
 930         equals = strchr(line, '=');
 931
 932         if (!equals)
 933                 return error(_("bundle-uri: line is not of the form 'key=value'"));
 934         if (line == equals || !*(equals + 1))
 935                 return error(_("bundle-uri: line has empty key or value"));
 936
 937         strbuf_add(&key, line, equals - line);
 938         result = bundle_list_update(key.buf, equals + 1, list);
 939         strbuf_release(&key);
 940
 941         return result;
 942 }