1 #define USE_THE_REPOSITORY_VARIABLE
3 #include "git-compat-util.h"
4 #include "bundle-uri.h"
9 #include "run-command.h"
13 #include "fetch-pack.h"
16 #include "object-store-ll.h"
19 enum bundle_list_heuristic heuristic
;
21 } heuristics
[BUNDLE_HEURISTIC__COUNT
] = {
22 { BUNDLE_HEURISTIC_NONE
, ""},
23 { BUNDLE_HEURISTIC_CREATIONTOKEN
, "creationToken" },
26 static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED
,
27 const struct hashmap_entry
*he1
,
28 const struct hashmap_entry
*he2
,
31 const struct remote_bundle_info
*e1
=
32 container_of(he1
, const struct remote_bundle_info
, ent
);
33 const struct remote_bundle_info
*e2
=
34 container_of(he2
, const struct remote_bundle_info
, ent
);
36 return strcmp(e1
->id
, id
? (const char *)id
: e2
->id
);
39 void init_bundle_list(struct bundle_list
*list
)
41 memset(list
, 0, sizeof(*list
));
43 /* Implied defaults. */
44 list
->mode
= BUNDLE_MODE_ALL
;
47 hashmap_init(&list
->bundles
, compare_bundles
, NULL
, 0);
50 static int clear_remote_bundle_info(struct remote_bundle_info
*bundle
,
53 FREE_AND_NULL(bundle
->id
);
54 FREE_AND_NULL(bundle
->uri
);
55 FREE_AND_NULL(bundle
->file
);
56 bundle
->unbundled
= 0;
60 void clear_bundle_list(struct bundle_list
*list
)
65 for_all_bundles_in_list(list
, clear_remote_bundle_info
, NULL
);
66 hashmap_clear_and_free(&list
->bundles
, struct remote_bundle_info
, ent
);
70 int for_all_bundles_in_list(struct bundle_list
*list
,
74 struct remote_bundle_info
*info
;
75 struct hashmap_iter i
;
77 hashmap_for_each_entry(&list
->bundles
, &i
, info
, ent
) {
78 int result
= iter(info
, data
);
87 static int summarize_bundle(struct remote_bundle_info
*info
, void *data
)
90 fprintf(fp
, "[bundle \"%s\"]\n", info
->id
);
91 fprintf(fp
, "\turi = %s\n", info
->uri
);
93 if (info
->creationToken
)
94 fprintf(fp
, "\tcreationToken = %"PRIu64
"\n", info
->creationToken
);
98 void print_bundle_list(FILE *fp
, struct bundle_list
*list
)
102 switch (list
->mode
) {
103 case BUNDLE_MODE_ALL
:
107 case BUNDLE_MODE_ANY
:
111 case BUNDLE_MODE_NONE
:
116 fprintf(fp
, "[bundle]\n");
117 fprintf(fp
, "\tversion = %d\n", list
->version
);
118 fprintf(fp
, "\tmode = %s\n", mode
);
120 if (list
->heuristic
) {
122 for (i
= 0; i
< BUNDLE_HEURISTIC__COUNT
; i
++) {
123 if (heuristics
[i
].heuristic
== list
->heuristic
) {
124 printf("\theuristic = %s\n",
125 heuristics
[list
->heuristic
].name
);
131 for_all_bundles_in_list(list
, summarize_bundle
, fp
);
135 * Given a key-value pair, update the state of the given bundle list.
136 * Returns 0 if the key-value pair is understood. Returns -1 if the key
137 * is not understood or the value is malformed.
139 static int bundle_list_update(const char *key
, const char *value
,
140 struct bundle_list
*list
)
142 struct strbuf id
= STRBUF_INIT
;
143 struct remote_bundle_info lookup
= REMOTE_BUNDLE_INFO_INIT
;
144 struct remote_bundle_info
*bundle
;
145 const char *subsection
, *subkey
;
146 size_t subsection_len
;
148 if (parse_config_key(key
, "bundle", &subsection
, &subsection_len
, &subkey
))
151 if (!subsection_len
) {
152 if (!strcmp(subkey
, "version")) {
154 if (!git_parse_int(value
, &version
))
159 list
->version
= version
;
163 if (!strcmp(subkey
, "mode")) {
164 if (!strcmp(value
, "all"))
165 list
->mode
= BUNDLE_MODE_ALL
;
166 else if (!strcmp(value
, "any"))
167 list
->mode
= BUNDLE_MODE_ANY
;
173 if (!strcmp(subkey
, "heuristic")) {
175 for (i
= 0; i
< BUNDLE_HEURISTIC__COUNT
; i
++) {
176 if (heuristics
[i
].heuristic
&&
177 heuristics
[i
].name
&&
178 !strcmp(value
, heuristics
[i
].name
)) {
179 list
->heuristic
= heuristics
[i
].heuristic
;
184 /* Ignore unknown heuristics. */
188 /* Ignore other unknown global keys. */
192 strbuf_add(&id
, subsection
, subsection_len
);
195 * Check for an existing bundle with this <id>, or create one
199 hashmap_entry_init(&lookup
.ent
, strhash(lookup
.id
));
200 if (!(bundle
= hashmap_get_entry(&list
->bundles
, &lookup
, ent
, NULL
))) {
201 CALLOC_ARRAY(bundle
, 1);
202 bundle
->id
= strbuf_detach(&id
, NULL
);
203 hashmap_entry_init(&bundle
->ent
, strhash(bundle
->id
));
204 hashmap_add(&list
->bundles
, &bundle
->ent
);
208 if (!strcmp(subkey
, "uri")) {
211 bundle
->uri
= relative_url(list
->baseURI
, value
, NULL
);
215 if (!strcmp(subkey
, "creationtoken")) {
216 if (sscanf(value
, "%"PRIu64
, &bundle
->creationToken
) != 1)
217 warning(_("could not parse bundle list key %s with value '%s'"),
218 "creationToken", value
);
223 * At this point, we ignore any information that we don't
224 * understand, assuming it to be hints for a heuristic the client
225 * does not currently understand.
230 static int config_to_bundle_list(const char *key
, const char *value
,
231 const struct config_context
*ctx UNUSED
,
234 struct bundle_list
*list
= data
;
235 return bundle_list_update(key
, value
, list
);
238 int bundle_uri_parse_config_format(const char *uri
,
239 const char *filename
,
240 struct bundle_list
*list
)
243 struct config_options opts
= {
244 .error_action
= CONFIG_ERROR_ERROR
,
247 if (!list
->baseURI
) {
248 struct strbuf baseURI
= STRBUF_INIT
;
249 strbuf_addstr(&baseURI
, uri
);
252 * If the URI does not end with a trailing slash, then
253 * remove the filename portion of the path. This is
254 * important for relative URIs.
256 strbuf_strip_file_from_path(&baseURI
);
257 list
->baseURI
= strbuf_detach(&baseURI
, NULL
);
259 result
= git_config_from_file_with_options(config_to_bundle_list
,
261 CONFIG_SCOPE_UNKNOWN
,
264 if (!result
&& list
->mode
== BUNDLE_MODE_NONE
) {
265 warning(_("bundle list at '%s' has no mode"), uri
);
272 static char *find_temp_filename(void)
275 struct strbuf name
= STRBUF_INIT
;
277 * Find a temporary filename that is available. This is briefly
278 * racy, but unlikely to collide.
280 fd
= odb_mkstemp(&name
, "bundles/tmp_uri_XXXXXX");
282 warning(_("failed to create temporary file"));
288 return strbuf_detach(&name
, NULL
);
291 static int download_https_uri_to_file(const char *file
, const char *uri
)
294 struct child_process cp
= CHILD_PROCESS_INIT
;
295 FILE *child_in
= NULL
, *child_out
= NULL
;
296 struct strbuf line
= STRBUF_INIT
;
299 strvec_pushl(&cp
.args
, "git-remote-https", uri
, NULL
);
304 if (start_command(&cp
))
307 child_in
= fdopen(cp
.in
, "w");
313 child_out
= fdopen(cp
.out
, "r");
319 fprintf(child_in
, "capabilities\n");
322 while (!strbuf_getline(&line
, child_out
)) {
325 if (!strcmp(line
.buf
, "get"))
328 strbuf_release(&line
);
331 result
= error(_("insufficient capabilities"));
335 fprintf(child_in
, "get %s %s\n\n", uri
, file
);
340 if (finish_command(&cp
))
347 static int copy_uri_to_file(const char *filename
, const char *uri
)
351 if (starts_with(uri
, "https:") ||
352 starts_with(uri
, "http:"))
353 return download_https_uri_to_file(filename
, uri
);
355 if (skip_prefix(uri
, "file://", &out
))
359 return copy_file(filename
, uri
, 0);
362 static int unbundle_from_file(struct repository
*r
, const char *file
)
366 struct bundle_header header
= BUNDLE_HEADER_INIT
;
367 struct string_list_item
*refname
;
368 struct strbuf bundle_ref
= STRBUF_INIT
;
369 size_t bundle_prefix_len
;
371 bundle_fd
= read_bundle_header(file
, &header
);
378 * Skip the reachability walk here, since we will be adding
379 * a reachable ref pointing to the new tips, which will reach
380 * the prerequisite commits.
382 result
= unbundle(r
, &header
, bundle_fd
, NULL
,
383 VERIFY_BUNDLE_QUIET
| (fetch_pack_fsck_objects() ? VERIFY_BUNDLE_FSCK
: 0));
390 * Convert all refs/heads/ from the bundle into refs/bundles/
391 * in the local repository.
393 strbuf_addstr(&bundle_ref
, "refs/bundles/");
394 bundle_prefix_len
= bundle_ref
.len
;
396 for_each_string_list_item(refname
, &header
.references
) {
397 struct object_id
*oid
= refname
->util
;
398 struct object_id old_oid
;
399 const char *branch_name
;
402 if (!skip_prefix(refname
->string
, "refs/heads/", &branch_name
))
405 strbuf_setlen(&bundle_ref
, bundle_prefix_len
);
406 strbuf_addstr(&bundle_ref
, branch_name
);
408 has_old
= !refs_read_ref(get_main_ref_store(the_repository
),
409 bundle_ref
.buf
, &old_oid
);
410 refs_update_ref(get_main_ref_store(the_repository
),
411 "fetched bundle", bundle_ref
.buf
, oid
,
412 has_old
? &old_oid
: NULL
,
413 0, UPDATE_REFS_MSG_ON_ERR
);
417 strbuf_release(&bundle_ref
);
418 bundle_header_release(&header
);
422 struct bundle_list_context
{
423 struct repository
*r
;
424 struct bundle_list
*list
;
425 enum bundle_list_mode mode
;
431 * This early definition is necessary because we use indirect recursion:
433 * While iterating through a bundle list that was downloaded as part
434 * of fetch_bundle_uri_internal(), iterator methods eventually call it
435 * again, but with depth + 1.
437 static int fetch_bundle_uri_internal(struct repository
*r
,
438 struct remote_bundle_info
*bundle
,
440 struct bundle_list
*list
);
442 static int download_bundle_to_file(struct remote_bundle_info
*bundle
, void *data
)
445 struct bundle_list_context
*ctx
= data
;
447 if (ctx
->mode
== BUNDLE_MODE_ANY
&& ctx
->count
)
450 res
= fetch_bundle_uri_internal(ctx
->r
, bundle
, ctx
->depth
+ 1, ctx
->list
);
453 * Only increment count if the download succeeded. If our mode is
454 * BUNDLE_MODE_ANY, then we will want to try other URIs in the
455 * list in case they work instead.
461 * To be opportunistic as possible, we continue iterating and
462 * download as many bundles as we can, so we can apply the ones
463 * that work, even in BUNDLE_MODE_ALL mode.
468 struct bundles_for_sorting
{
469 struct remote_bundle_info
**items
;
474 static int append_bundle(struct remote_bundle_info
*bundle
, void *data
)
476 struct bundles_for_sorting
*list
= data
;
477 list
->items
[list
->nr
++] = bundle
;
482 * For use in QSORT() to get a list sorted by creationToken
483 * in decreasing order.
485 static int compare_creation_token_decreasing(const void *va
, const void *vb
)
487 const struct remote_bundle_info
* const *a
= va
;
488 const struct remote_bundle_info
* const *b
= vb
;
490 if ((*a
)->creationToken
> (*b
)->creationToken
)
492 if ((*a
)->creationToken
< (*b
)->creationToken
)
497 static int fetch_bundles_by_token(struct repository
*r
,
498 struct bundle_list
*list
)
501 int move_direction
= 0;
502 const char *creationTokenStr
;
503 uint64_t maxCreationToken
= 0, newMaxCreationToken
= 0;
504 struct bundle_list_context ctx
= {
509 struct bundles_for_sorting bundles
= {
510 .alloc
= hashmap_get_size(&list
->bundles
),
513 ALLOC_ARRAY(bundles
.items
, bundles
.alloc
);
515 for_all_bundles_in_list(list
, append_bundle
, &bundles
);
522 QSORT(bundles
.items
, bundles
.nr
, compare_creation_token_decreasing
);
525 * If fetch.bundleCreationToken exists, parses to a uint64t, and
526 * is not strictly smaller than the maximum creation token in the
527 * bundle list, then do not download any bundles.
529 if (!repo_config_get_value(r
,
530 "fetch.bundlecreationtoken",
531 &creationTokenStr
) &&
532 sscanf(creationTokenStr
, "%"PRIu64
, &maxCreationToken
) == 1 &&
533 bundles
.items
[0]->creationToken
<= maxCreationToken
) {
539 * Attempt to download and unbundle the minimum number of bundles by
540 * creationToken in decreasing order. If we fail to unbundle (after
541 * a successful download) then move to the next non-downloaded bundle
542 * and attempt downloading. Once we succeed in applying a bundle,
543 * move to the previous unapplied bundle and attempt to unbundle it
546 * In the case of a fresh clone, we will likely download all of the
547 * bundles before successfully unbundling the oldest one, then the
548 * rest of the bundles unbundle successfully in increasing order
551 * If there are existing objects, then this process may terminate
552 * early when all required commits from "new" bundles exist in the
553 * repo's object store.
556 while (cur
>= 0 && cur
< bundles
.nr
) {
557 struct remote_bundle_info
*bundle
= bundles
.items
[cur
];
560 * If we need to dig into bundles below the previous
561 * creation token value, then likely we are in an erroneous
562 * state due to missing or invalid bundles. Halt the process
563 * instead of continuing to download extra data.
565 if (bundle
->creationToken
<= maxCreationToken
)
570 * Not downloaded yet. Try downloading.
572 * Note that bundle->file is non-NULL if a download
573 * was attempted, even if it failed to download.
575 if (fetch_bundle_uri_internal(ctx
.r
, bundle
, ctx
.depth
+ 1, ctx
.list
)) {
576 /* Mark as unbundled so we do not retry. */
577 bundle
->unbundled
= 1;
579 /* Try looking deeper in the list. */
584 /* We expect bundles when using creationTokens. */
585 if (!is_bundle(bundle
->file
, 1)) {
586 warning(_("file downloaded from '%s' is not a bundle"),
592 if (bundle
->file
&& !bundle
->unbundled
) {
594 * This was downloaded, but not successfully
595 * unbundled. Try unbundling again.
597 if (unbundle_from_file(ctx
.r
, bundle
->file
)) {
598 /* Try looking deeper in the list. */
602 * Succeeded in unbundle. Retry bundles
603 * that previously failed to unbundle.
606 bundle
->unbundled
= 1;
608 if (bundle
->creationToken
> newMaxCreationToken
)
609 newMaxCreationToken
= bundle
->creationToken
;
614 * Else case: downloaded and unbundled successfully.
615 * Skip this by moving in the same direction as the
620 /* Move in the specified direction and repeat. */
621 cur
+= move_direction
;
625 * We succeed if the loop terminates because 'cur' drops below
626 * zero. The other case is that we terminate because 'cur'
627 * reaches the end of the list, so we have a failure no matter
628 * which bundles we apply from the list.
631 struct strbuf value
= STRBUF_INIT
;
632 strbuf_addf(&value
, "%"PRIu64
"", newMaxCreationToken
);
633 if (repo_config_set_multivar_gently(ctx
.r
,
634 "fetch.bundleCreationToken",
636 warning(_("failed to store maximum creation token"));
638 strbuf_release(&value
);
645 static int download_bundle_list(struct repository
*r
,
646 struct bundle_list
*local_list
,
647 struct bundle_list
*global_list
,
650 struct bundle_list_context ctx
= {
654 .mode
= local_list
->mode
,
657 return for_all_bundles_in_list(local_list
, download_bundle_to_file
, &ctx
);
660 static int fetch_bundle_list_in_config_format(struct repository
*r
,
661 struct bundle_list
*global_list
,
662 struct remote_bundle_info
*bundle
,
666 struct bundle_list list_from_bundle
;
668 init_bundle_list(&list_from_bundle
);
670 if ((result
= bundle_uri_parse_config_format(bundle
->uri
,
675 if (list_from_bundle
.mode
== BUNDLE_MODE_NONE
) {
676 warning(_("unrecognized bundle mode from URI '%s'"),
683 * If this list uses the creationToken heuristic, then the URIs
684 * it advertises are expected to be bundles, not nested lists.
685 * We can drop 'global_list' and 'depth'.
687 if (list_from_bundle
.heuristic
== BUNDLE_HEURISTIC_CREATIONTOKEN
) {
688 result
= fetch_bundles_by_token(r
, &list_from_bundle
);
689 global_list
->heuristic
= BUNDLE_HEURISTIC_CREATIONTOKEN
;
690 } else if ((result
= download_bundle_list(r
, &list_from_bundle
,
691 global_list
, depth
)))
695 clear_bundle_list(&list_from_bundle
);
700 * This limits the recursion on fetch_bundle_uri_internal() when following
703 static int max_bundle_uri_depth
= 4;
706 * Recursively download all bundles advertised at the given URI
707 * to files. If the file is a bundle, then add it to the given
708 * 'list'. Otherwise, expect a bundle list and recurse on the
709 * URIs in that list according to the list mode (ANY or ALL).
711 static int fetch_bundle_uri_internal(struct repository
*r
,
712 struct remote_bundle_info
*bundle
,
714 struct bundle_list
*list
)
717 struct remote_bundle_info
*bcopy
;
719 if (depth
>= max_bundle_uri_depth
) {
720 warning(_("exceeded bundle URI recursion limit (%d)"),
721 max_bundle_uri_depth
);
726 !(bundle
->file
= find_temp_filename())) {
731 if ((result
= copy_uri_to_file(bundle
->file
, bundle
->uri
))) {
732 warning(_("failed to download bundle from URI '%s'"), bundle
->uri
);
736 if ((result
= !is_bundle(bundle
->file
, 1))) {
737 result
= fetch_bundle_list_in_config_format(
738 r
, list
, bundle
, depth
);
740 warning(_("file at URI '%s' is not a bundle or bundle list"),
745 /* Copy the bundle and insert it into the global list. */
746 CALLOC_ARRAY(bcopy
, 1);
747 bcopy
->id
= xstrdup(bundle
->id
);
748 bcopy
->file
= xstrdup(bundle
->file
);
749 hashmap_entry_init(&bcopy
->ent
, strhash(bcopy
->id
));
750 hashmap_add(&list
->bundles
, &bcopy
->ent
);
753 if (result
&& bundle
->file
)
754 unlink(bundle
->file
);
759 * This loop iterator breaks the loop with nonzero return code on the
760 * first successful unbundling of a bundle.
762 static int attempt_unbundle(struct remote_bundle_info
*info
, void *data
)
764 struct repository
*r
= data
;
766 if (!info
->file
|| info
->unbundled
)
769 if (!unbundle_from_file(r
, info
->file
)) {
777 static int unbundle_all_bundles(struct repository
*r
,
778 struct bundle_list
*list
)
781 * Iterate through all bundles looking for ones that can
782 * successfully unbundle. If any succeed, then perhaps another
783 * will succeed in the next attempt.
785 * Keep in mind that a non-zero result for the loop here means
786 * the loop terminated early on a successful unbundling, which
787 * signals that we can try again.
789 while (for_all_bundles_in_list(list
, attempt_unbundle
, r
)) ;
794 static int unlink_bundle(struct remote_bundle_info
*info
, void *data UNUSED
)
797 unlink_or_warn(info
->file
);
801 int fetch_bundle_uri(struct repository
*r
, const char *uri
,
805 struct bundle_list list
;
806 struct remote_bundle_info bundle
= {
811 trace2_region_enter("fetch", "fetch-bundle-uri", the_repository
);
813 init_bundle_list(&list
);
816 * Do not fetch an empty bundle URI. An empty bundle URI
817 * could signal that a configured bundle URI has been disabled.
824 /* If a bundle is added to this global list, then it is required. */
825 list
.mode
= BUNDLE_MODE_ALL
;
827 if ((result
= fetch_bundle_uri_internal(r
, &bundle
, 0, &list
)))
830 result
= unbundle_all_bundles(r
, &list
);
834 *has_heuristic
= (list
.heuristic
!= BUNDLE_HEURISTIC_NONE
);
835 for_all_bundles_in_list(&list
, unlink_bundle
, NULL
);
836 clear_bundle_list(&list
);
837 clear_remote_bundle_info(&bundle
, NULL
);
838 trace2_region_leave("fetch", "fetch-bundle-uri", the_repository
);
842 int fetch_bundle_list(struct repository
*r
, struct bundle_list
*list
)
845 struct bundle_list global_list
;
848 * If the creationToken heuristic is used, then the URIs
849 * advertised by 'list' are not nested lists and instead
850 * direct bundles. We do not need to use global_list.
852 if (list
->heuristic
== BUNDLE_HEURISTIC_CREATIONTOKEN
)
853 return fetch_bundles_by_token(r
, list
);
855 init_bundle_list(&global_list
);
857 /* If a bundle is added to this global list, then it is required. */
858 global_list
.mode
= BUNDLE_MODE_ALL
;
860 if ((result
= download_bundle_list(r
, list
, &global_list
, 0)))
863 if (list
->heuristic
== BUNDLE_HEURISTIC_CREATIONTOKEN
)
864 result
= fetch_bundles_by_token(r
, list
);
866 result
= unbundle_all_bundles(r
, &global_list
);
869 for_all_bundles_in_list(&global_list
, unlink_bundle
, NULL
);
870 clear_bundle_list(&global_list
);
878 int bundle_uri_advertise(struct repository
*r
, struct strbuf
*value UNUSED
)
880 static int advertise_bundle_uri
= -1;
882 if (advertise_bundle_uri
!= -1)
885 advertise_bundle_uri
= 0;
886 repo_config_get_maybe_bool(r
, "uploadpack.advertisebundleuris", &advertise_bundle_uri
);
889 return advertise_bundle_uri
;
892 static int config_to_packet_line(const char *key
, const char *value
,
893 const struct config_context
*ctx UNUSED
,
896 struct packet_reader
*writer
= data
;
898 if (starts_with(key
, "bundle."))
899 packet_write_fmt(writer
->fd
, "%s=%s", key
, value
);
904 int bundle_uri_command(struct repository
*r
,
905 struct packet_reader
*request
)
907 struct packet_writer writer
;
908 packet_writer_init(&writer
, 1);
910 while (packet_reader_read(request
) == PACKET_READ_NORMAL
)
911 die(_("bundle-uri: unexpected argument: '%s'"), request
->line
);
912 if (request
->status
!= PACKET_READ_FLUSH
)
913 die(_("bundle-uri: expected flush after arguments"));
916 * Read all "bundle.*" config lines to the client as key=value
919 repo_config(r
, config_to_packet_line
, &writer
);
921 packet_writer_flush(&writer
);
927 * General API for {transport,connect}.c etc.
929 int bundle_uri_parse_line(struct bundle_list
*list
, const char *line
)
933 struct strbuf key
= STRBUF_INIT
;
936 return error(_("bundle-uri: got an empty line"));
938 equals
= strchr(line
, '=');
941 return error(_("bundle-uri: line is not of the form 'key=value'"));
942 if (line
== equals
|| !*(equals
+ 1))
943 return error(_("bundle-uri: line has empty key or value"));
945 strbuf_add(&key
, line
, equals
- line
);
946 result
= bundle_list_update(key
.buf
, equals
+ 1, list
);
947 strbuf_release(&key
);