add storage size output to 'git verify-pack -v'
[git/spearce.git] / builtin-fetch-pack.c
blob5ea48ca7dbc22785f4e4f3c35f3dc48ee18f0d8a
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 parse_commit(commit);
46 insert_by_date(commit, &rev_list);
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
60 return 0;
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
75 if (!ancestors_only)
76 o->flags |= COMMON;
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
97 Get the next rev to send, ignoring the common.
100 static const unsigned char* get_rev(void)
102 struct commit *commit = NULL;
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
118 parents = commit->parents;
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
139 rev_list = rev_list->next;
142 return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
154 for_each_ref(rev_list_insert_ref, NULL);
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (args.use_thin_pack ? " thin-pack" : ""),
183 (args.no_progress ? " no-progress" : ""),
184 " ofs-delta");
185 else
186 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187 fetching++;
189 if (is_repository_shallow())
190 write_shallow_commits(fd[1], 1);
191 if (args.depth > 0)
192 packet_write(fd[1], "deepen %d", args.depth);
193 packet_flush(fd[1]);
194 if (!fetching)
195 return 1;
197 if (args.depth > 0) {
198 char line[1024];
199 unsigned char sha1[20];
200 int len;
202 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203 if (!prefixcmp(line, "shallow ")) {
204 if (get_sha1_hex(line + 8, sha1))
205 die("invalid shallow line: %s", line);
206 register_shallow(sha1);
207 continue;
209 if (!prefixcmp(line, "unshallow ")) {
210 if (get_sha1_hex(line + 10, sha1))
211 die("invalid unshallow line: %s", line);
212 if (!lookup_object(sha1))
213 die("object not found: %s", line);
214 /* make sure that it is parsed as shallow */
215 parse_object(sha1);
216 if (unregister_shallow(sha1))
217 die("no shallow found: %s", line);
218 continue;
220 die("expected shallow/unshallow, got %s", line);
224 flushes = 0;
225 retval = -1;
226 while ((sha1 = get_rev())) {
227 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228 if (args.verbose)
229 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230 in_vain++;
231 if (!(31 & ++count)) {
232 int ack;
234 packet_flush(fd[1]);
235 flushes++;
238 * We keep one window "ahead" of the other side, and
239 * will wait for an ACK only on the next one
241 if (count == 32)
242 continue;
244 do {
245 ack = get_ack(fd[0], result_sha1);
246 if (args.verbose && ack)
247 fprintf(stderr, "got ack %d %s\n", ack,
248 sha1_to_hex(result_sha1));
249 if (ack == 1) {
250 flushes = 0;
251 multi_ack = 0;
252 retval = 0;
253 goto done;
254 } else if (ack == 2) {
255 struct commit *commit =
256 lookup_commit(result_sha1);
257 mark_common(commit, 0, 1);
258 retval = 0;
259 in_vain = 0;
260 got_continue = 1;
262 } while (ack);
263 flushes--;
264 if (got_continue && MAX_IN_VAIN < in_vain) {
265 if (args.verbose)
266 fprintf(stderr, "giving up\n");
267 break; /* give up */
271 done:
272 packet_write(fd[1], "done\n");
273 if (args.verbose)
274 fprintf(stderr, "done\n");
275 if (retval != 0) {
276 multi_ack = 0;
277 flushes++;
279 while (flushes || multi_ack) {
280 int ack = get_ack(fd[0], result_sha1);
281 if (ack) {
282 if (args.verbose)
283 fprintf(stderr, "got ack (%d) %s\n", ack,
284 sha1_to_hex(result_sha1));
285 if (ack == 1)
286 return 0;
287 multi_ack = 1;
288 continue;
290 flushes--;
292 return retval;
295 static struct commit_list *complete;
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
299 struct object *o = parse_object(sha1);
301 while (o && o->type == OBJ_TAG) {
302 struct tag *t = (struct tag *) o;
303 if (!t->tagged)
304 break; /* broken repository */
305 o->flags |= COMPLETE;
306 o = parse_object(t->tagged->sha1);
308 if (o && o->type == OBJ_COMMIT) {
309 struct commit *commit = (struct commit *)o;
310 commit->object.flags |= COMPLETE;
311 insert_by_date(commit, &complete);
313 return 0;
316 static void mark_recent_complete_commits(unsigned long cutoff)
318 while (complete && cutoff <= complete->item->date) {
319 if (args.verbose)
320 fprintf(stderr, "Marking %s as complete\n",
321 sha1_to_hex(complete->item->object.sha1));
322 pop_most_recent_commit(&complete, COMPLETE);
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
328 struct ref **return_refs;
329 struct ref *newlist = NULL;
330 struct ref **newtail = &newlist;
331 struct ref *ref, *next;
332 struct ref *fastarray[32];
334 if (nr_match && !args.fetch_all) {
335 if (ARRAY_SIZE(fastarray) < nr_match)
336 return_refs = xcalloc(nr_match, sizeof(struct ref *));
337 else {
338 return_refs = fastarray;
339 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
342 else
343 return_refs = NULL;
345 for (ref = *refs; ref; ref = next) {
346 next = ref->next;
347 if (!memcmp(ref->name, "refs/", 5) &&
348 check_ref_format(ref->name + 5))
349 ; /* trash */
350 else if (args.fetch_all &&
351 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
352 *newtail = ref;
353 ref->next = NULL;
354 newtail = &ref->next;
355 continue;
357 else {
358 int order = path_match(ref->name, nr_match, match);
359 if (order) {
360 return_refs[order-1] = ref;
361 continue; /* we will link it later */
364 free(ref);
367 if (!args.fetch_all) {
368 int i;
369 for (i = 0; i < nr_match; i++) {
370 ref = return_refs[i];
371 if (ref) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
377 if (return_refs != fastarray)
378 free(return_refs);
380 *refs = newlist;
383 static int everything_local(struct ref **refs, int nr_match, char **match)
385 struct ref *ref;
386 int retval;
387 unsigned long cutoff = 0;
389 track_object_refs = 0;
390 save_commit_buffer = 0;
392 for (ref = *refs; ref; ref = ref->next) {
393 struct object *o;
395 o = parse_object(ref->old_sha1);
396 if (!o)
397 continue;
399 /* We already have it -- which may mean that we were
400 * in sync with the other side at some time after
401 * that (it is OK if we guess wrong here).
403 if (o->type == OBJ_COMMIT) {
404 struct commit *commit = (struct commit *)o;
405 if (!cutoff || cutoff < commit->date)
406 cutoff = commit->date;
410 if (!args.depth) {
411 for_each_ref(mark_complete, NULL);
412 if (cutoff)
413 mark_recent_complete_commits(cutoff);
417 * Mark all complete remote refs as common refs.
418 * Don't mark them common yet; the server has to be told so first.
420 for (ref = *refs; ref; ref = ref->next) {
421 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422 NULL, 0);
424 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425 continue;
427 if (!(o->flags & SEEN)) {
428 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
430 mark_common((struct commit *)o, 1, 1);
434 filter_refs(refs, nr_match, match);
436 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437 const unsigned char *remote = ref->old_sha1;
438 unsigned char local[20];
439 struct object *o;
441 o = lookup_object(remote);
442 if (!o || !(o->flags & COMPLETE)) {
443 retval = 0;
444 if (!args.verbose)
445 continue;
446 fprintf(stderr,
447 "want %s (%s)\n", sha1_to_hex(remote),
448 ref->name);
449 continue;
452 hashcpy(ref->new_sha1, local);
453 if (!args.verbose)
454 continue;
455 fprintf(stderr,
456 "already have %s (%s)\n", sha1_to_hex(remote),
457 ref->name);
459 return retval;
462 static int sideband_demux(int fd, void *data)
464 int *xd = data;
466 return recv_sideband("fetch-pack", xd[0], fd, 2);
469 static int get_pack(int xd[2], char **pack_lockfile)
471 struct async demux;
472 const char *argv[20];
473 char keep_arg[256];
474 char hdr_arg[256];
475 const char **av;
476 int do_keep = args.keep_pack;
477 struct child_process cmd;
479 memset(&demux, 0, sizeof(demux));
480 if (use_sideband) {
481 /* xd[] is talking with upload-pack; subprocess reads from
482 * xd[0], spits out band#2 to stderr, and feeds us band#1
483 * through demux->out.
485 demux.proc = sideband_demux;
486 demux.data = xd;
487 if (start_async(&demux))
488 die("fetch-pack: unable to fork off sideband"
489 " demultiplexer");
491 else
492 demux.out = xd[0];
494 memset(&cmd, 0, sizeof(cmd));
495 cmd.argv = argv;
496 av = argv;
497 *hdr_arg = 0;
498 if (!args.keep_pack && unpack_limit) {
499 struct pack_header header;
501 if (read_pack_header(demux.out, &header))
502 die("protocol error: bad pack header");
503 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
504 ntohl(header.hdr_version), ntohl(header.hdr_entries));
505 if (ntohl(header.hdr_entries) < unpack_limit)
506 do_keep = 0;
507 else
508 do_keep = 1;
511 if (do_keep) {
512 if (pack_lockfile)
513 cmd.out = -1;
514 *av++ = "index-pack";
515 *av++ = "--stdin";
516 if (!args.quiet && !args.no_progress)
517 *av++ = "-v";
518 if (args.use_thin_pack)
519 *av++ = "--fix-thin";
520 if (args.lock_pack || unpack_limit) {
521 int s = sprintf(keep_arg,
522 "--keep=fetch-pack %d on ", getpid());
523 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
524 strcpy(keep_arg + s, "localhost");
525 *av++ = keep_arg;
528 else {
529 *av++ = "unpack-objects";
530 if (args.quiet)
531 *av++ = "-q";
533 if (*hdr_arg)
534 *av++ = hdr_arg;
535 *av++ = NULL;
537 cmd.in = demux.out;
538 cmd.git_cmd = 1;
539 if (start_command(&cmd))
540 die("fetch-pack: unable to fork off %s", argv[0]);
541 if (do_keep && pack_lockfile) {
542 *pack_lockfile = index_pack_lockfile(cmd.out);
543 close(cmd.out);
546 if (finish_command(&cmd))
547 die("%s failed", argv[0]);
548 if (use_sideband && finish_async(&demux))
549 die("error in sideband demultiplexer");
550 return 0;
553 static struct ref *do_fetch_pack(int fd[2],
554 const struct ref *orig_ref,
555 int nr_match,
556 char **match,
557 char **pack_lockfile)
559 struct ref *ref = copy_ref_list(orig_ref);
560 unsigned char sha1[20];
562 if (is_repository_shallow() && !server_supports("shallow"))
563 die("Server does not support shallow clients");
564 if (server_supports("multi_ack")) {
565 if (args.verbose)
566 fprintf(stderr, "Server supports multi_ack\n");
567 multi_ack = 1;
569 if (server_supports("side-band-64k")) {
570 if (args.verbose)
571 fprintf(stderr, "Server supports side-band-64k\n");
572 use_sideband = 2;
574 else if (server_supports("side-band")) {
575 if (args.verbose)
576 fprintf(stderr, "Server supports side-band\n");
577 use_sideband = 1;
579 if (everything_local(&ref, nr_match, match)) {
580 packet_flush(fd[1]);
581 goto all_done;
583 if (find_common(fd, sha1, ref) < 0)
584 if (!args.keep_pack)
585 /* When cloning, it is not unusual to have
586 * no common commit.
588 fprintf(stderr, "warning: no common commits\n");
590 if (get_pack(fd, pack_lockfile))
591 die("git-fetch-pack: fetch failed.");
593 all_done:
594 return ref;
597 static int remove_duplicates(int nr_heads, char **heads)
599 int src, dst;
601 for (src = dst = 0; src < nr_heads; src++) {
602 /* If heads[src] is different from any of
603 * heads[0..dst], push it in.
605 int i;
606 for (i = 0; i < dst; i++) {
607 if (!strcmp(heads[i], heads[src]))
608 break;
610 if (i < dst)
611 continue;
612 if (src != dst)
613 heads[dst] = heads[src];
614 dst++;
616 return dst;
619 static int fetch_pack_config(const char *var, const char *value)
621 if (strcmp(var, "fetch.unpacklimit") == 0) {
622 fetch_unpack_limit = git_config_int(var, value);
623 return 0;
626 if (strcmp(var, "transfer.unpacklimit") == 0) {
627 transfer_unpack_limit = git_config_int(var, value);
628 return 0;
631 return git_default_config(var, value);
634 static struct lock_file lock;
636 static void fetch_pack_setup(void)
638 static int did_setup;
639 if (did_setup)
640 return;
641 git_config(fetch_pack_config);
642 if (0 <= transfer_unpack_limit)
643 unpack_limit = transfer_unpack_limit;
644 else if (0 <= fetch_unpack_limit)
645 unpack_limit = fetch_unpack_limit;
646 did_setup = 1;
649 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
651 int i, ret, nr_heads;
652 struct ref *ref = NULL;
653 char *dest = NULL, **heads;
654 int fd[2];
655 struct child_process *conn;
657 nr_heads = 0;
658 heads = NULL;
659 for (i = 1; i < argc; i++) {
660 const char *arg = argv[i];
662 if (*arg == '-') {
663 if (!prefixcmp(arg, "--upload-pack=")) {
664 args.uploadpack = arg + 14;
665 continue;
667 if (!prefixcmp(arg, "--exec=")) {
668 args.uploadpack = arg + 7;
669 continue;
671 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
672 args.quiet = 1;
673 continue;
675 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
676 args.lock_pack = args.keep_pack;
677 args.keep_pack = 1;
678 continue;
680 if (!strcmp("--thin", arg)) {
681 args.use_thin_pack = 1;
682 continue;
684 if (!strcmp("--all", arg)) {
685 args.fetch_all = 1;
686 continue;
688 if (!strcmp("-v", arg)) {
689 args.verbose = 1;
690 continue;
692 if (!prefixcmp(arg, "--depth=")) {
693 args.depth = strtol(arg + 8, NULL, 0);
694 continue;
696 if (!strcmp("--no-progress", arg)) {
697 args.no_progress = 1;
698 continue;
700 usage(fetch_pack_usage);
702 dest = (char *)arg;
703 heads = (char **)(argv + i + 1);
704 nr_heads = argc - i - 1;
705 break;
707 if (!dest)
708 usage(fetch_pack_usage);
710 conn = git_connect(fd, (char *)dest, args.uploadpack,
711 args.verbose ? CONNECT_VERBOSE : 0);
712 if (conn) {
713 get_remote_heads(fd[0], &ref, 0, NULL, 0);
715 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
716 close(fd[0]);
717 close(fd[1]);
718 if (finish_connect(conn))
719 ref = NULL;
720 } else {
721 ref = NULL;
723 ret = !ref;
725 if (!ret && nr_heads) {
726 /* If the heads to pull were given, we should have
727 * consumed all of them by matching the remote.
728 * Otherwise, 'git-fetch remote no-such-ref' would
729 * silently succeed without issuing an error.
731 for (i = 0; i < nr_heads; i++)
732 if (heads[i] && heads[i][0]) {
733 error("no such remote ref %s", heads[i]);
734 ret = 1;
737 while (ref) {
738 printf("%s %s\n",
739 sha1_to_hex(ref->old_sha1), ref->name);
740 ref = ref->next;
743 return ret;
746 struct ref *fetch_pack(struct fetch_pack_args *my_args,
747 int fd[], struct child_process *conn,
748 const struct ref *ref,
749 const char *dest,
750 int nr_heads,
751 char **heads,
752 char **pack_lockfile)
754 struct stat st;
755 struct ref *ref_cpy;
757 fetch_pack_setup();
758 memcpy(&args, my_args, sizeof(args));
759 if (args.depth > 0) {
760 if (stat(git_path("shallow"), &st))
761 st.st_mtime = 0;
764 if (heads && nr_heads)
765 nr_heads = remove_duplicates(nr_heads, heads);
766 if (!ref) {
767 packet_flush(fd[1]);
768 die("no matching remote head");
770 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
772 if (args.depth > 0) {
773 struct cache_time mtime;
774 char *shallow = git_path("shallow");
775 int fd;
777 mtime.sec = st.st_mtime;
778 #ifdef USE_NSEC
779 mtime.usec = st.st_mtim.usec;
780 #endif
781 if (stat(shallow, &st)) {
782 if (mtime.sec)
783 die("shallow file was removed during fetch");
784 } else if (st.st_mtime != mtime.sec
785 #ifdef USE_NSEC
786 || st.st_mtim.usec != mtime.usec
787 #endif
789 die("shallow file was changed during fetch");
791 fd = hold_lock_file_for_update(&lock, shallow, 1);
792 if (!write_shallow_commits(fd, 0)) {
793 unlink(shallow);
794 rollback_lock_file(&lock);
795 } else {
796 commit_lock_file(&lock);
800 return ref_cpy;