Merge branch 'pu' of git://repo.or.cz/git-gui into devel
[git/platforms.git] / fetch-pack.c
blobff79e56ac4cd1a85979f026e0f70af5a84ba12d9
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int transfer_unpack_limit = -1;
12 static int fetch_unpack_limit = -1;
13 static int unpack_limit = 100;
14 static int quiet;
15 static int verbose;
16 static int fetch_all;
17 static int depth;
18 static int no_progress;
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
21 static const char *uploadpack = "git-upload-pack";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 parse_commit(commit);
46 insert_by_date(commit, &rev_list);
48 if (!(commit->object.flags & COMMON))
49 non_common_revs++;
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55 struct object *o = deref_tag(parse_object(sha1), path, 0);
57 if (o && o->type == OBJ_COMMIT)
58 rev_list_push((struct commit *)o, SEEN);
60 return 0;
64 This function marks a rev and its ancestors as common.
65 In some cases, it is desirable to mark only the ancestors (for example
66 when only the server does not yet know that they are common).
69 static void mark_common(struct commit *commit,
70 int ancestors_only, int dont_parse)
72 if (commit != NULL && !(commit->object.flags & COMMON)) {
73 struct object *o = (struct object *)commit;
75 if (!ancestors_only)
76 o->flags |= COMMON;
78 if (!(o->flags & SEEN))
79 rev_list_push(commit, SEEN);
80 else {
81 struct commit_list *parents;
83 if (!ancestors_only && !(o->flags & POPPED))
84 non_common_revs--;
85 if (!o->parsed && !dont_parse)
86 parse_commit(commit);
88 for (parents = commit->parents;
89 parents;
90 parents = parents->next)
91 mark_common(parents->item, 0, dont_parse);
97 Get the next rev to send, ignoring the common.
100 static const unsigned char* get_rev(void)
102 struct commit *commit = NULL;
104 while (commit == NULL) {
105 unsigned int mark;
106 struct commit_list* parents;
108 if (rev_list == NULL || non_common_revs == 0)
109 return NULL;
111 commit = rev_list->item;
112 if (!(commit->object.parsed))
113 parse_commit(commit);
114 commit->object.flags |= POPPED;
115 if (!(commit->object.flags & COMMON))
116 non_common_revs--;
118 parents = commit->parents;
120 if (commit->object.flags & COMMON) {
121 /* do not send "have", and ignore ancestors */
122 commit = NULL;
123 mark = COMMON | SEEN;
124 } else if (commit->object.flags & COMMON_REF)
125 /* send "have", and ignore ancestors */
126 mark = COMMON | SEEN;
127 else
128 /* send "have", also for its ancestors */
129 mark = SEEN;
131 while (parents) {
132 if (!(parents->item->object.flags & SEEN))
133 rev_list_push(parents->item, mark);
134 if (mark & COMMON)
135 mark_common(parents->item, 1, 0);
136 parents = parents->next;
139 rev_list = rev_list->next;
142 return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146 struct ref *refs)
148 int fetching;
149 int count = 0, flushes = 0, retval;
150 const unsigned char *sha1;
151 unsigned in_vain = 0;
152 int got_continue = 0;
154 for_each_ref(rev_list_insert_ref, NULL);
156 fetching = 0;
157 for ( ; refs ; refs = refs->next) {
158 unsigned char *remote = refs->old_sha1;
159 struct object *o;
162 * If that object is complete (i.e. it is an ancestor of a
163 * local ref), we tell them we have it but do not have to
164 * tell them about its ancestors, which they already know
165 * about.
167 * We use lookup_object here because we are only
168 * interested in the case we *know* the object is
169 * reachable and we have already scanned it.
171 if (((o = lookup_object(remote)) != NULL) &&
172 (o->flags & COMPLETE)) {
173 continue;
176 if (!fetching)
177 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178 sha1_to_hex(remote),
179 (multi_ack ? " multi_ack" : ""),
180 (use_sideband == 2 ? " side-band-64k" : ""),
181 (use_sideband == 1 ? " side-band" : ""),
182 (use_thin_pack ? " thin-pack" : ""),
183 (no_progress ? " no-progress" : ""),
184 " ofs-delta");
185 else
186 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187 fetching++;
189 if (is_repository_shallow())
190 write_shallow_commits(fd[1], 1);
191 if (depth > 0)
192 packet_write(fd[1], "deepen %d", depth);
193 packet_flush(fd[1]);
194 if (!fetching)
195 return 1;
197 if (depth > 0) {
198 char line[1024];
199 unsigned char sha1[20];
200 int len;
202 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203 if (!prefixcmp(line, "shallow ")) {
204 if (get_sha1_hex(line + 8, sha1))
205 die("invalid shallow line: %s", line);
206 register_shallow(sha1);
207 continue;
209 if (!prefixcmp(line, "unshallow ")) {
210 if (get_sha1_hex(line + 10, sha1))
211 die("invalid unshallow line: %s", line);
212 if (!lookup_object(sha1))
213 die("object not found: %s", line);
214 /* make sure that it is parsed as shallow */
215 parse_object(sha1);
216 if (unregister_shallow(sha1))
217 die("no shallow found: %s", line);
218 continue;
220 die("expected shallow/unshallow, got %s", line);
224 flushes = 0;
225 retval = -1;
226 while ((sha1 = get_rev())) {
227 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228 if (verbose)
229 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230 in_vain++;
231 if (!(31 & ++count)) {
232 int ack;
234 packet_flush(fd[1]);
235 flushes++;
238 * We keep one window "ahead" of the other side, and
239 * will wait for an ACK only on the next one
241 if (count == 32)
242 continue;
244 do {
245 ack = get_ack(fd[0], result_sha1);
246 if (verbose && ack)
247 fprintf(stderr, "got ack %d %s\n", ack,
248 sha1_to_hex(result_sha1));
249 if (ack == 1) {
250 flushes = 0;
251 multi_ack = 0;
252 retval = 0;
253 goto done;
254 } else if (ack == 2) {
255 struct commit *commit =
256 lookup_commit(result_sha1);
257 mark_common(commit, 0, 1);
258 retval = 0;
259 in_vain = 0;
260 got_continue = 1;
262 } while (ack);
263 flushes--;
264 if (got_continue && MAX_IN_VAIN < in_vain) {
265 if (verbose)
266 fprintf(stderr, "giving up\n");
267 break; /* give up */
271 done:
272 packet_write(fd[1], "done\n");
273 if (verbose)
274 fprintf(stderr, "done\n");
275 if (retval != 0) {
276 multi_ack = 0;
277 flushes++;
279 while (flushes || multi_ack) {
280 int ack = get_ack(fd[0], result_sha1);
281 if (ack) {
282 if (verbose)
283 fprintf(stderr, "got ack (%d) %s\n", ack,
284 sha1_to_hex(result_sha1));
285 if (ack == 1)
286 return 0;
287 multi_ack = 1;
288 continue;
290 flushes--;
292 /* it is no error to fetch into a completely empty repo */
293 return count ? retval : 0;
296 static struct commit_list *complete;
298 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
300 struct object *o = parse_object(sha1);
302 while (o && o->type == OBJ_TAG) {
303 struct tag *t = (struct tag *) o;
304 if (!t->tagged)
305 break; /* broken repository */
306 o->flags |= COMPLETE;
307 o = parse_object(t->tagged->sha1);
309 if (o && o->type == OBJ_COMMIT) {
310 struct commit *commit = (struct commit *)o;
311 commit->object.flags |= COMPLETE;
312 insert_by_date(commit, &complete);
314 return 0;
317 static void mark_recent_complete_commits(unsigned long cutoff)
319 while (complete && cutoff <= complete->item->date) {
320 if (verbose)
321 fprintf(stderr, "Marking %s as complete\n",
322 sha1_to_hex(complete->item->object.sha1));
323 pop_most_recent_commit(&complete, COMPLETE);
327 static void filter_refs(struct ref **refs, int nr_match, char **match)
329 struct ref **return_refs;
330 struct ref *newlist = NULL;
331 struct ref **newtail = &newlist;
332 struct ref *ref, *next;
333 struct ref *fastarray[32];
335 if (nr_match && !fetch_all) {
336 if (ARRAY_SIZE(fastarray) < nr_match)
337 return_refs = xcalloc(nr_match, sizeof(struct ref *));
338 else {
339 return_refs = fastarray;
340 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
343 else
344 return_refs = NULL;
346 for (ref = *refs; ref; ref = next) {
347 next = ref->next;
348 if (!memcmp(ref->name, "refs/", 5) &&
349 check_ref_format(ref->name + 5))
350 ; /* trash */
351 else if (fetch_all &&
352 (!depth || prefixcmp(ref->name, "refs/tags/") )) {
353 *newtail = ref;
354 ref->next = NULL;
355 newtail = &ref->next;
356 continue;
358 else {
359 int order = path_match(ref->name, nr_match, match);
360 if (order) {
361 return_refs[order-1] = ref;
362 continue; /* we will link it later */
365 free(ref);
368 if (!fetch_all) {
369 int i;
370 for (i = 0; i < nr_match; i++) {
371 ref = return_refs[i];
372 if (ref) {
373 *newtail = ref;
374 ref->next = NULL;
375 newtail = &ref->next;
378 if (return_refs != fastarray)
379 free(return_refs);
381 *refs = newlist;
384 static int everything_local(struct ref **refs, int nr_match, char **match)
386 struct ref *ref;
387 int retval;
388 unsigned long cutoff = 0;
390 track_object_refs = 0;
391 save_commit_buffer = 0;
393 for (ref = *refs; ref; ref = ref->next) {
394 struct object *o;
396 o = parse_object(ref->old_sha1);
397 if (!o)
398 continue;
400 /* We already have it -- which may mean that we were
401 * in sync with the other side at some time after
402 * that (it is OK if we guess wrong here).
404 if (o->type == OBJ_COMMIT) {
405 struct commit *commit = (struct commit *)o;
406 if (!cutoff || cutoff < commit->date)
407 cutoff = commit->date;
411 if (!depth) {
412 for_each_ref(mark_complete, NULL);
413 if (cutoff)
414 mark_recent_complete_commits(cutoff);
418 * Mark all complete remote refs as common refs.
419 * Don't mark them common yet; the server has to be told so first.
421 for (ref = *refs; ref; ref = ref->next) {
422 struct object *o = deref_tag(lookup_object(ref->old_sha1),
423 NULL, 0);
425 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
426 continue;
428 if (!(o->flags & SEEN)) {
429 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
431 mark_common((struct commit *)o, 1, 1);
435 filter_refs(refs, nr_match, match);
437 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
438 const unsigned char *remote = ref->old_sha1;
439 unsigned char local[20];
440 struct object *o;
442 o = lookup_object(remote);
443 if (!o || !(o->flags & COMPLETE)) {
444 retval = 0;
445 if (!verbose)
446 continue;
447 fprintf(stderr,
448 "want %s (%s)\n", sha1_to_hex(remote),
449 ref->name);
450 continue;
453 hashcpy(ref->new_sha1, local);
454 if (!verbose)
455 continue;
456 fprintf(stderr,
457 "already have %s (%s)\n", sha1_to_hex(remote),
458 ref->name);
460 return retval;
463 static pid_t setup_sideband(int fd[2], int xd[2])
465 pid_t side_pid;
467 if (!use_sideband) {
468 fd[0] = xd[0];
469 fd[1] = xd[1];
470 return 0;
472 /* xd[] is talking with upload-pack; subprocess reads from
473 * xd[0], spits out band#2 to stderr, and feeds us band#1
474 * through our fd[0].
476 if (pipe(fd) < 0)
477 die("fetch-pack: unable to set up pipe");
478 side_pid = fork();
479 if (side_pid < 0)
480 die("fetch-pack: unable to fork off sideband demultiplexer");
481 if (!side_pid) {
482 /* subprocess */
483 close(fd[0]);
484 if (xd[0] != xd[1])
485 close(xd[1]);
486 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
487 exit(1);
488 exit(0);
490 close(xd[0]);
491 close(fd[1]);
492 fd[1] = xd[1];
493 return side_pid;
496 static int get_pack(int xd[2])
498 int status;
499 pid_t pid, side_pid;
500 int fd[2];
501 const char *argv[20];
502 char keep_arg[256];
503 char hdr_arg[256];
504 const char **av;
505 int do_keep = keep_pack;
507 side_pid = setup_sideband(fd, xd);
509 av = argv;
510 *hdr_arg = 0;
511 if (unpack_limit) {
512 struct pack_header header;
514 if (read_pack_header(fd[0], &header))
515 die("protocol error: bad pack header");
516 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%lu,%lu",
517 ntohl(header.hdr_version), ntohl(header.hdr_entries));
518 if (ntohl(header.hdr_entries) < unpack_limit)
519 do_keep = 0;
520 else
521 do_keep = 1;
524 if (do_keep) {
525 *av++ = "index-pack";
526 *av++ = "--stdin";
527 if (!quiet && !no_progress)
528 *av++ = "-v";
529 if (use_thin_pack)
530 *av++ = "--fix-thin";
531 if (keep_pack > 1 || unpack_limit) {
532 int s = sprintf(keep_arg,
533 "--keep=fetch-pack %d on ", getpid());
534 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
535 strcpy(keep_arg + s, "localhost");
536 *av++ = keep_arg;
539 else {
540 *av++ = "unpack-objects";
541 if (quiet)
542 *av++ = "-q";
544 if (*hdr_arg)
545 *av++ = hdr_arg;
546 *av++ = NULL;
548 pid = spawnv_git_cmd(argv, fd, NULL);
549 if (pid < 0)
550 die("fetch-pack: unable to fork off %s", argv[0]);
551 close(fd[1]);
552 while (waitpid(pid, &status, 0) < 0) {
553 if (errno != EINTR)
554 die("waiting for %s: %s", argv[0], strerror(errno));
556 if (WIFEXITED(status)) {
557 int code = WEXITSTATUS(status);
558 if (code)
559 die("%s died with error code %d", argv[0], code);
560 return 0;
562 if (WIFSIGNALED(status)) {
563 int sig = WTERMSIG(status);
564 die("%s died of signal %d", argv[0], sig);
566 die("%s died of unnatural causes %d", argv[0], status);
569 static int fetch_pack(int fd[2], int nr_match, char **match)
571 struct ref *ref;
572 unsigned char sha1[20];
574 get_remote_heads(fd[0], &ref, 0, NULL, 0);
575 if (is_repository_shallow() && !server_supports("shallow"))
576 die("Server does not support shallow clients");
577 if (server_supports("multi_ack")) {
578 if (verbose)
579 fprintf(stderr, "Server supports multi_ack\n");
580 multi_ack = 1;
582 #ifndef __MINGW32__
583 if (server_supports("side-band-64k")) {
584 if (verbose)
585 fprintf(stderr, "Server supports side-band-64k\n");
586 use_sideband = 2;
588 else if (server_supports("side-band")) {
589 if (verbose)
590 fprintf(stderr, "Server supports side-band\n");
591 use_sideband = 1;
593 #endif
594 if (!ref) {
595 packet_flush(fd[1]);
596 die("no matching remote head");
598 if (everything_local(&ref, nr_match, match)) {
599 packet_flush(fd[1]);
600 goto all_done;
602 if (find_common(fd, sha1, ref) < 0)
603 if (keep_pack != 1)
604 /* When cloning, it is not unusual to have
605 * no common commit.
607 fprintf(stderr, "warning: no common commits\n");
609 if (get_pack(fd))
610 die("git-fetch-pack: fetch failed.");
612 all_done:
613 while (ref) {
614 printf("%s %s\n",
615 sha1_to_hex(ref->old_sha1), ref->name);
616 ref = ref->next;
618 return 0;
621 static int remove_duplicates(int nr_heads, char **heads)
623 int src, dst;
625 for (src = dst = 0; src < nr_heads; src++) {
626 /* If heads[src] is different from any of
627 * heads[0..dst], push it in.
629 int i;
630 for (i = 0; i < dst; i++) {
631 if (!strcmp(heads[i], heads[src]))
632 break;
634 if (i < dst)
635 continue;
636 if (src != dst)
637 heads[dst] = heads[src];
638 dst++;
640 heads[dst] = 0;
641 return dst;
644 static int fetch_pack_config(const char *var, const char *value)
646 if (strcmp(var, "fetch.unpacklimit") == 0) {
647 fetch_unpack_limit = git_config_int(var, value);
648 return 0;
651 if (strcmp(var, "transfer.unpacklimit") == 0) {
652 transfer_unpack_limit = git_config_int(var, value);
653 return 0;
656 return git_default_config(var, value);
659 static struct lock_file lock;
661 int main(int argc, char **argv)
663 int i, ret, nr_heads;
664 char *dest = NULL, **heads;
665 int fd[2];
666 pid_t pid;
667 struct stat st;
669 setup_git_directory();
670 git_config(fetch_pack_config);
672 if (0 <= transfer_unpack_limit)
673 unpack_limit = transfer_unpack_limit;
674 else if (0 <= fetch_unpack_limit)
675 unpack_limit = fetch_unpack_limit;
677 nr_heads = 0;
678 heads = NULL;
679 for (i = 1; i < argc; i++) {
680 char *arg = argv[i];
682 if (*arg == '-') {
683 if (!prefixcmp(arg, "--upload-pack=")) {
684 uploadpack = arg + 14;
685 continue;
687 if (!prefixcmp(arg, "--exec=")) {
688 uploadpack = arg + 7;
689 continue;
691 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
692 quiet = 1;
693 continue;
695 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
696 keep_pack++;
697 unpack_limit = 0;
698 continue;
700 if (!strcmp("--thin", arg)) {
701 use_thin_pack = 1;
702 continue;
704 if (!strcmp("--all", arg)) {
705 fetch_all = 1;
706 continue;
708 if (!strcmp("-v", arg)) {
709 verbose = 1;
710 continue;
712 if (!prefixcmp(arg, "--depth=")) {
713 depth = strtol(arg + 8, NULL, 0);
714 if (stat(git_path("shallow"), &st))
715 st.st_mtime = 0;
716 continue;
718 if (!strcmp("--no-progress", arg)) {
719 no_progress = 1;
720 continue;
722 usage(fetch_pack_usage);
724 dest = arg;
725 heads = argv + i + 1;
726 nr_heads = argc - i - 1;
727 break;
729 if (!dest)
730 usage(fetch_pack_usage);
731 pid = git_connect(fd, dest, uploadpack, verbose ? CONNECT_VERBOSE : 0);
732 if (pid < 0)
733 return 1;
734 if (heads && nr_heads)
735 nr_heads = remove_duplicates(nr_heads, heads);
736 ret = fetch_pack(fd, nr_heads, heads);
737 close(fd[0]);
738 close(fd[1]);
739 ret |= finish_connect(pid);
741 if (!ret && nr_heads) {
742 /* If the heads to pull were given, we should have
743 * consumed all of them by matching the remote.
744 * Otherwise, 'git-fetch remote no-such-ref' would
745 * silently succeed without issuing an error.
747 for (i = 0; i < nr_heads; i++)
748 if (heads[i] && heads[i][0]) {
749 error("no such remote ref %s", heads[i]);
750 ret = 1;
754 if (!ret && depth > 0) {
755 struct cache_time mtime;
756 char *shallow = git_path("shallow");
757 int fd;
759 mtime.sec = st.st_mtime;
760 #ifdef USE_NSEC
761 mtime.usec = st.st_mtim.usec;
762 #endif
763 if (stat(shallow, &st)) {
764 if (mtime.sec)
765 die("shallow file was removed during fetch");
766 } else if (st.st_mtime != mtime.sec
767 #ifdef USE_NSEC
768 || st.st_mtim.usec != mtime.usec
769 #endif
771 die("shallow file was changed during fetch");
773 fd = hold_lock_file_for_update(&lock, shallow, 1);
774 if (!write_shallow_commits(fd, 0)) {
775 unlink(shallow);
776 rollback_lock_file(&lock);
777 } else {
778 close(fd);
779 commit_lock_file(&lock);
783 return !!ret;