fast-import: fix unalinged allocation and access
[git/git-p4.git] / builtin-fetch-pack.c
blob807fa93b53ad16552b7d869470f2ab59a325808a
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args = {
16 /* .uploadpack = */ "git-upload-pack",
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22 #define COMPLETE (1U << 0)
23 #define COMMON (1U << 1)
24 #define COMMON_REF (1U << 2)
25 #define SEEN (1U << 3)
26 #define POPPED (1U << 4)
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
32 #define MAX_IN_VAIN 256
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_sideband;
37 static void rev_list_push(struct commit *commit, int mark)
39 if (!(commit->object.flags & mark)) {
40 commit->object.flags |= mark;
42 if (!(commit->object.parsed))
43 parse_commit(commit);
45 insert_by_date(commit, &rev_list);
47 if (!(commit->object.flags & COMMON))
48 non_common_revs++;
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 struct object *o = deref_tag(parse_object(sha1), path, 0);
56 if (o && o->type == OBJ_COMMIT)
57 rev_list_push((struct commit *)o, SEEN);
59 return 0;
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
68 static void mark_common(struct commit *commit,
69 int ancestors_only, int dont_parse)
71 if (commit != NULL && !(commit->object.flags & COMMON)) {
72 struct object *o = (struct object *)commit;
74 if (!ancestors_only)
75 o->flags |= COMMON;
77 if (!(o->flags & SEEN))
78 rev_list_push(commit, SEEN);
79 else {
80 struct commit_list *parents;
82 if (!ancestors_only && !(o->flags & POPPED))
83 non_common_revs--;
84 if (!o->parsed && !dont_parse)
85 parse_commit(commit);
87 for (parents = commit->parents;
88 parents;
89 parents = parents->next)
90 mark_common(parents->item, 0, dont_parse);
96 Get the next rev to send, ignoring the common.
99 static const unsigned char* get_rev(void)
101 struct commit *commit = NULL;
103 while (commit == NULL) {
104 unsigned int mark;
105 struct commit_list* parents;
107 if (rev_list == NULL || non_common_revs == 0)
108 return NULL;
110 commit = rev_list->item;
111 if (!(commit->object.parsed))
112 parse_commit(commit);
113 commit->object.flags |= POPPED;
114 if (!(commit->object.flags & COMMON))
115 non_common_revs--;
117 parents = commit->parents;
119 if (commit->object.flags & COMMON) {
120 /* do not send "have", and ignore ancestors */
121 commit = NULL;
122 mark = COMMON | SEEN;
123 } else if (commit->object.flags & COMMON_REF)
124 /* send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 else
127 /* send "have", also for its ancestors */
128 mark = SEEN;
130 while (parents) {
131 if (!(parents->item->object.flags & SEEN))
132 rev_list_push(parents->item, mark);
133 if (mark & COMMON)
134 mark_common(parents->item, 1, 0);
135 parents = parents->next;
138 rev_list = rev_list->next;
141 return commit->object.sha1;
144 static int find_common(int fd[2], unsigned char *result_sha1,
145 struct ref *refs)
147 int fetching;
148 int count = 0, flushes = 0, retval;
149 const unsigned char *sha1;
150 unsigned in_vain = 0;
151 int got_continue = 0;
153 for_each_ref(rev_list_insert_ref, NULL);
155 fetching = 0;
156 for ( ; refs ; refs = refs->next) {
157 unsigned char *remote = refs->old_sha1;
158 struct object *o;
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
164 * about.
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
170 if (((o = lookup_object(remote)) != NULL) &&
171 (o->flags & COMPLETE)) {
172 continue;
175 if (!fetching)
176 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
177 sha1_to_hex(remote),
178 (multi_ack ? " multi_ack" : ""),
179 (use_sideband == 2 ? " side-band-64k" : ""),
180 (use_sideband == 1 ? " side-band" : ""),
181 (args.use_thin_pack ? " thin-pack" : ""),
182 (args.no_progress ? " no-progress" : ""),
183 " ofs-delta");
184 else
185 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186 fetching++;
188 if (is_repository_shallow())
189 write_shallow_commits(fd[1], 1);
190 if (args.depth > 0)
191 packet_write(fd[1], "deepen %d", args.depth);
192 packet_flush(fd[1]);
193 if (!fetching)
194 return 1;
196 if (args.depth > 0) {
197 char line[1024];
198 unsigned char sha1[20];
199 int len;
201 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202 if (!prefixcmp(line, "shallow ")) {
203 if (get_sha1_hex(line + 8, sha1))
204 die("invalid shallow line: %s", line);
205 register_shallow(sha1);
206 continue;
208 if (!prefixcmp(line, "unshallow ")) {
209 if (get_sha1_hex(line + 10, sha1))
210 die("invalid unshallow line: %s", line);
211 if (!lookup_object(sha1))
212 die("object not found: %s", line);
213 /* make sure that it is parsed as shallow */
214 parse_object(sha1);
215 if (unregister_shallow(sha1))
216 die("no shallow found: %s", line);
217 continue;
219 die("expected shallow/unshallow, got %s", line);
223 flushes = 0;
224 retval = -1;
225 while ((sha1 = get_rev())) {
226 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227 if (args.verbose)
228 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229 in_vain++;
230 if (!(31 & ++count)) {
231 int ack;
233 packet_flush(fd[1]);
234 flushes++;
237 * We keep one window "ahead" of the other side, and
238 * will wait for an ACK only on the next one
240 if (count == 32)
241 continue;
243 do {
244 ack = get_ack(fd[0], result_sha1);
245 if (args.verbose && ack)
246 fprintf(stderr, "got ack %d %s\n", ack,
247 sha1_to_hex(result_sha1));
248 if (ack == 1) {
249 flushes = 0;
250 multi_ack = 0;
251 retval = 0;
252 goto done;
253 } else if (ack == 2) {
254 struct commit *commit =
255 lookup_commit(result_sha1);
256 mark_common(commit, 0, 1);
257 retval = 0;
258 in_vain = 0;
259 got_continue = 1;
261 } while (ack);
262 flushes--;
263 if (got_continue && MAX_IN_VAIN < in_vain) {
264 if (args.verbose)
265 fprintf(stderr, "giving up\n");
266 break; /* give up */
270 done:
271 packet_write(fd[1], "done\n");
272 if (args.verbose)
273 fprintf(stderr, "done\n");
274 if (retval != 0) {
275 multi_ack = 0;
276 flushes++;
278 while (flushes || multi_ack) {
279 int ack = get_ack(fd[0], result_sha1);
280 if (ack) {
281 if (args.verbose)
282 fprintf(stderr, "got ack (%d) %s\n", ack,
283 sha1_to_hex(result_sha1));
284 if (ack == 1)
285 return 0;
286 multi_ack = 1;
287 continue;
289 flushes--;
291 return retval;
294 static struct commit_list *complete;
296 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298 struct object *o = parse_object(sha1);
300 while (o && o->type == OBJ_TAG) {
301 struct tag *t = (struct tag *) o;
302 if (!t->tagged)
303 break; /* broken repository */
304 o->flags |= COMPLETE;
305 o = parse_object(t->tagged->sha1);
307 if (o && o->type == OBJ_COMMIT) {
308 struct commit *commit = (struct commit *)o;
309 commit->object.flags |= COMPLETE;
310 insert_by_date(commit, &complete);
312 return 0;
315 static void mark_recent_complete_commits(unsigned long cutoff)
317 while (complete && cutoff <= complete->item->date) {
318 if (args.verbose)
319 fprintf(stderr, "Marking %s as complete\n",
320 sha1_to_hex(complete->item->object.sha1));
321 pop_most_recent_commit(&complete, COMPLETE);
325 static void filter_refs(struct ref **refs, int nr_match, char **match)
327 struct ref **return_refs;
328 struct ref *newlist = NULL;
329 struct ref **newtail = &newlist;
330 struct ref *ref, *next;
331 struct ref *fastarray[32];
333 if (nr_match && !args.fetch_all) {
334 if (ARRAY_SIZE(fastarray) < nr_match)
335 return_refs = xcalloc(nr_match, sizeof(struct ref *));
336 else {
337 return_refs = fastarray;
338 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
341 else
342 return_refs = NULL;
344 for (ref = *refs; ref; ref = next) {
345 next = ref->next;
346 if (!memcmp(ref->name, "refs/", 5) &&
347 check_ref_format(ref->name + 5))
348 ; /* trash */
349 else if (args.fetch_all &&
350 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
351 *newtail = ref;
352 ref->next = NULL;
353 newtail = &ref->next;
354 continue;
356 else {
357 int order = path_match(ref->name, nr_match, match);
358 if (order) {
359 return_refs[order-1] = ref;
360 continue; /* we will link it later */
363 free(ref);
366 if (!args.fetch_all) {
367 int i;
368 for (i = 0; i < nr_match; i++) {
369 ref = return_refs[i];
370 if (ref) {
371 *newtail = ref;
372 ref->next = NULL;
373 newtail = &ref->next;
376 if (return_refs != fastarray)
377 free(return_refs);
379 *refs = newlist;
382 static int everything_local(struct ref **refs, int nr_match, char **match)
384 struct ref *ref;
385 int retval;
386 unsigned long cutoff = 0;
388 track_object_refs = 0;
389 save_commit_buffer = 0;
391 for (ref = *refs; ref; ref = ref->next) {
392 struct object *o;
394 o = parse_object(ref->old_sha1);
395 if (!o)
396 continue;
398 /* We already have it -- which may mean that we were
399 * in sync with the other side at some time after
400 * that (it is OK if we guess wrong here).
402 if (o->type == OBJ_COMMIT) {
403 struct commit *commit = (struct commit *)o;
404 if (!cutoff || cutoff < commit->date)
405 cutoff = commit->date;
409 if (!args.depth) {
410 for_each_ref(mark_complete, NULL);
411 if (cutoff)
412 mark_recent_complete_commits(cutoff);
416 * Mark all complete remote refs as common refs.
417 * Don't mark them common yet; the server has to be told so first.
419 for (ref = *refs; ref; ref = ref->next) {
420 struct object *o = deref_tag(lookup_object(ref->old_sha1),
421 NULL, 0);
423 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
424 continue;
426 if (!(o->flags & SEEN)) {
427 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
429 mark_common((struct commit *)o, 1, 1);
433 filter_refs(refs, nr_match, match);
435 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
436 const unsigned char *remote = ref->old_sha1;
437 unsigned char local[20];
438 struct object *o;
440 o = lookup_object(remote);
441 if (!o || !(o->flags & COMPLETE)) {
442 retval = 0;
443 if (!args.verbose)
444 continue;
445 fprintf(stderr,
446 "want %s (%s)\n", sha1_to_hex(remote),
447 ref->name);
448 continue;
451 hashcpy(ref->new_sha1, local);
452 if (!args.verbose)
453 continue;
454 fprintf(stderr,
455 "already have %s (%s)\n", sha1_to_hex(remote),
456 ref->name);
458 return retval;
461 static int sideband_demux(int fd, void *data)
463 int *xd = data;
465 return recv_sideband("fetch-pack", xd[0], fd, 2);
468 static int get_pack(int xd[2], char **pack_lockfile)
470 struct async demux;
471 const char *argv[20];
472 char keep_arg[256];
473 char hdr_arg[256];
474 const char **av;
475 int do_keep = args.keep_pack;
476 struct child_process cmd;
478 memset(&demux, 0, sizeof(demux));
479 if (use_sideband) {
480 /* xd[] is talking with upload-pack; subprocess reads from
481 * xd[0], spits out band#2 to stderr, and feeds us band#1
482 * through demux->out.
484 demux.proc = sideband_demux;
485 demux.data = xd;
486 if (start_async(&demux))
487 die("fetch-pack: unable to fork off sideband"
488 " demultiplexer");
490 else
491 demux.out = xd[0];
493 memset(&cmd, 0, sizeof(cmd));
494 cmd.argv = argv;
495 av = argv;
496 *hdr_arg = 0;
497 if (!args.keep_pack && unpack_limit) {
498 struct pack_header header;
500 if (read_pack_header(demux.out, &header))
501 die("protocol error: bad pack header");
502 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
503 ntohl(header.hdr_version), ntohl(header.hdr_entries));
504 if (ntohl(header.hdr_entries) < unpack_limit)
505 do_keep = 0;
506 else
507 do_keep = 1;
510 if (do_keep) {
511 if (pack_lockfile)
512 cmd.out = -1;
513 *av++ = "index-pack";
514 *av++ = "--stdin";
515 if (!args.quiet && !args.no_progress)
516 *av++ = "-v";
517 if (args.use_thin_pack)
518 *av++ = "--fix-thin";
519 if (args.lock_pack || unpack_limit) {
520 int s = sprintf(keep_arg,
521 "--keep=fetch-pack %d on ", getpid());
522 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
523 strcpy(keep_arg + s, "localhost");
524 *av++ = keep_arg;
527 else {
528 *av++ = "unpack-objects";
529 if (args.quiet)
530 *av++ = "-q";
532 if (*hdr_arg)
533 *av++ = hdr_arg;
534 *av++ = NULL;
536 cmd.in = demux.out;
537 cmd.git_cmd = 1;
538 if (start_command(&cmd))
539 die("fetch-pack: unable to fork off %s", argv[0]);
540 if (do_keep && pack_lockfile)
541 *pack_lockfile = index_pack_lockfile(cmd.out);
543 if (finish_command(&cmd))
544 die("%s failed", argv[0]);
545 if (use_sideband && finish_async(&demux))
546 die("error in sideband demultiplexer");
547 return 0;
550 static struct ref *do_fetch_pack(int fd[2],
551 int nr_match,
552 char **match,
553 char **pack_lockfile)
555 struct ref *ref;
556 unsigned char sha1[20];
558 get_remote_heads(fd[0], &ref, 0, NULL, 0);
559 if (is_repository_shallow() && !server_supports("shallow"))
560 die("Server does not support shallow clients");
561 if (server_supports("multi_ack")) {
562 if (args.verbose)
563 fprintf(stderr, "Server supports multi_ack\n");
564 multi_ack = 1;
566 if (server_supports("side-band-64k")) {
567 if (args.verbose)
568 fprintf(stderr, "Server supports side-band-64k\n");
569 use_sideband = 2;
571 else if (server_supports("side-band")) {
572 if (args.verbose)
573 fprintf(stderr, "Server supports side-band\n");
574 use_sideband = 1;
576 if (!ref) {
577 packet_flush(fd[1]);
578 die("no matching remote head");
580 if (everything_local(&ref, nr_match, match)) {
581 packet_flush(fd[1]);
582 goto all_done;
584 if (find_common(fd, sha1, ref) < 0)
585 if (!args.keep_pack)
586 /* When cloning, it is not unusual to have
587 * no common commit.
589 fprintf(stderr, "warning: no common commits\n");
591 if (get_pack(fd, pack_lockfile))
592 die("git-fetch-pack: fetch failed.");
594 all_done:
595 return ref;
598 static int remove_duplicates(int nr_heads, char **heads)
600 int src, dst;
602 for (src = dst = 0; src < nr_heads; src++) {
603 /* If heads[src] is different from any of
604 * heads[0..dst], push it in.
606 int i;
607 for (i = 0; i < dst; i++) {
608 if (!strcmp(heads[i], heads[src]))
609 break;
611 if (i < dst)
612 continue;
613 if (src != dst)
614 heads[dst] = heads[src];
615 dst++;
617 return dst;
620 static int fetch_pack_config(const char *var, const char *value)
622 if (strcmp(var, "fetch.unpacklimit") == 0) {
623 fetch_unpack_limit = git_config_int(var, value);
624 return 0;
627 if (strcmp(var, "transfer.unpacklimit") == 0) {
628 transfer_unpack_limit = git_config_int(var, value);
629 return 0;
632 return git_default_config(var, value);
635 static struct lock_file lock;
637 static void fetch_pack_setup(void)
639 static int did_setup;
640 if (did_setup)
641 return;
642 git_config(fetch_pack_config);
643 if (0 <= transfer_unpack_limit)
644 unpack_limit = transfer_unpack_limit;
645 else if (0 <= fetch_unpack_limit)
646 unpack_limit = fetch_unpack_limit;
647 did_setup = 1;
650 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
652 int i, ret, nr_heads;
653 struct ref *ref;
654 char *dest = NULL, **heads;
656 nr_heads = 0;
657 heads = NULL;
658 for (i = 1; i < argc; i++) {
659 const char *arg = argv[i];
661 if (*arg == '-') {
662 if (!prefixcmp(arg, "--upload-pack=")) {
663 args.uploadpack = arg + 14;
664 continue;
666 if (!prefixcmp(arg, "--exec=")) {
667 args.uploadpack = arg + 7;
668 continue;
670 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
671 args.quiet = 1;
672 continue;
674 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
675 args.lock_pack = args.keep_pack;
676 args.keep_pack = 1;
677 continue;
679 if (!strcmp("--thin", arg)) {
680 args.use_thin_pack = 1;
681 continue;
683 if (!strcmp("--all", arg)) {
684 args.fetch_all = 1;
685 continue;
687 if (!strcmp("-v", arg)) {
688 args.verbose = 1;
689 continue;
691 if (!prefixcmp(arg, "--depth=")) {
692 args.depth = strtol(arg + 8, NULL, 0);
693 continue;
695 if (!strcmp("--no-progress", arg)) {
696 args.no_progress = 1;
697 continue;
699 usage(fetch_pack_usage);
701 dest = (char *)arg;
702 heads = (char **)(argv + i + 1);
703 nr_heads = argc - i - 1;
704 break;
706 if (!dest)
707 usage(fetch_pack_usage);
709 ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
710 ret = !ref;
712 while (ref) {
713 printf("%s %s\n",
714 sha1_to_hex(ref->old_sha1), ref->name);
715 ref = ref->next;
718 return ret;
721 struct ref *fetch_pack(struct fetch_pack_args *my_args,
722 const char *dest,
723 int nr_heads,
724 char **heads,
725 char **pack_lockfile)
727 int i, ret;
728 int fd[2];
729 struct child_process *conn;
730 struct ref *ref;
731 struct stat st;
733 fetch_pack_setup();
734 memcpy(&args, my_args, sizeof(args));
735 if (args.depth > 0) {
736 if (stat(git_path("shallow"), &st))
737 st.st_mtime = 0;
740 conn = git_connect(fd, (char *)dest, args.uploadpack,
741 args.verbose ? CONNECT_VERBOSE : 0);
742 if (heads && nr_heads)
743 nr_heads = remove_duplicates(nr_heads, heads);
744 ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
745 close(fd[0]);
746 close(fd[1]);
747 ret = finish_connect(conn);
749 if (!ret && nr_heads) {
750 /* If the heads to pull were given, we should have
751 * consumed all of them by matching the remote.
752 * Otherwise, 'git-fetch remote no-such-ref' would
753 * silently succeed without issuing an error.
755 for (i = 0; i < nr_heads; i++)
756 if (heads[i] && heads[i][0]) {
757 error("no such remote ref %s", heads[i]);
758 ret = 1;
762 if (!ret && args.depth > 0) {
763 struct cache_time mtime;
764 char *shallow = git_path("shallow");
765 int fd;
767 mtime.sec = st.st_mtime;
768 #ifdef USE_NSEC
769 mtime.usec = st.st_mtim.usec;
770 #endif
771 if (stat(shallow, &st)) {
772 if (mtime.sec)
773 die("shallow file was removed during fetch");
774 } else if (st.st_mtime != mtime.sec
775 #ifdef USE_NSEC
776 || st.st_mtim.usec != mtime.usec
777 #endif
779 die("shallow file was changed during fetch");
781 fd = hold_lock_file_for_update(&lock, shallow, 1);
782 if (!write_shallow_commits(fd, 0)) {
783 unlink(shallow);
784 rollback_lock_file(&lock);
785 } else {
786 close(fd);
787 commit_lock_file(&lock);
791 if (ret)
792 ref = NULL;
794 return ref;