libexec/got-fetch-pack/got-fetch-pack.c

   1 /*
   2  * Copyright (c) 2019 Ori Bernstein <ori@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/types.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/time.h>
  23 #include <sys/stat.h>
  24
  25 #include <stdint.h>
  26 #include <errno.h>
  27 #include <limits.h>
  28 #include <signal.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <ctype.h>
  33 #include <fcntl.h>
  34 #include <unistd.h>
  35 #include <zlib.h>
  36 #include <err.h>
  37
  38 #include "got_error.h"
  39 #include "got_object.h"
  40 #include "got_path.h"
  41 #include "got_version.h"
  42 #include "got_fetch.h"
  43 #include "got_reference.h"
  44
  45 #include "got_lib_hash.h"
  46 #include "got_lib_delta.h"
  47 #include "got_lib_object.h"
  48 #include "got_lib_object_parse.h"
  49 #include "got_lib_privsep.h"
  50 #include "got_lib_pack.h"
  51 #include "got_lib_pkt.h"
  52 #include "got_lib_gitproto.h"
  53 #include "got_lib_ratelimit.h"
  54
  55 #ifndef MIN
  56 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  57 #endif
  58
  59 #ifndef nitems
  60 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  61 #endif
  62
  63 struct got_object *indexed;
  64 static int chattygot;
  65
  66 static const struct got_capability got_capabilities[] = {
  67         { GOT_CAPA_AGENT, "got/" GOT_VERSION_STR },
  68         { GOT_CAPA_OFS_DELTA, NULL },
  69         { GOT_CAPA_SIDE_BAND_64K, NULL },
  70 };
  71
  72 static void
  73 match_remote_ref(struct got_pathlist_head *have_refs,
  74     struct got_object_id *my_id, const char *refname)
  75 {
  76         struct got_pathlist_entry *pe;
  77
  78         /* XXX zero-hash signifies we don't have this ref;
  79          * we should use a flag instead */
  80         memset(my_id, 0, sizeof(*my_id));
  81
  82         TAILQ_FOREACH(pe, have_refs, entry) {
  83                 struct got_object_id *id = pe->data;
  84                 if (strcmp(pe->path, refname) == 0) {
  85                         memcpy(my_id, id, sizeof(*my_id));
  86                         break;
  87                 }
  88         }
  89 }
  90
  91 static int
  92 match_branch(const char *branch, const char *wanted_branch)
  93 {
  94         if (strncmp(branch, "refs/heads/", 11) != 0)
  95                 return 0;
  96
  97         if (strncmp(wanted_branch, "refs/heads/", 11) == 0)
  98                 wanted_branch += 11;
  99
 100         return (strcmp(branch + 11, wanted_branch) == 0);
 101 }
 102
 103 static int
 104 match_wanted_ref(const char *refname, const char *wanted_ref)
 105 {
 106         if (strncmp(refname, "refs/", 5) != 0)
 107                 return 0;
 108         refname += 5;
 109
 110         /*
 111          * Prevent fetching of references that won't make any
 112          * sense outside of the remote repository's context.
 113          */
 114         if (strncmp(refname, "got/", 4) == 0)
 115                 return 0;
 116         if (strncmp(refname, "remotes/", 8) == 0)
 117                 return 0;
 118
 119         if (strncmp(wanted_ref, "refs/", 5) == 0)
 120                 wanted_ref += 5;
 121
 122         /* Allow prefix match. */
 123         if (got_path_is_child(refname, wanted_ref, strlen(wanted_ref)))
 124                 return 1;
 125
 126         /* Allow exact match. */
 127         return (strcmp(refname, wanted_ref) == 0);
 128 }
 129
 130 static const struct got_error *
 131 send_fetch_server_progress(struct imsgbuf *ibuf, const char *msg, size_t msglen)
 132 {
 133         if (msglen > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 134                 return got_error(GOT_ERR_NO_SPACE);
 135
 136         if (msglen == 0)
 137                 return NULL;
 138
 139         if (imsg_compose(ibuf, GOT_IMSG_FETCH_SERVER_PROGRESS, 0, 0, -1,
 140             msg, msglen) == -1)
 141                 return got_error_from_errno(
 142                     "imsg_compose FETCH_SERVER_PROGRESS");
 143
 144         return got_privsep_flush_imsg(ibuf);
 145 }
 146
 147 static const struct got_error *
 148 send_fetch_download_progress(struct imsgbuf *ibuf, off_t bytes,
 149     struct got_ratelimit *rl)
 150 {
 151         const struct got_error *err;
 152         int elapsed = 0;
 153
 154         if (rl) {
 155                 err = got_ratelimit_check(&elapsed, rl);
 156                 if (err || !elapsed)
 157                         return err;
 158         }
 159
 160         if (imsg_compose(ibuf, GOT_IMSG_FETCH_DOWNLOAD_PROGRESS, 0, 0, -1,
 161             &bytes, sizeof(bytes)) == -1)
 162                 return got_error_from_errno(
 163                     "imsg_compose FETCH_DOWNLOAD_PROGRESS");
 164
 165         return got_privsep_flush_imsg(ibuf);
 166 }
 167
 168 static const struct got_error *
 169 send_fetch_done(struct imsgbuf *ibuf, uint8_t *pack_sha1)
 170 {
 171         if (imsg_compose(ibuf, GOT_IMSG_FETCH_DONE, 0, 0, -1,
 172             pack_sha1, SHA1_DIGEST_LENGTH) == -1)
 173                 return got_error_from_errno("imsg_compose FETCH");
 174         return got_privsep_flush_imsg(ibuf);
 175 }
 176
 177 static const struct got_error *
 178 fetch_progress(struct imsgbuf *ibuf, const char *buf, size_t len)
 179 {
 180         size_t i;
 181
 182         if (len == 0)
 183                 return NULL;
 184
 185         /*
 186          * Truncate messages which exceed the maximum imsg payload size.
 187          * Server may send up to 64k.
 188          */
 189         if (len > MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 190                 len = MAX_IMSGSIZE - IMSG_HEADER_SIZE;
 191
 192         /* Only allow printable ASCII. */
 193         for (i = 0; i < len; i++) {
 194                 if (isprint((unsigned char)buf[i]) ||
 195                     isspace((unsigned char)buf[i]))
 196                         continue;
 197                 return got_error_msg(GOT_ERR_BAD_PACKET,
 198                     "non-printable progress message received from server");
 199         }
 200
 201         return send_fetch_server_progress(ibuf, buf, len);
 202 }
 203
 204 static const struct got_error *
 205 fetch_error(const char *buf, size_t len)
 206 {
 207         static char msg[1024];
 208         size_t i;
 209
 210         for (i = 0; i < len && i < sizeof(msg) - 1; i++) {
 211                 if (!isprint((unsigned char)buf[i]))
 212                         return got_error_msg(GOT_ERR_BAD_PACKET,
 213                             "non-printable error message received from server");
 214                 msg[i] = buf[i];
 215         }
 216         msg[i] = '\0';
 217         return got_error_msg(GOT_ERR_FETCH_FAILED, msg);
 218 }
 219
 220 static const struct got_error *
 221 send_fetch_symrefs(struct imsgbuf *ibuf, struct got_pathlist_head *symrefs)
 222 {
 223         struct ibuf *wbuf;
 224         size_t len, nsymrefs = 0;
 225         struct got_pathlist_entry *pe;
 226
 227         len = sizeof(struct got_imsg_fetch_symrefs);
 228         TAILQ_FOREACH(pe, symrefs, entry) {
 229                 const char *target = pe->data;
 230                 len += sizeof(struct got_imsg_fetch_symref) +
 231                     pe->path_len + strlen(target);
 232                 nsymrefs++;
 233         }
 234
 235         if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 236                 return got_error(GOT_ERR_NO_SPACE);
 237
 238         wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_SYMREFS, 0, 0, len);
 239         if (wbuf == NULL)
 240                 return got_error_from_errno("imsg_create FETCH_SYMREFS");
 241
 242         /* Keep in sync with struct got_imsg_fetch_symrefs definition! */
 243         if (imsg_add(wbuf, &nsymrefs, sizeof(nsymrefs)) == -1)
 244                 return got_error_from_errno("imsg_add FETCH_SYMREFS");
 245
 246         TAILQ_FOREACH(pe, symrefs, entry) {
 247                 const char *name = pe->path;
 248                 size_t name_len = pe->path_len;
 249                 const char *target = pe->data;
 250                 size_t target_len = strlen(target);
 251
 252                 /* Keep in sync with struct got_imsg_fetch_symref definition! */
 253                 if (imsg_add(wbuf, &name_len, sizeof(name_len)) == -1)
 254                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 255                 if (imsg_add(wbuf, &target_len, sizeof(target_len)) == -1)
 256                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 257                 if (imsg_add(wbuf, name, name_len) == -1)
 258                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 259                 if (imsg_add(wbuf, target, target_len) == -1)
 260                         return got_error_from_errno("imsg_add FETCH_SYMREFS");
 261         }
 262
 263         imsg_close(ibuf, wbuf);
 264         return got_privsep_flush_imsg(ibuf);
 265 }
 266
 267 static const struct got_error *
 268 send_fetch_ref(struct imsgbuf *ibuf, struct got_object_id *refid,
 269     const char *refname)
 270 {
 271         struct ibuf *wbuf;
 272         size_t len, reflen = strlen(refname);
 273
 274         len = sizeof(struct got_imsg_fetch_ref) + reflen;
 275         if (len >= MAX_IMSGSIZE - IMSG_HEADER_SIZE)
 276                 return got_error(GOT_ERR_NO_SPACE);
 277
 278         wbuf = imsg_create(ibuf, GOT_IMSG_FETCH_REF, 0, 0, len);
 279         if (wbuf == NULL)
 280                 return got_error_from_errno("imsg_create FETCH_REF");
 281
 282         /* Keep in sync with struct got_imsg_fetch_ref definition! */
 283         if (imsg_add(wbuf, refid, sizeof(*refid)) == -1)
 284                 return got_error_from_errno("imsg_add FETCH_REF");
 285         if (imsg_add(wbuf, refname, reflen) == -1)
 286                 return got_error_from_errno("imsg_add FETCH_REF");
 287
 288         imsg_close(ibuf, wbuf);
 289         return got_privsep_flush_imsg(ibuf);
 290 }
 291
 292 static const struct got_error *
 293 fetch_ref(struct imsgbuf *ibuf, struct got_pathlist_head *have_refs,
 294     struct got_object_id *have, struct got_object_id *want,
 295     const char *refname, const char *id_str)
 296 {
 297         const struct got_error *err;
 298         char *theirs = NULL, *mine = NULL;
 299
 300         if (!got_parse_object_id(want, id_str, GOT_HASH_SHA1)) {
 301                 err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
 302                 goto done;
 303         }
 304
 305         match_remote_ref(have_refs, have, refname);
 306         err = send_fetch_ref(ibuf, want, refname);
 307         if (err)
 308                 goto done;
 309
 310         if (chattygot)
 311                 fprintf(stderr, "%s: %s will be fetched\n",
 312                     getprogname(), refname);
 313         if (chattygot > 1) {
 314                 err = got_object_id_str(&theirs, want);
 315                 if (err)
 316                         goto done;
 317                 err = got_object_id_str(&mine, have);
 318                 if (err)
 319                         goto done;
 320                 fprintf(stderr, "%s: remote: %s\n%s: local:  %s\n",
 321                     getprogname(), theirs, getprogname(), mine);
 322         }
 323 done:
 324         free(theirs);
 325         free(mine);
 326         return err;
 327 }
 328
 329 static const struct got_error *
 330 fetch_pack(int fd, int packfd, uint8_t *pack_sha1,
 331     struct got_pathlist_head *have_refs, int fetch_all_branches,
 332     struct got_pathlist_head *wanted_branches,
 333     struct got_pathlist_head *wanted_refs, int list_refs_only,
 334     const char *worktree_branch, const char *remote_head,
 335     int no_head, struct imsgbuf *ibuf)
 336 {
 337         const struct got_error *err = NULL;
 338         char buf[GOT_PKT_MAX];
 339         char hashstr[SHA1_DIGEST_STRING_LENGTH];
 340         struct got_object_id *have, *want;
 341         int is_firstpkt = 1, nref = 0, refsz = 16;
 342         int i, n, nwant = 0, nhave = 0, acked = 0;
 343         off_t packsz = 0, last_reported_packsz = 0;
 344         char *id_str = NULL, *default_id_str = NULL, *refname = NULL;
 345         char *server_capabilities = NULL, *my_capabilities = NULL;
 346         const char *default_branch = NULL;
 347         struct got_pathlist_head symrefs;
 348         struct got_pathlist_entry *pe;
 349         int sent_my_capabilites = 0, have_sidebands = 0;
 350         int found_branch = 0;
 351         struct got_hash ctx;
 352         uint8_t sha1_buf[SHA1_DIGEST_LENGTH];
 353         size_t sha1_buf_len = 0;
 354         ssize_t w;
 355         struct got_ratelimit rl;
 356
 357         TAILQ_INIT(&symrefs);
 358         got_hash_init(&ctx, GOT_HASH_SHA1);
 359         got_ratelimit_init(&rl, 0, 500);
 360
 361         have = malloc(refsz * sizeof(have[0]));
 362         if (have == NULL)
 363                 return got_error_from_errno("malloc");
 364         want = malloc(refsz * sizeof(want[0]));
 365         if (want == NULL) {
 366                 err = got_error_from_errno("malloc");
 367                 goto done;
 368         }
 369         while (1) {
 370                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot);
 371                 if (err)
 372                         goto done;
 373                 if (n == 0)
 374                         break;
 375                 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
 376                         err = fetch_error(&buf[4], n - 4);
 377                         goto done;
 378                 }
 379                 free(id_str);
 380                 free(refname);
 381                 err = got_gitproto_parse_refline(&id_str, &refname,
 382                     &server_capabilities, buf, n);
 383                 if (err)
 384                         goto done;
 385
 386                 if (refsz == nref + 1) {
 387                         struct got_object_id *h, *w;
 388
 389                         refsz *= 2;
 390                         h = reallocarray(have, refsz, sizeof(have[0]));
 391                         if (h == NULL) {
 392                                 err = got_error_from_errno("reallocarray");
 393                                 goto done;
 394                         }
 395                         have = h;
 396                         w = reallocarray(want, refsz, sizeof(want[0]));
 397                         if (w == NULL) {
 398                                 err = got_error_from_errno("reallocarray");
 399                                 goto done;
 400                         }
 401                         want = w;
 402                 }
 403
 404                 if (is_firstpkt) {
 405                         if (chattygot && server_capabilities[0] != '\0')
 406                                 fprintf(stderr, "%s: server capabilities: %s\n",
 407                                     getprogname(), server_capabilities);
 408                         err = got_gitproto_match_capabilities(&my_capabilities,
 409                             &symrefs, server_capabilities,
 410                             got_capabilities, nitems(got_capabilities));
 411                         if (err)
 412                                 goto done;
 413                         if (chattygot)
 414                                 fprintf(stderr, "%s: my capabilities:%s\n",
 415                                     getprogname(), my_capabilities != NULL ?
 416                                     my_capabilities : "");
 417                         err = send_fetch_symrefs(ibuf, &symrefs);
 418                         if (err)
 419                                 goto done;
 420                         is_firstpkt = 0;
 421                         if (!fetch_all_branches) {
 422                                 TAILQ_FOREACH(pe, &symrefs, entry) {
 423                                         const char *name = pe->path;
 424                                         const char *symref_target = pe->data;
 425                                         if (strcmp(name, GOT_REF_HEAD) != 0)
 426                                                 continue;
 427                                         default_branch = symref_target;
 428                                         break;
 429                                 }
 430                         }
 431                         if (default_branch)
 432                                 continue;
 433                 }
 434                 if (strstr(refname, "^{}")) {
 435                         if (chattygot) {
 436                                 fprintf(stderr, "%s: ignoring %s\n",
 437                                     getprogname(), refname);
 438                         }
 439                         continue;
 440                 }
 441                 if (default_branch && default_id_str == NULL &&
 442                     strcmp(refname, default_branch) == 0) {
 443                         default_id_str = strdup(id_str);
 444                         if (default_id_str == NULL) {
 445                                 err = got_error_from_errno("strdup");
 446                                 goto done;
 447                         }
 448                 }
 449
 450                 if (list_refs_only || strncmp(refname, "refs/tags/", 10) == 0) {
 451                         err = fetch_ref(ibuf, have_refs, &have[nref],
 452                             &want[nref], refname, id_str);
 453                         if (err)
 454                                 goto done;
 455                         nref++;
 456                 } else if (strncmp(refname, "refs/heads/", 11) == 0) {
 457                         if (fetch_all_branches) {
 458                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 459                                     &want[nref], refname, id_str);
 460                                 if (err)
 461                                         goto done;
 462                                 nref++;
 463                                 found_branch = 1;
 464                                 continue;
 465                         }
 466                         TAILQ_FOREACH(pe, wanted_branches, entry) {
 467                                 if (match_branch(refname, pe->path))
 468                                         break;
 469                         }
 470                         if (pe != NULL || (worktree_branch != NULL &&
 471                             match_branch(refname, worktree_branch))) {
 472                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 473                                     &want[nref], refname, id_str);
 474                                 if (err)
 475                                         goto done;
 476                                 nref++;
 477                                 found_branch = 1;
 478                         } else if (chattygot) {
 479                                 fprintf(stderr, "%s: ignoring %s\n",
 480                                     getprogname(), refname);
 481                         }
 482                 } else {
 483                         TAILQ_FOREACH(pe, wanted_refs, entry) {
 484                                 if (match_wanted_ref(refname, pe->path))
 485                                         break;
 486                         }
 487                         if (pe != NULL) {
 488                                 err = fetch_ref(ibuf, have_refs, &have[nref],
 489                                     &want[nref], refname, id_str);
 490                                 if (err)
 491                                         goto done;
 492                                 nref++;
 493                         } else if (chattygot) {
 494                                 fprintf(stderr, "%s: ignoring %s\n",
 495                                     getprogname(), refname);
 496                         }
 497                 }
 498         }
 499
 500         if (list_refs_only)
 501                 goto done;
 502
 503         /*
 504          * If -b was not used and either none of the requested branches
 505          * (got.conf, worktree) were found or the client already has the
 506          * remote HEAD symref but its target changed, fetch remote's HEAD.
 507          */
 508         if (!no_head && default_branch && default_id_str &&
 509             strncmp(default_branch, "refs/heads/", 11) == 0) {
 510                 int remote_head_changed = 0;
 511
 512                 if (remote_head) {
 513                         if (strcmp(remote_head, default_branch + 11) != 0)
 514                                 remote_head_changed = 1;
 515                 }
 516
 517                 if (!found_branch || remote_head_changed) {
 518                         err = fetch_ref(ibuf, have_refs, &have[nref],
 519                             &want[nref], default_branch, default_id_str);
 520                         if (err)
 521                                 goto done;
 522                         nref++;
 523                 }
 524         }
 525
 526         /* Abort if we haven't found anything to fetch. */
 527         if (nref == 0) {
 528                 struct got_pathlist_entry *pe;
 529                 static char msg[PATH_MAX + 33];
 530
 531                 pe = TAILQ_FIRST(wanted_branches);
 532                 if (pe) {
 533                         snprintf(msg, sizeof(msg),
 534                             "branch \"%s\" not found on server", pe->path);
 535                         err = got_error_msg(GOT_ERR_FETCH_NO_BRANCH, msg);
 536                         goto done;
 537                 }
 538
 539                 pe = TAILQ_FIRST(wanted_refs);
 540                 if (pe) {
 541                         snprintf(msg, sizeof(msg),
 542                             "reference \"%s\" not found on server", pe->path);
 543                         err = got_error_msg(GOT_ERR_FETCH_NO_BRANCH, msg);
 544                         goto done;
 545                 }
 546
 547                 err = got_error(GOT_ERR_FETCH_NO_BRANCH);
 548                 goto done;
 549         }
 550
 551         for (i = 0; i < nref; i++) {
 552                 if (got_object_id_cmp(&have[i], &want[i]) == 0)
 553                         continue;
 554                 got_sha1_digest_to_str(want[i].sha1, hashstr, sizeof(hashstr));
 555                 n = snprintf(buf, sizeof(buf), "want %s%s\n", hashstr,
 556                     sent_my_capabilites || my_capabilities == NULL ?
 557                     "" : my_capabilities);
 558                 if (n < 0 || (size_t)n >= sizeof(buf)) {
 559                         err = got_error(GOT_ERR_NO_SPACE);
 560                         goto done;
 561                 }
 562                 err = got_pkt_writepkt(fd, buf, n, chattygot);
 563                 if (err)
 564                         goto done;
 565                 sent_my_capabilites = 1;
 566                 nwant++;
 567         }
 568         err = got_pkt_flushpkt(fd, chattygot);
 569         if (err)
 570                 goto done;
 571
 572         if (nwant == 0)
 573                 goto done;
 574
 575         TAILQ_FOREACH(pe, have_refs, entry) {
 576                 struct got_object_id *id = pe->data;
 577                 got_sha1_digest_to_str(id->sha1, hashstr, sizeof(hashstr));
 578                 n = snprintf(buf, sizeof(buf), "have %s\n", hashstr);
 579                 if (n < 0 || (size_t)n >= sizeof(buf)) {
 580                         err = got_error(GOT_ERR_NO_SPACE);
 581                         goto done;
 582                 }
 583                 err = got_pkt_writepkt(fd, buf, n, chattygot);
 584                 if (err)
 585                         goto done;
 586                 nhave++;
 587         }
 588
 589         n = strlcpy(buf, "done\n", sizeof(buf));
 590         err = got_pkt_writepkt(fd, buf, n, chattygot);
 591         if (err)
 592                 goto done;
 593
 594         while (nhave > 0 && !acked) {
 595                 struct got_object_id common_id;
 596
 597                 /* The server should ACK the object IDs we need. */
 598                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot);
 599                 if (err)
 600                         goto done;
 601                 if (n >= 4 && strncmp(buf, "ERR ", 4) == 0) {
 602                         err = fetch_error(&buf[4], n - 4);
 603                         goto done;
 604                 }
 605                 if (n >= 4 && strncmp(buf, "NAK\n", 4) == 0) {
 606                         /*
 607                          * Server could not find a common ancestor.
 608                          * Perhaps it is an out-of-date mirror, or there
 609                          * is a repository with unrelated history.
 610                          */
 611                         break;
 612                 }
 613                 if (n < 4 + SHA1_DIGEST_STRING_LENGTH ||
 614                     strncmp(buf, "ACK ", 4) != 0) {
 615                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 616                             "unexpected message from server");
 617                         goto done;
 618                 }
 619                 if (!got_parse_object_id(&common_id, buf + 4,
 620                     GOT_HASH_SHA1)) {
 621                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 622                             "bad object ID in ACK packet from server");
 623                         goto done;
 624                 }
 625                 acked++;
 626         }
 627
 628         if (nhave == 0) {
 629                 err = got_pkt_readpkt(&n, fd, buf, sizeof(buf), chattygot);
 630                 if (err)
 631                         goto done;
 632                 if (n != 4 || strncmp(buf, "NAK\n", n) != 0) {
 633                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 634                             "unexpected message from server");
 635                         goto done;
 636                 }
 637         }
 638
 639         if (chattygot)
 640                 fprintf(stderr, "%s: fetching...\n", getprogname());
 641
 642         if (my_capabilities != NULL &&
 643             strstr(my_capabilities, GOT_CAPA_SIDE_BAND_64K) != NULL)
 644                 have_sidebands = 1;
 645
 646         while (1) {
 647                 ssize_t r = 0;
 648                 int datalen = -1;
 649
 650                 if (have_sidebands) {
 651                         err = got_pkt_readhdr(&datalen, fd, chattygot);
 652                         if (err)
 653                                 goto done;
 654                         if (datalen <= 0)
 655                                 break;
 656
 657                         /* Read sideband channel ID (one byte). */
 658                         r = read(fd, buf, 1);
 659                         if (r == -1) {
 660                                 err = got_error_from_errno("read");
 661                                 goto done;
 662                         }
 663                         if (r != 1) {
 664                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 665                                     "short packet");
 666                                 goto done;
 667                         }
 668                         if (datalen > sizeof(buf) - 5) {
 669                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 670                                     "bad packet length");
 671                                 goto done;
 672                         }
 673                         datalen--; /* sideband ID has been read */
 674                         if (buf[0] == GOT_SIDEBAND_PACKFILE_DATA) {
 675                                 /* Read packfile data. */
 676                                 err = got_pkt_readn(&r, fd, buf, datalen);
 677                                 if (err)
 678                                         goto done;
 679                                 if (r != datalen) {
 680                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 681                                             "packet too short");
 682                                         goto done;
 683                                 }
 684                         } else if (buf[0] == GOT_SIDEBAND_PROGRESS_INFO) {
 685                                 err = got_pkt_readn(&r, fd, buf, datalen);
 686                                 if (err)
 687                                         goto done;
 688                                 if (r != datalen) {
 689                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 690                                             "packet too short");
 691                                         goto done;
 692                                 }
 693                                 err = fetch_progress(ibuf, buf, r);
 694                                 if (err)
 695                                         goto done;
 696                                 continue;
 697                         } else if (buf[0] == GOT_SIDEBAND_ERROR_INFO) {
 698                                 err = got_pkt_readn(&r, fd, buf, datalen);
 699                                 if (err)
 700                                         goto done;
 701                                 if (r != datalen) {
 702                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 703                                             "packet too short");
 704                                         goto done;
 705                                 }
 706                                 err = fetch_error(buf, r);
 707                                 goto done;
 708                         } else if (buf[0] == 'A') {
 709                                 err = got_pkt_readn(&r, fd, buf, datalen);
 710                                 if (err)
 711                                         goto done;
 712                                 if (r != datalen) {
 713                                         err = got_error_msg(GOT_ERR_BAD_PACKET,
 714                                             "packet too short");
 715                                         goto done;
 716                                 }
 717                                 /*
 718                                  * Git server responds with ACK after 'done'
 719                                  * even though multi_ack is disabled?!?
 720                                  */
 721                                 buf[r] = '\0';
 722                                 if (strncmp(buf, "CK ", 3) == 0)
 723                                         continue; /* ignore */
 724                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 725                                     "unexpected message from server");
 726                                 goto done;
 727                         } else {
 728                                 err = got_error_msg(GOT_ERR_BAD_PACKET,
 729                                     "unknown side-band received from server");
 730                                 goto done;
 731                         }
 732                 } else {
 733                         /* No sideband channel. Every byte is packfile data. */
 734                         err = got_pkt_readn(&r, fd, buf, sizeof buf);
 735                         if (err)
 736                                 goto done;
 737                         if (r <= 0)
 738                                 break;
 739                 }
 740
 741                 /*
 742                  * An expected SHA1 checksum sits at the end of the pack file.
 743                  * Since we don't know the file size ahead of time we have to
 744                  * keep SHA1_DIGEST_LENGTH bytes buffered and avoid mixing
 745                  * those bytes into our SHA1 checksum computation until we
 746                  * know for sure that additional pack file data bytes follow.
 747                  *
 748                  * We can assume r > 0 since otherwise the loop would exit.
 749                  */
 750                 if (r < SHA1_DIGEST_LENGTH) {
 751                         if (sha1_buf_len < SHA1_DIGEST_LENGTH) {
 752                                 /*
 753                                  * If there's enough buffered + read data to
 754                                  * fill up the buffer then shift a sufficient
 755                                  * amount of bytes out at the front to make
 756                                  * room, mixing those bytes into the checksum.
 757                                  */
 758                                 if (sha1_buf_len > 0 &&
 759                                     sha1_buf_len + r > SHA1_DIGEST_LENGTH) {
 760                                         size_t nshift = MIN(sha1_buf_len + r -
 761                                             SHA1_DIGEST_LENGTH, sha1_buf_len);
 762                                         got_hash_update(&ctx, sha1_buf,
 763                                             nshift);
 764                                         memmove(sha1_buf, sha1_buf + nshift,
 765                                             sha1_buf_len - nshift);
 766                                         sha1_buf_len -= nshift;
 767                                 }
 768
 769                                 /* Buffer potential checksum bytes. */
 770                                 memcpy(sha1_buf + sha1_buf_len, buf, r);
 771                                 sha1_buf_len += r;
 772                         } else {
 773                                 /*
 774                                  * Mix in previously buffered bytes which
 775                                  * are not part of the checksum after all.
 776                                  */
 777                                 got_hash_update(&ctx, sha1_buf, r);
 778
 779                                 /* Update potential checksum buffer. */
 780                                 memmove(sha1_buf, sha1_buf + r,
 781                                     sha1_buf_len - r);
 782                                 memcpy(sha1_buf + sha1_buf_len - r, buf, r);
 783                         }
 784                 } else {
 785                         /* Mix in any previously buffered bytes. */
 786                         got_hash_update(&ctx, sha1_buf, sha1_buf_len);
 787
 788                         /* Mix in bytes read minus potential checksum bytes. */
 789                         got_hash_update(&ctx, buf, r - SHA1_DIGEST_LENGTH);
 790
 791                         /* Buffer potential checksum bytes. */
 792                         memcpy(sha1_buf, buf + r - SHA1_DIGEST_LENGTH,
 793                             SHA1_DIGEST_LENGTH);
 794                         sha1_buf_len = SHA1_DIGEST_LENGTH;
 795                 }
 796
 797                 /* Write packfile data to temporary pack file. */
 798                 w = write(packfd, buf, r);
 799                 if (w == -1) {
 800                         err = got_error_from_errno("write");
 801                         goto done;
 802                 }
 803                 if (w != r) {
 804                         err = got_error(GOT_ERR_IO);
 805                         goto done;
 806                 }
 807                 packsz += w;
 808
 809                 /* Don't send too many progress privsep messages. */
 810                 if (packsz > last_reported_packsz + 1024) {
 811                         err = send_fetch_download_progress(ibuf, packsz, &rl);
 812                         if (err)
 813                                 goto done;
 814                         last_reported_packsz = packsz;
 815                 }
 816         }
 817         err = send_fetch_download_progress(ibuf, packsz, NULL);
 818         if (err)
 819                 goto done;
 820
 821         got_hash_final(&ctx, pack_sha1);
 822         if (sha1_buf_len != SHA1_DIGEST_LENGTH ||
 823             memcmp(pack_sha1, sha1_buf, sha1_buf_len) != 0) {
 824                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 825                     "pack file checksum mismatch");
 826         }
 827 done:
 828         got_pathlist_free(&symrefs, GOT_PATHLIST_FREE_ALL);
 829         free(have);
 830         free(want);
 831         free(id_str);
 832         free(default_id_str);
 833         free(refname);
 834         free(server_capabilities);
 835         return err;
 836 }
 837
 838
 839 int
 840 main(int argc, char **argv)
 841 {
 842         const struct got_error *err = NULL;
 843         int fetchfd = -1, packfd = -1;
 844         uint8_t pack_sha1[SHA1_DIGEST_LENGTH];
 845         struct imsgbuf ibuf;
 846         struct imsg imsg;
 847         struct got_pathlist_head have_refs;
 848         struct got_pathlist_head wanted_branches;
 849         struct got_pathlist_head wanted_refs;
 850         struct got_imsg_fetch_request fetch_req;
 851         struct got_imsg_fetch_have_ref href;
 852         struct got_imsg_fetch_wanted_branch wbranch;
 853         struct got_imsg_fetch_wanted_ref wref;
 854         size_t datalen, i;
 855         char *remote_head = NULL, *worktree_branch = NULL;
 856 #if 0
 857         static int attached;
 858         while (!attached)
 859                 sleep (1);
 860 #endif
 861
 862         TAILQ_INIT(&have_refs);
 863         TAILQ_INIT(&wanted_branches);
 864         TAILQ_INIT(&wanted_refs);
 865
 866         imsg_init(&ibuf, GOT_IMSG_FD_CHILD);
 867 #ifndef PROFILE
 868         /* revoke access to most system calls */
 869         if (pledge("stdio recvfd", NULL) == -1) {
 870                 err = got_error_from_errno("pledge");
 871                 got_privsep_send_error(&ibuf, err);
 872                 return 1;
 873         }
 874
 875         /* revoke fs access */
 876         if (landlock_no_fs() == -1) {
 877                 err = got_error_from_errno("landlock_no_fs");
 878                 got_privsep_send_error(&ibuf, err);
 879                 return 1;
 880         }
 881         if (cap_enter() == -1) {
 882                 err = got_error_from_errno("cap_enter");
 883                 got_privsep_send_error(&ibuf, err);
 884                 return 1;
 885         }
 886 #endif
 887         err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
 888         if (err) {
 889                 if (err->code == GOT_ERR_PRIVSEP_PIPE)
 890                         err = NULL;
 891                 goto done;
 892         }
 893         if (imsg.hdr.type == GOT_IMSG_STOP)
 894                 goto done;
 895         if (imsg.hdr.type != GOT_IMSG_FETCH_REQUEST) {
 896                 err = got_error(GOT_ERR_PRIVSEP_MSG);
 897                 goto done;
 898         }
 899         datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
 900         if (datalen < sizeof(fetch_req)) {
 901                 err = got_error(GOT_ERR_PRIVSEP_LEN);
 902                 goto done;
 903         }
 904         memcpy(&fetch_req, imsg.data, sizeof(fetch_req));
 905         fetchfd = imsg_get_fd(&imsg);
 906
 907         if (datalen != sizeof(fetch_req) +
 908             fetch_req.worktree_branch_len + fetch_req.remote_head_len) {
 909                 err = got_error(GOT_ERR_PRIVSEP_LEN);
 910                 goto done;
 911         }
 912
 913         if (fetch_req.worktree_branch_len != 0) {
 914                 worktree_branch = strndup(imsg.data +
 915                     sizeof(fetch_req), fetch_req.worktree_branch_len);
 916                 if (worktree_branch == NULL) {
 917                         err = got_error_from_errno("strndup");
 918                         goto done;
 919                 }
 920         }
 921
 922         if (fetch_req.remote_head_len != 0) {
 923                 remote_head = strndup(imsg.data + sizeof(fetch_req) +
 924                     fetch_req.worktree_branch_len, fetch_req.remote_head_len);
 925                 if (remote_head == NULL) {
 926                         err = got_error_from_errno("strndup");
 927                         goto done;
 928                 }
 929         }
 930
 931         imsg_free(&imsg);
 932
 933         if (fetch_req.verbosity > 0)
 934                 chattygot += fetch_req.verbosity;
 935
 936         for (i = 0; i < fetch_req.n_have_refs; i++) {
 937                 struct got_object_id *id;
 938                 char *refname;
 939
 940                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
 941                 if (err) {
 942                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
 943                                 err = NULL;
 944                         goto done;
 945                 }
 946                 if (imsg.hdr.type == GOT_IMSG_STOP)
 947                         goto done;
 948                 if (imsg.hdr.type != GOT_IMSG_FETCH_HAVE_REF) {
 949                         err = got_error(GOT_ERR_PRIVSEP_MSG);
 950                         goto done;
 951                 }
 952                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
 953                 if (datalen < sizeof(href)) {
 954                         err = got_error(GOT_ERR_PRIVSEP_LEN);
 955                         goto done;
 956                 }
 957                 memcpy(&href, imsg.data, sizeof(href));
 958                 if (datalen - sizeof(href) < href.name_len) {
 959                         err = got_error(GOT_ERR_PRIVSEP_LEN);
 960                         goto done;
 961                 }
 962                 refname = strndup(imsg.data + sizeof(href), href.name_len);
 963                 if (refname == NULL) {
 964                         err = got_error_from_errno("strndup");
 965                         goto done;
 966                 }
 967
 968                 id = malloc(sizeof(*id));
 969                 if (id == NULL) {
 970                         free(refname);
 971                         err = got_error_from_errno("malloc");
 972                         goto done;
 973                 }
 974                 memcpy(id, &href.id, sizeof(*id));
 975                 err = got_pathlist_append(&have_refs, refname, id);
 976                 if (err) {
 977                         free(refname);
 978                         free(id);
 979                         goto done;
 980                 }
 981
 982                 imsg_free(&imsg);
 983         }
 984
 985         for (i = 0; i < fetch_req.n_wanted_branches; i++) {
 986                 char *refname;
 987
 988                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
 989                 if (err) {
 990                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
 991                                 err = NULL;
 992                         goto done;
 993                 }
 994                 if (imsg.hdr.type == GOT_IMSG_STOP)
 995                         goto done;
 996                 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_BRANCH) {
 997                         err = got_error(GOT_ERR_PRIVSEP_MSG);
 998                         goto done;
 999                 }
1000                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1001                 if (datalen < sizeof(wbranch)) {
1002                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1003                         goto done;
1004                 }
1005                 memcpy(&wbranch, imsg.data, sizeof(wbranch));
1006                 if (datalen - sizeof(wbranch) < wbranch.name_len) {
1007                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1008                         goto done;
1009                 }
1010                 refname = strndup(imsg.data + sizeof(wbranch),
1011                     wbranch.name_len);
1012                 if (refname == NULL) {
1013                         err = got_error_from_errno("strndup");
1014                         goto done;
1015                 }
1016
1017                 err = got_pathlist_append(&wanted_branches, refname, NULL);
1018                 if (err) {
1019                         free(refname);
1020                         goto done;
1021                 }
1022
1023                 imsg_free(&imsg);
1024         }
1025
1026         for (i = 0; i < fetch_req.n_wanted_refs; i++) {
1027                 char *refname;
1028
1029                 err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1030                 if (err) {
1031                         if (err->code == GOT_ERR_PRIVSEP_PIPE)
1032                                 err = NULL;
1033                         goto done;
1034                 }
1035                 if (imsg.hdr.type == GOT_IMSG_STOP)
1036                         goto done;
1037                 if (imsg.hdr.type != GOT_IMSG_FETCH_WANTED_REF) {
1038                         err = got_error(GOT_ERR_PRIVSEP_MSG);
1039                         goto done;
1040                 }
1041                 datalen = imsg.hdr.len - IMSG_HEADER_SIZE;
1042                 if (datalen < sizeof(wref)) {
1043                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1044                         goto done;
1045                 }
1046                 memcpy(&wref, imsg.data, sizeof(wref));
1047                 if (datalen - sizeof(wref) < wref.name_len) {
1048                         err = got_error(GOT_ERR_PRIVSEP_LEN);
1049                         goto done;
1050                 }
1051                 refname = strndup(imsg.data + sizeof(wref), wref.name_len);
1052                 if (refname == NULL) {
1053                         err = got_error_from_errno("strndup");
1054                         goto done;
1055                 }
1056
1057                 err = got_pathlist_append(&wanted_refs, refname, NULL);
1058                 if (err) {
1059                         free(refname);
1060                         goto done;
1061                 }
1062
1063                 imsg_free(&imsg);
1064         }
1065
1066         err = got_privsep_recv_imsg(&imsg, &ibuf, 0);
1067         if (err) {
1068                 if (err->code == GOT_ERR_PRIVSEP_PIPE)
1069                         err = NULL;
1070                 goto done;
1071         }
1072         if (imsg.hdr.type == GOT_IMSG_STOP)
1073                 goto done;
1074         if (imsg.hdr.type != GOT_IMSG_FETCH_OUTFD) {
1075                 err = got_error(GOT_ERR_PRIVSEP_MSG);
1076                 goto done;
1077         }
1078         if (imsg.hdr.len - IMSG_HEADER_SIZE != 0) {
1079                 err = got_error(GOT_ERR_PRIVSEP_LEN);
1080                 goto done;
1081         }
1082         packfd = imsg_get_fd(&imsg);
1083
1084         err = fetch_pack(fetchfd, packfd, pack_sha1, &have_refs,
1085             fetch_req.fetch_all_branches, &wanted_branches,
1086             &wanted_refs, fetch_req.list_refs_only,
1087             worktree_branch, remote_head, fetch_req.no_head, &ibuf);
1088 done:
1089         free(worktree_branch);
1090         free(remote_head);
1091         got_pathlist_free(&have_refs, GOT_PATHLIST_FREE_ALL);
1092         got_pathlist_free(&wanted_branches, GOT_PATHLIST_FREE_PATH);
1093         if (fetchfd != -1 && close(fetchfd) == -1 && err == NULL)
1094                 err = got_error_from_errno("close");
1095         if (packfd != -1 && close(packfd) == -1 && err == NULL)
1096                 err = got_error_from_errno("close");
1097         if (err != NULL)
1098                 got_privsep_send_error(&ibuf, err);
1099         else
1100                 err = send_fetch_done(&ibuf, pack_sha1);
1101         if (err != NULL) {
1102                 fprintf(stderr, "%s: %s\n", getprogname(), err->msg);
1103                 got_privsep_send_error(&ibuf, err);
1104         }
1105
1106         exit(0);
1107 }