lib/fetch.c

   1 /*
   2  * Copyright (c) 2018, 2019 Ori Bernstein <ori@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16 #include "got_compat.h"
  17
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/socket.h>
  23 #include <sys/wait.h>
  24 #include <sys/resource.h>
  25 #include <sys/socket.h>
  26
  27 #include <errno.h>
  28 #include <err.h>
  29 #include <fcntl.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <stdint.h>
  34 #include <unistd.h>
  35 #include <zlib.h>
  36 #include <ctype.h>
  37 #include <limits.h>
  38 #include <time.h>
  39
  40 #include "got_error.h"
  41 #include "got_reference.h"
  42 #include "got_repository.h"
  43 #include "got_path.h"
  44 #include "got_cancel.h"
  45 #include "got_worktree.h"
  46 #include "got_object.h"
  47 #include "got_opentemp.h"
  48 #include "got_fetch.h"
  49
  50 #include "got_lib_hash.h"
  51 #include "got_lib_delta.h"
  52 #include "got_lib_inflate.h"
  53 #include "got_lib_object.h"
  54 #include "got_lib_object_parse.h"
  55 #include "got_lib_object_create.h"
  56 #include "got_lib_pack.h"
  57 #include "got_lib_privsep.h"
  58 #include "got_lib_object_cache.h"
  59 #include "got_lib_repository.h"
  60 #include "got_lib_dial.h"
  61 #include "got_lib_pkt.h"
  62
  63 #ifndef nitems
  64 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  65 #endif
  66
  67 #ifndef ssizeof
  68 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
  69 #endif
  70
  71 #ifndef MIN
  72 #define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
  73 #endif
  74
  75 const struct got_error *
  76 got_fetch_connect(pid_t *fetchpid, int *fetchfd, const char *proto,
  77     const char *host, const char *port, const char *server_path,
  78     const char *jumphost, int verbosity)
  79 {
  80         const struct got_error *err = NULL;
  81
  82         *fetchpid = -1;
  83         *fetchfd = -1;
  84
  85         if (strcmp(proto, "ssh") == 0 || strcmp(proto, "git+ssh") == 0)
  86                 err = got_dial_ssh(fetchpid, fetchfd, host, port,
  87                     server_path, jumphost, GOT_DIAL_CMD_FETCH, verbosity);
  88         else if (strcmp(proto, "git") == 0)
  89                 err = got_dial_git(fetchfd, host, port, server_path,
  90                     GOT_DIAL_CMD_FETCH);
  91         else if (strcmp(proto, "http") == 0 ||
  92             strcmp(proto, "git+http") == 0 ||
  93             strcmp(proto, "https") == 0 ||
  94             strcmp(proto, "git+https") == 0)
  95                 err = got_dial_http(fetchpid, fetchfd, host, port,
  96                     server_path, verbosity, strstr(proto, "https") != NULL);
  97         else
  98                 err = got_error_path(proto, GOT_ERR_BAD_PROTO);
  99         return err;
 100 }
 101
 102 const struct got_error *
 103 got_fetch_pack(struct got_object_id **pack_hash, struct got_pathlist_head *refs,
 104     struct got_pathlist_head *symrefs, const char *remote_name,
 105     int mirror_references, int fetch_all_branches,
 106     struct got_pathlist_head *wanted_branches,
 107     struct got_pathlist_head *wanted_refs, int list_refs_only, int verbosity,
 108     int fetchfd, struct got_repository *repo, const char *worktree_refname,
 109     const char *remote_head, int no_head, got_fetch_progress_cb progress_cb,
 110     void *progress_arg)
 111 {
 112         size_t i;
 113         int imsg_fetchfds[2], imsg_idxfds[2];
 114         int packfd = -1, npackfd = -1, idxfd = -1, nidxfd = -1, nfetchfd = -1;
 115         int tmpfds[3];
 116         int fetchstatus, idxstatus, done = 0;
 117         const struct got_error *err;
 118         struct imsgbuf fetchibuf, idxibuf;
 119         pid_t fetchpid, idxpid;
 120         char *tmppackpath = NULL, *tmpidxpath = NULL;
 121         char *packpath = NULL, *idxpath = NULL, *id_str = NULL;
 122         const char *repo_path = NULL;
 123         struct got_pathlist_head have_refs;
 124         struct got_pathlist_entry *pe, *new;
 125         struct got_reflist_head my_refs;
 126         struct got_reflist_entry *re;
 127         off_t packfile_size = 0;
 128         struct got_packfile_hdr pack_hdr;
 129         uint32_t nobj = 0;
 130         char *path;
 131         char *progress = NULL;
 132
 133         *pack_hash = NULL;
 134         memset(&fetchibuf, 0, sizeof(fetchibuf));
 135         memset(&idxibuf, 0, sizeof(idxibuf));
 136
 137         if (repo && got_repo_get_object_format(repo) != GOT_HASH_SHA1)
 138                 return got_error_fmt(GOT_ERR_NOT_IMPL,
 139                     "sha256 object IDs unsupported in network protocol");
 140
 141         /*
 142          * Prevent fetching of references that won't make any
 143          * sense outside of the remote repository's context.
 144          */
 145         RB_FOREACH(pe, got_pathlist_head, wanted_refs) {
 146                 const char *refname = pe->path;
 147                 if (strncmp(refname, "refs/got/", 9) == 0 ||
 148                     strncmp(refname, "got/", 4) == 0 ||
 149                     strncmp(refname, "refs/remotes/", 13) == 0 ||
 150                     strncmp(refname, "remotes/", 8) == 0)
 151                         return got_error_path(refname, GOT_ERR_FETCH_BAD_REF);
 152         }
 153
 154         if (!list_refs_only)
 155                 repo_path = got_repo_get_path_git_dir(repo);
 156
 157         for (i = 0; i < nitems(tmpfds); i++)
 158                 tmpfds[i] = -1;
 159
 160         RB_INIT(&have_refs);
 161         TAILQ_INIT(&my_refs);
 162
 163         if (!list_refs_only) {
 164                 err = got_ref_list(&my_refs, repo, NULL,
 165                     got_ref_cmp_by_name, NULL);
 166                 if (err)
 167                         goto done;
 168         }
 169
 170         TAILQ_FOREACH(re, &my_refs, entry) {
 171                 struct got_object_id *id;
 172                 char *refname;
 173
 174                 if (got_ref_is_symbolic(re->ref))
 175                         continue;
 176
 177                 err = got_ref_resolve(&id, repo, re->ref);
 178                 if (err)
 179                         goto done;
 180                 refname = strdup(got_ref_get_name(re->ref));
 181                 if (refname == NULL) {
 182                         err = got_error_from_errno("strdup");
 183                         goto done;
 184                 }
 185                 err = got_pathlist_insert(&new, &have_refs, refname, id);
 186                 if (err)
 187                         goto done;
 188                 if (new == NULL){
 189                         free(refname);
 190                         free(id);
 191                 }
 192
 193         }
 194
 195         if (list_refs_only) {
 196                 packfd = got_opentempfd();
 197                 if (packfd == -1) {
 198                         err = got_error_from_errno("got_opentempfd");
 199                         goto done;
 200                 }
 201         } else {
 202                 if (asprintf(&path, "%s/%s/fetching.pack",
 203                     repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
 204                         err = got_error_from_errno("asprintf");
 205                         goto done;
 206                 }
 207                 err = got_opentemp_named_fd(&tmppackpath, &packfd, path, "");
 208                 free(path);
 209                 if (err)
 210                         goto done;
 211                 if (fchmod(packfd, GOT_DEFAULT_FILE_MODE) != 0) {
 212                         err = got_error_from_errno2("fchmod", tmppackpath);
 213                         goto done;
 214                 }
 215         }
 216         if (list_refs_only) {
 217                 idxfd = got_opentempfd();
 218                 if (idxfd == -1) {
 219                         err = got_error_from_errno("got_opentempfd");
 220                         goto done;
 221                 }
 222         } else {
 223                 if (asprintf(&path, "%s/%s/fetching.idx",
 224                     repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
 225                         err = got_error_from_errno("asprintf");
 226                         goto done;
 227                 }
 228                 err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path, "");
 229                 free(path);
 230                 if (err)
 231                         goto done;
 232                 if (fchmod(idxfd, GOT_DEFAULT_FILE_MODE) != 0) {
 233                         err = got_error_from_errno2("fchmod", tmpidxpath);
 234                         goto done;
 235                 }
 236         }
 237         nidxfd = dup(idxfd);
 238         if (nidxfd == -1) {
 239                 err = got_error_from_errno("dup");
 240                 goto done;
 241         }
 242
 243         for (i = 0; i < nitems(tmpfds); i++) {
 244                 tmpfds[i] = got_opentempfd();
 245                 if (tmpfds[i] == -1) {
 246                         err = got_error_from_errno("got_opentempfd");
 247                         goto done;
 248                 }
 249         }
 250
 251         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_fetchfds) == -1) {
 252                 err = got_error_from_errno("socketpair");
 253                 goto done;
 254         }
 255
 256         fetchpid = fork();
 257         if (fetchpid == -1) {
 258                 err = got_error_from_errno("fork");
 259                 goto done;
 260         } else if (fetchpid == 0){
 261                 got_privsep_exec_child(imsg_fetchfds,
 262                     GOT_PATH_PROG_FETCH_PACK, tmppackpath);
 263         }
 264
 265         if (close(imsg_fetchfds[1]) == -1) {
 266                 err = got_error_from_errno("close");
 267                 goto done;
 268         }
 269         if (imsgbuf_init(&fetchibuf, imsg_fetchfds[0]) == -1) {
 270                 err = got_error_from_errno("imsgbuf_init");
 271                 goto done;
 272         }
 273         imsgbuf_allow_fdpass(&fetchibuf);
 274         nfetchfd = dup(fetchfd);
 275         if (nfetchfd == -1) {
 276                 err = got_error_from_errno("dup");
 277                 goto done;
 278         }
 279         err = got_privsep_send_fetch_req(&fetchibuf, nfetchfd, &have_refs,
 280             fetch_all_branches, wanted_branches, wanted_refs,
 281             list_refs_only, worktree_refname, remote_head, no_head, verbosity);
 282         if (err != NULL)
 283                 goto done;
 284         nfetchfd = -1;
 285         npackfd = dup(packfd);
 286         if (npackfd == -1) {
 287                 err = got_error_from_errno("dup");
 288                 goto done;
 289         }
 290         err = got_privsep_send_fetch_outfd(&fetchibuf, npackfd);
 291         if (err != NULL)
 292                 goto done;
 293         npackfd = -1;
 294
 295         packfile_size = 0;
 296         progress = calloc(GOT_PKT_MAX, 1);
 297         if (progress == NULL) {
 298                 err = got_error_from_errno("calloc");
 299                 goto done;
 300         }
 301
 302         *pack_hash = calloc(1, sizeof(**pack_hash));
 303         if (*pack_hash == NULL) {
 304                 err = got_error_from_errno("calloc");
 305                 goto done;
 306         }
 307
 308         while (!done) {
 309                 struct got_object_id *id = NULL;
 310                 char *refname = NULL;
 311                 char *server_progress = NULL;
 312                 off_t packfile_size_cur = 0;
 313
 314                 err = got_privsep_recv_fetch_progress(&done,
 315                     &id, &refname, symrefs, &server_progress,
 316                     &packfile_size_cur, (*pack_hash)->hash, &fetchibuf);
 317                 if (err != NULL)
 318                         goto done;
 319                 /* Don't report size progress for an empty pack file. */
 320                 if (packfile_size_cur <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH)
 321                         packfile_size_cur = 0;
 322                 if (!done && refname && id) {
 323                         err = got_pathlist_insert(&pe, refs, refname, id);
 324                         if (err || pe == NULL) {
 325                                 free(id);
 326                                 free(refname);
 327                                 if (err != NULL)
 328                                         goto done;
 329                         }
 330                 } else if (!done && server_progress) {
 331                         char *p;
 332                         /*
 333                          * XXX git-daemon tends to send batched output with
 334                          * lines spanning separate packets. Buffer progress
 335                          * output until we see a CR or LF to avoid giving
 336                          * partial lines of progress output to the callback.
 337                          */
 338                         if (strlcat(progress, server_progress,
 339                             GOT_PKT_MAX) >= GOT_PKT_MAX) {
 340                                 progress[0] = '\0'; /* discard */
 341                                 continue;
 342                         }
 343                         while ((p = strchr(progress, '\r')) != NULL ||
 344                             (p = strchr(progress, '\n')) != NULL) {
 345                                 char *s;
 346                                 size_t n;
 347                                 char c = *p;
 348                                 *p = '\0';
 349                                 if (asprintf(&s, "%s%s", progress,
 350                                     c == '\n' ? "\n" : "") == -1) {
 351                                         err = got_error_from_errno("asprintf");
 352                                         goto done;
 353                                 }
 354                                 err = progress_cb(progress_arg, s,
 355                                     packfile_size_cur, 0, 0, 0, 0);
 356                                 free(s);
 357                                 if (err)
 358                                         break;
 359                                 n = strlen(progress);
 360                                 if (n < GOT_PKT_MAX - 1) {
 361                                         memmove(progress, &progress[n + 1],
 362                                             GOT_PKT_MAX - n - 1);
 363                                 } else
 364                                         progress[0] = '\0';
 365                         }
 366                         free(server_progress);
 367                         if (err)
 368                                 goto done;
 369                 } else if (!done && packfile_size_cur != packfile_size) {
 370                         err = progress_cb(progress_arg, NULL,
 371                             packfile_size_cur, 0, 0, 0, 0);
 372                         if (err)
 373                                 break;
 374                         packfile_size = packfile_size_cur;
 375                 }
 376         }
 377         if (close(imsg_fetchfds[0]) == -1) {
 378                 err = got_error_from_errno("close");
 379                 goto done;
 380         }
 381         if (waitpid(fetchpid, &fetchstatus, 0) == -1) {
 382                 err = got_error_from_errno("waitpid");
 383                 goto done;
 384         }
 385
 386         if (lseek(packfd, 0, SEEK_SET) == -1) {
 387                 err = got_error_from_errno("lseek");
 388                 goto done;
 389         }
 390
 391         /* If zero data was fetched without error we are already up-to-date. */
 392         if (packfile_size == 0) {
 393                 free(*pack_hash);
 394                 *pack_hash = NULL;
 395                 goto done;
 396         } else if (packfile_size < ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 397                 err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
 398                 goto done;
 399         } else {
 400                 ssize_t n;
 401
 402                 n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
 403                 if (n == -1) {
 404                         err = got_error_from_errno("read");
 405                         goto done;
 406                 }
 407                 if (n != ssizeof(pack_hdr)) {
 408                         err = got_error(GOT_ERR_IO);
 409                         goto done;
 410                 }
 411                 if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
 412                         err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 413                             "bad pack file signature");
 414                         goto done;
 415                 }
 416                 if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
 417                         err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 418                             "bad pack file version");
 419                         goto done;
 420                 }
 421                 nobj = be32toh(pack_hdr.nobjects);
 422                 if (nobj == 0 &&
 423                     packfile_size > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 424                         err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 425                             "bad pack file with zero objects");
 426                         goto done;
 427                 }
 428                 if (nobj != 0 &&
 429                     packfile_size <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 430                         err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 431                             "empty pack file with non-zero object count");
 432                         goto done;
 433                 }
 434         }
 435
 436         /*
 437          * If the pack file contains no objects, we may only need to update
 438          * references in our repository. The caller will take care of that.
 439          */
 440         if (nobj == 0) {
 441                 free(*pack_hash);
 442                 *pack_hash = NULL;
 443                 goto done;
 444         }
 445
 446         if (lseek(packfd, 0, SEEK_SET) == -1) {
 447                 err = got_error_from_errno("lseek");
 448                 goto done;
 449         }
 450
 451         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 452                 err = got_error_from_errno("socketpair");
 453                 goto done;
 454         }
 455         idxpid = fork();
 456         if (idxpid == -1) {
 457                 err= got_error_from_errno("fork");
 458                 goto done;
 459         } else if (idxpid == 0)
 460                 got_privsep_exec_child(imsg_idxfds,
 461                     GOT_PATH_PROG_INDEX_PACK, tmppackpath);
 462         if (close(imsg_idxfds[1]) == -1) {
 463                 err = got_error_from_errno("close");
 464                 goto done;
 465         }
 466         if (imsgbuf_init(&idxibuf, imsg_idxfds[0]) == -1) {
 467                 err = got_error_from_errno("imsgbuf_init");
 468                 goto done;
 469         }
 470         imsgbuf_allow_fdpass(&idxibuf);
 471
 472         npackfd = dup(packfd);
 473         if (npackfd == -1) {
 474                 err = got_error_from_errno("dup");
 475                 goto done;
 476         }
 477         err = got_privsep_send_index_pack_req(&idxibuf, *pack_hash, npackfd);
 478         if (err != NULL)
 479                 goto done;
 480         npackfd = -1;
 481         err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
 482         if (err != NULL)
 483                 goto done;
 484         nidxfd = -1;
 485         for (i = 0; i < nitems(tmpfds); i++) {
 486                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 487                 if (err != NULL)
 488                         goto done;
 489                 tmpfds[i] = -1;
 490         }
 491         done = 0;
 492         while (!done) {
 493                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 494
 495                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 496                     &nobj_indexed, &nobj_loose, &nobj_resolved,
 497                     &idxibuf);
 498                 if (err != NULL)
 499                         goto done;
 500                 if (nobj_indexed != 0) {
 501                         err = progress_cb(progress_arg, NULL,
 502                             packfile_size, nobj_total,
 503                             nobj_indexed, nobj_loose, nobj_resolved);
 504                         if (err)
 505                                 break;
 506                 }
 507         }
 508         if (close(imsg_idxfds[0]) == -1) {
 509                 err = got_error_from_errno("close");
 510                 goto done;
 511         }
 512         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 513                 err = got_error_from_errno("waitpid");
 514                 goto done;
 515         }
 516
 517         err = got_object_id_str(&id_str, *pack_hash);
 518         if (err)
 519                 goto done;
 520         if (asprintf(&packpath, "%s/%s/pack-%s.pack",
 521             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 522                 err = got_error_from_errno("asprintf");
 523                 goto done;
 524         }
 525
 526         if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
 527             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 528                 err = got_error_from_errno("asprintf");
 529                 goto done;
 530         }
 531         free(id_str);
 532         id_str = NULL;
 533
 534         if (rename(tmppackpath, packpath) == -1) {
 535                 err = got_error_from_errno3("rename", tmppackpath, packpath);
 536                 goto done;
 537         }
 538         free(tmppackpath);
 539         tmppackpath = NULL;
 540         if (rename(tmpidxpath, idxpath) == -1) {
 541                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 542                 goto done;
 543         }
 544         free(tmpidxpath);
 545         tmpidxpath = NULL;
 546
 547 done:
 548         if (fetchibuf.w)
 549                 imsgbuf_clear(&fetchibuf);
 550         if (idxibuf.w)
 551                 imsgbuf_clear(&idxibuf);
 552         if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
 553                 err = got_error_from_errno2("unlink", tmppackpath);
 554         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 555                 err = got_error_from_errno2("unlink", tmpidxpath);
 556         if (nfetchfd != -1 && close(nfetchfd) == -1 && err == NULL)
 557                 err = got_error_from_errno("close");
 558         if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
 559                 err = got_error_from_errno("close");
 560         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 561                 err = got_error_from_errno("close");
 562         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 563                 err = got_error_from_errno("close");
 564         for (i = 0; i < nitems(tmpfds); i++) {
 565                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 566                         err = got_error_from_errno("close");
 567         }
 568         free(tmppackpath);
 569         free(tmpidxpath);
 570         free(idxpath);
 571         free(id_str);
 572         free(packpath);
 573         free(progress);
 574
 575         got_pathlist_free(&have_refs, GOT_PATHLIST_FREE_ALL);
 576         got_ref_list_free(&my_refs);
 577         if (err) {
 578                 free(*pack_hash);
 579                 *pack_hash = NULL;
 580                 got_pathlist_free(refs, GOT_PATHLIST_FREE_ALL);
 581                 got_pathlist_free(symrefs, GOT_PATHLIST_FREE_ALL);
 582         }
 583         return err;
 584 }