lib/load.c

   1 /*
   2  * Copyright (c) 2023 Omar Polo <op@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/time.h>
  23 #include <sys/types.h>
  24 #include <sys/uio.h>
  25 #include <sys/wait.h>
  26
  27 #include <limits.h>
  28 #include <stdint.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <imsg.h>
  34
  35 #include "got_error.h"
  36 #include "got_cancel.h"
  37 #include "got_object.h"
  38 #include "got_opentemp.h"
  39 #include "got_path.h"
  40 #include "got_reference.h"
  41 #include "got_repository.h"
  42 #include "got_repository_load.h"
  43
  44 #include "got_lib_delta.h"
  45 #include "got_lib_hash.h"
  46 #include "got_lib_object.h"
  47 #include "got_lib_object_cache.h"
  48 #include "got_lib_pack.h"
  49 #include "got_lib_ratelimit.h"
  50 #include "got_lib_repository.h"
  51 #include "got_lib_privsep.h"
  52
  53 #define GIT_BUNDLE_SIGNATURE_V2 "# v2 git bundle\n"
  54 #define GIT_BUNDLE_SIGNATURE_V3 "# v3 git bundle\n"
  55
  56 #ifndef nitems
  57 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  58 #endif
  59
  60 #ifndef ssizeof
  61 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
  62 #endif
  63
  64 static const struct got_error *
  65 temp_file(int *fd, char **path, const char *ext, struct got_repository *repo)
  66 {
  67         const struct got_error *err;
  68         char p[PATH_MAX];
  69         int r;
  70
  71         *path = NULL;
  72
  73         r = snprintf(p, sizeof(p), "%s/%s/loading",
  74             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR);
  75         if (r < 0 || (size_t)r >= sizeof(p))
  76                 return got_error_from_errno("snprintf");
  77
  78         err = got_opentemp_named_fd(path, fd, p, ext);
  79         if (err)
  80                 return err;
  81
  82         if (fchmod(*fd, GOT_DEFAULT_FILE_MODE) == -1)
  83                 return got_error_from_errno("fchmod");
  84
  85         return NULL;
  86 }
  87
  88 static const struct got_error *
  89 load_report_progress(got_load_progress_cb progress_cb, void *progress_arg,
  90     struct got_ratelimit *rl, off_t packsiz, int nobj_total,
  91     int nobj_indexed, int nobj_loose, int nobj_resolved)
  92 {
  93         const struct got_error *err;
  94         int elapsed;
  95
  96         if (progress_cb == NULL)
  97                 return NULL;
  98
  99         err = got_ratelimit_check(&elapsed, rl);
 100         if (err || !elapsed)
 101                 return err;
 102
 103         return progress_cb(progress_arg, packsiz, nobj_total, nobj_indexed,
 104             nobj_loose, nobj_resolved);
 105 }
 106
 107 static const struct got_error *
 108 copypack(FILE *in, int outfd, off_t *tot, struct got_object_id *id,
 109     enum got_hash_algorithm algo, struct got_ratelimit *rl,
 110     got_load_progress_cb progress_cb, void *progress_arg,
 111     got_cancel_cb cancel_cb, void *cancel_arg)
 112 {
 113         const struct got_error *err;
 114         struct got_hash hash;
 115         struct got_object_id expected_id;
 116         char buf[BUFSIZ], hashbuf[GOT_HASH_DIGEST_MAXLEN];
 117         size_t r, digest_len, hashlen = 0;
 118
 119         *tot = 0;
 120         digest_len = got_hash_digest_length(algo);
 121         got_hash_init(&hash, algo);
 122
 123         for (;;) {
 124                 err = cancel_cb(cancel_arg);
 125                 if (err)
 126                         return err;
 127
 128                 r = fread(buf, 1, sizeof(buf), in);
 129                 if (r == 0)
 130                         break;
 131
 132                 /*
 133                  * An expected a checksum sits at the end of the pack
 134                  * file.  Since we don't know the file size ahead of
 135                  * time we have to keep digest_len bytes buffered and
 136                  * avoid mixing those bytes int our hash computation
 137                  * until we know for sure that additional pack file
 138                  * data bytes follow.
 139                  *
 140                  * We can assume that BUFSIZE is greater than
 141                  * digest_len and that a short read means that we've
 142                  * reached EOF.
 143                  */
 144
 145                 if (r >= digest_len) {
 146                         *tot += hashlen;
 147                         got_hash_update(&hash, hashbuf, hashlen);
 148                         if (write(outfd, hashbuf, hashlen) == -1)
 149                                 return got_error_from_errno("write");
 150
 151                         r -= digest_len;
 152                         memcpy(hashbuf, &buf[r], digest_len);
 153                         hashlen = digest_len;
 154
 155                         *tot += r;
 156                         got_hash_update(&hash, buf, r);
 157                         if (write(outfd, buf, r) == -1)
 158                                 return got_error_from_errno("write");
 159
 160                         err = load_report_progress(progress_cb, progress_arg,
 161                             rl, *tot, 0, 0, 0, 0);
 162                         if (err)
 163                                 return err;
 164
 165                         continue;
 166                 }
 167
 168                 if (hashlen == 0)
 169                         return got_error(GOT_ERR_BAD_PACKFILE);
 170
 171                 /* short read, we've reached EOF */
 172                 *tot += r;
 173                 got_hash_update(&hash, hashbuf, r);
 174                 if (write(outfd, hashbuf, r) == -1)
 175                         return got_error_from_errno("write");
 176
 177                 memmove(&hashbuf[0], &hashbuf[r], digest_len - r);
 178                 memcpy(&hashbuf[digest_len - r], buf, r);
 179                 break;
 180         }
 181
 182         if (hashlen == 0)
 183                 return got_error(GOT_ERR_BAD_PACKFILE);
 184
 185         got_hash_final_object_id(&hash, id);
 186
 187         memset(&expected_id, 0, sizeof(expected_id));
 188         expected_id.algo = algo;
 189         memcpy(&expected_id.hash, hashbuf, digest_len);
 190
 191         if (got_object_id_cmp(id, &expected_id) != 0)
 192                 return got_error(GOT_ERR_PACKIDX_CSUM);
 193
 194         /* re-add the expected hash at the end of the pack */
 195         if (write(outfd, hashbuf, digest_len) == -1)
 196                 return got_error_from_errno("write");
 197
 198         *tot += digest_len;
 199         err = progress_cb(progress_arg, *tot, 0, 0, 0, 0);
 200         if (err)
 201                 return err;
 202
 203         return NULL;
 204 }
 205
 206 const struct got_error *
 207 got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
 208     struct got_repository *repo, int list_refs_only, int noop,
 209     got_load_progress_cb progress_cb, void *progress_arg,
 210     got_cancel_cb cancel_cb, void *cancel_arg)
 211 {
 212         const struct got_error *err = NULL;
 213         struct got_object_id id;
 214         struct got_object *obj;
 215         struct got_packfile_hdr pack_hdr;
 216         struct got_ratelimit rl;
 217         struct imsgbuf idxibuf;
 218         const char *repo_path;
 219         char *packpath = NULL, *idxpath = NULL;
 220         char *tmppackpath = NULL, *tmpidxpath = NULL;
 221         int packfd = -1, idxfd = -1;
 222         char *spc, *refname, *id_str = NULL;
 223         char *line = NULL;
 224         size_t linesize = 0;
 225         ssize_t linelen;
 226         size_t i, digest_len;
 227         ssize_t n;
 228         off_t packsiz;
 229         int tmpfds[3] = {-1, -1, -1};
 230         int imsg_idxfds[2] = {-1, -1};
 231         int ch, done, nobj, idxstatus;
 232         pid_t idxpid;
 233         enum got_hash_algorithm repo_algo, bundle_algo;
 234
 235         memset(&idxibuf, 0, sizeof(idxibuf));
 236
 237         got_ratelimit_init(&rl, 0, 500);
 238         repo_algo = got_repo_get_object_format(repo);
 239         digest_len = got_hash_digest_length(repo_algo);
 240         repo_path = got_repo_get_path_git_dir(repo);
 241
 242         /* bundles will use v3 and a capability to advertise sha256 */
 243         bundle_algo = GOT_HASH_SHA1;
 244
 245         linelen = getline(&line, &linesize, in);
 246         if (linelen == -1) {
 247                 err = got_ferror(in, GOT_ERR_IO);
 248                 goto done;
 249         }
 250
 251         if (strcmp(line, GIT_BUNDLE_SIGNATURE_V2) != 0 &&
 252             strcmp(line, GIT_BUNDLE_SIGNATURE_V3) != 0) {
 253                 err = got_error(GOT_ERR_BUNDLE_FORMAT);
 254                 goto done;
 255         }
 256
 257         /* Parse the capabilities */
 258         for (;;) {
 259                 char *key, *val;
 260
 261                 ch = fgetc(in);
 262                 if (ch != '@') {
 263                         if (ch != EOF)
 264                                 ungetc(ch, in);
 265                         break;
 266                 }
 267
 268                 linelen = getline(&line, &linesize, in);
 269                 if (linelen == -1) {
 270                         err = got_ferror(in, GOT_ERR_IO);
 271                         goto done;
 272                 }
 273
 274                 if (line[linelen - 1] == '\n')
 275                         line[linelen - 1] = '\0';
 276
 277                 key = line;
 278                 val = strchr(key, '=');
 279                 if (val == NULL) {
 280                         err = got_error_path(key, GOT_ERR_UNKNOWN_CAPA);
 281                         goto done;
 282                 }
 283                 *val++ = '\0';
 284                 if (!strcmp(key, "object-format")) {
 285                         if (!strcmp(val, "sha1")) {
 286                                 bundle_algo = GOT_HASH_SHA1;
 287                                 continue;
 288                         }
 289                         if (!strcmp(val, "sha256")) {
 290                                 bundle_algo = GOT_HASH_SHA256;
 291                                 continue;
 292                         }
 293                 }
 294                 err = got_error_path(key, GOT_ERR_UNKNOWN_CAPA);
 295                 goto done;
 296         }
 297
 298         if (bundle_algo != repo_algo) {
 299                 err = got_error(GOT_ERR_OBJECT_FORMAT);
 300                 goto done;
 301         }
 302
 303         /* Parse the prerequisite */
 304         for (;;) {
 305                 ch = fgetc(in);
 306                 if (ch != '-') {
 307                         if (ch != EOF)
 308                                 ungetc(ch, in);
 309                         break;
 310                 }
 311
 312                 linelen = getline(&line, &linesize, in);
 313                 if (linelen == -1) {
 314                         err = got_ferror(in, GOT_ERR_IO);
 315                         goto done;
 316                 }
 317
 318                 if (line[linelen - 1] == '\n')
 319                         line[linelen - 1] = '\0';
 320
 321                 if (!got_parse_object_id(&id, line, repo_algo)) {
 322                         err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR);
 323                         goto done;
 324                 }
 325
 326                 err = got_object_open(&obj, repo, &id);
 327                 if (err)
 328                         goto done;
 329                 got_object_close(obj);
 330         }
 331
 332         /* Read references */
 333         for (;;) {
 334                 struct got_object_id *id;
 335                 char *dup;
 336                 struct got_pathlist_entry *new;
 337
 338                 linelen = getline(&line, &linesize, in);
 339                 if (linelen == -1) {
 340                         err = got_ferror(in, GOT_ERR_IO);
 341                         goto done;
 342                 }
 343                 if (line[linelen - 1] == '\n')
 344                         line[linelen - 1] = '\0';
 345                 if (*line == '\0')
 346                         break;
 347
 348                 spc = strchr(line, ' ');
 349                 if (spc == NULL) {
 350                         err = got_error(GOT_ERR_IO);
 351                         goto done;
 352                 }
 353                 *spc = '\0';
 354
 355                 refname = spc + 1;
 356                 if (!got_ref_name_is_valid(refname)) {
 357                         err = got_error(GOT_ERR_BAD_REF_DATA);
 358                         goto done;
 359                 }
 360
 361                 id = malloc(sizeof(*id));
 362                 if (id == NULL) {
 363                         err = got_error_from_errno("malloc");
 364                         goto done;
 365                 }
 366
 367                 if (!got_parse_object_id(id, line, repo_algo)) {
 368                         free(id);
 369                         err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
 370                         goto done;
 371                 }
 372
 373                 dup = strdup(refname);
 374                 if (dup == NULL) {
 375                         free(id);
 376                         err = got_error_from_errno("strdup");
 377                         goto done;
 378                 }
 379
 380                 err = got_pathlist_insert(&new, refs_found, dup, id);
 381                 if (err || new == NULL) {
 382                         free(id);
 383                         free(dup);
 384                         if (err)
 385                                 goto done;
 386                 }
 387         }
 388
 389         if (list_refs_only)
 390                 goto done;
 391
 392         err = temp_file(&packfd, &tmppackpath, ".pack", repo);
 393         if (err)
 394                 goto done;
 395
 396         err = temp_file(&idxfd, &tmpidxpath, ".idx", repo);
 397         if (err)
 398                 goto done;
 399
 400         err = copypack(in, packfd, &packsiz, &id, repo_algo, &rl,
 401             progress_cb, progress_arg, cancel_cb, cancel_arg);
 402         if (err)
 403                 goto done;
 404
 405         if (lseek(packfd, 0, SEEK_SET) == -1) {
 406                 err = got_error_from_errno("lseek");
 407                 goto done;
 408         }
 409
 410         /* Safety checks on the pack' content. */
 411         if (packsiz <= ssizeof(pack_hdr) + digest_len) {
 412                 err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
 413                 goto done;
 414         }
 415
 416         n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
 417         if (n == -1) {
 418                 err = got_error_from_errno("read");
 419                 goto done;
 420         }
 421         if (n != ssizeof(pack_hdr)) {
 422                 err = got_error(GOT_ERR_IO);
 423                 goto done;
 424         }
 425         if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
 426                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 427                     "bad pack file signature");
 428                 goto done;
 429         }
 430         if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
 431                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 432                     "bad pack file version");
 433                 goto done;
 434         }
 435         nobj = be32toh(pack_hdr.nobjects);
 436         if (nobj == 0 &&
 437             packsiz > ssizeof(pack_hdr) + digest_len) {
 438                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 439                     "bad pack file with zero objects");
 440                 goto done;
 441         }
 442         if (nobj != 0 &&
 443             packsiz <= ssizeof(pack_hdr) + digest_len) {
 444                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 445                     "empty pack file with non-zero object count");
 446                 goto done;
 447         }
 448
 449         /* nothing to do if there are no objects. */
 450         if (nobj == 0)
 451                 goto done;
 452
 453         for (i = 0; i < nitems(tmpfds); i++) {
 454                 tmpfds[i] = got_opentempfd();
 455                 if (tmpfds[i] == -1) {
 456                         err = got_error_from_errno("got_opentempfd");
 457                         goto done;
 458                 }
 459         }
 460
 461         if (lseek(packfd, 0, SEEK_SET) == -1) {
 462                 err = got_error_from_errno("lseek");
 463                 goto done;
 464         }
 465
 466         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 467                 err = got_error_from_errno("socketpair");
 468                 goto done;
 469         }
 470         idxpid = fork();
 471         if (idxpid == -1) {
 472                 err= got_error_from_errno("fork");
 473                 goto done;
 474         } else if (idxpid == 0)
 475                 got_privsep_exec_child(imsg_idxfds,
 476                     GOT_PATH_PROG_INDEX_PACK, tmppackpath);
 477         if (close(imsg_idxfds[1]) == -1) {
 478                 err = got_error_from_errno("close");
 479                 goto done;
 480         }
 481         imsg_idxfds[1] = -1;
 482         if (imsgbuf_init(&idxibuf, imsg_idxfds[0]) == -1) {
 483                 err = got_error_from_errno("imsgbuf_init");
 484                 goto done;
 485         }
 486         imsgbuf_allow_fdpass(&idxibuf);
 487
 488         err = got_privsep_send_index_pack_req(&idxibuf, &id, packfd);
 489         if (err)
 490                 goto done;
 491         packfd = -1;
 492
 493         err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
 494         if (err)
 495                 goto done;
 496         idxfd = -1;
 497
 498         for (i = 0; i < nitems(tmpfds); i++) {
 499                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 500                 if (err != NULL)
 501                         goto done;
 502                 tmpfds[i] = -1;
 503         }
 504
 505         done = 0;
 506         while (!done) {
 507                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 508
 509                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 510                     &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
 511                 if (err)
 512                         goto done;
 513                 if (nobj_indexed != 0) {
 514                         err = load_report_progress(progress_cb, progress_arg,
 515                             &rl, packsiz, nobj_total, nobj_indexed,
 516                             nobj_loose, nobj_resolved);
 517                         if (err)
 518                                 goto done;
 519                 }
 520         }
 521         if (close(imsg_idxfds[0]) == -1) {
 522                 err = got_error_from_errno("close");
 523                 goto done;
 524         }
 525         imsg_idxfds[0] = -1;
 526         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 527                 err = got_error_from_errno("waitpid");
 528                 goto done;
 529         }
 530
 531         if (noop)
 532                 goto done;
 533
 534         err = got_object_id_str(&id_str, &id);
 535         if (err)
 536                 goto done;
 537
 538         if (asprintf(&packpath, "%s/%s/pack-%s.pack", repo_path,
 539             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 540                 err = got_error_from_errno("asprintf");
 541                 goto done;
 542         }
 543
 544         if (asprintf(&idxpath, "%s/%s/pack-%s.idx", repo_path,
 545             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 546                 err = got_error_from_errno("asprintf");
 547                 goto done;
 548         }
 549
 550         if (rename(tmppackpath, packpath) == -1) {
 551                 err = got_error_from_errno3("rename", tmppackpath, packpath);
 552                 goto done;
 553         }
 554         free(tmppackpath);
 555         tmppackpath = NULL;
 556
 557         if (rename(tmpidxpath, idxpath) == -1) {
 558                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 559                 goto done;
 560         }
 561         free(tmpidxpath);
 562         tmpidxpath = NULL;
 563
 564  done:
 565         if (idxibuf.w)
 566                 imsgbuf_clear(&idxibuf);
 567         free(line);
 568         free(packpath);
 569         free(idxpath);
 570         free(id_str);
 571
 572         if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
 573                 err = got_error_from_errno2("unlink", tmppackpath);
 574         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 575                 err = got_error_from_errno("close");
 576         free(tmppackpath);
 577
 578         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 579                 err = got_error_from_errno2("unlink", tmpidxpath);
 580         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 581                 err = got_error_from_errno("close");
 582         free(tmpidxpath);
 583
 584         if (imsg_idxfds[0] != -1 && close(imsg_idxfds[0]) == -1 && err == NULL)
 585                 err = got_error_from_errno("close");
 586         if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
 587                 err = got_error_from_errno("close");
 588
 589         for (i = 0; i < nitems(tmpfds); ++i)
 590                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 591                         err = got_error_from_errno("close");
 592
 593         return err;
 594 }