lib/load.c

   1 /*
   2  * Copyright (c) 2023 Omar Polo <op@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/time.h>
  23 #include <sys/types.h>
  24 #include <sys/uio.h>
  25 #include <sys/wait.h>
  26
  27 #include <limits.h>
  28 #include <stdint.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <imsg.h>
  34
  35 #include "got_error.h"
  36 #include "got_cancel.h"
  37 #include "got_object.h"
  38 #include "got_opentemp.h"
  39 #include "got_path.h"
  40 #include "got_reference.h"
  41 #include "got_repository.h"
  42 #include "got_repository_load.h"
  43
  44 #include "got_lib_delta.h"
  45 #include "got_lib_hash.h"
  46 #include "got_lib_object.h"
  47 #include "got_lib_object_cache.h"
  48 #include "got_lib_pack.h"
  49 #include "got_lib_ratelimit.h"
  50 #include "got_lib_repository.h"
  51 #include "got_lib_privsep.h"
  52
  53 #define GIT_BUNDLE_SIGNATURE_V2 "# v2 git bundle\n"
  54
  55 #ifndef nitems
  56 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  57 #endif
  58
  59 #ifndef ssizeof
  60 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
  61 #endif
  62
  63 static const struct got_error *
  64 temp_file(int *fd, char **path, const char *ext, struct got_repository *repo)
  65 {
  66         const struct got_error *err;
  67         char p[PATH_MAX];
  68         int r;
  69
  70         *path = NULL;
  71
  72         r = snprintf(p, sizeof(p), "%s/%s/loading",
  73             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR);
  74         if (r < 0 || (size_t)r >= sizeof(p))
  75                 return got_error_from_errno("snprintf");
  76
  77         err = got_opentemp_named_fd(path, fd, p, ext);
  78         if (err)
  79                 return err;
  80
  81         if (fchmod(*fd, GOT_DEFAULT_FILE_MODE) == -1)
  82                 return got_error_from_errno("fchmod");
  83
  84         return NULL;
  85 }
  86
  87 static const struct got_error *
  88 load_report_progress(got_load_progress_cb progress_cb, void *progress_arg,
  89     struct got_ratelimit *rl, off_t packsiz, int nobj_total,
  90     int nobj_indexed, int nobj_loose, int nobj_resolved)
  91 {
  92         const struct got_error *err;
  93         int elapsed;
  94
  95         if (progress_cb == NULL)
  96                 return NULL;
  97
  98         err = got_ratelimit_check(&elapsed, rl);
  99         if (err || !elapsed)
 100                 return err;
 101
 102         return progress_cb(progress_arg, packsiz, nobj_total, nobj_indexed,
 103             nobj_loose, nobj_resolved);
 104 }
 105
 106 static const struct got_error *
 107 copypack(FILE *in, int outfd, off_t *tot,
 108     struct got_object_id *id, struct got_ratelimit *rl,
 109     got_load_progress_cb progress_cb, void *progress_arg,
 110     got_cancel_cb cancel_cb, void *cancel_arg)
 111 {
 112         const struct got_error *err;
 113         struct got_hash hash;
 114         struct got_object_id expected_id;
 115         char buf[BUFSIZ], sha1buf[SHA1_DIGEST_LENGTH];
 116         size_t r, sha1buflen = 0;
 117
 118         *tot = 0;
 119         got_hash_init(&hash, GOT_HASH_SHA1);
 120
 121         for (;;) {
 122                 err = cancel_cb(cancel_arg);
 123                 if (err)
 124                         return err;
 125
 126                 r = fread(buf, 1, sizeof(buf), in);
 127                 if (r == 0)
 128                         break;
 129
 130                 /*
 131                  * An expected SHA1 checksum sits at the end of the
 132                  * pack file.  Since we don't know the file size ahead
 133                  * of time we have to keep SHA1_DIGEST_LENGTH bytes
 134                  * buffered and avoid mixing those bytes int our hash
 135                  * computation until we know for sure that additional
 136                  * pack file data bytes follow.
 137                  *
 138                  * We can assume that BUFSIZE is greater than
 139                  * SHA1_DIGEST_LENGTH and that a short read means that
 140                  * we've reached EOF.
 141                  */
 142
 143                 if (r >= sizeof(sha1buf)) {
 144                         *tot += sha1buflen;
 145                         got_hash_update(&hash, sha1buf, sha1buflen);
 146                         if (write(outfd, sha1buf, sha1buflen) == -1)
 147                                 return got_error_from_errno("write");
 148
 149                         r -= sizeof(sha1buf);
 150                         memcpy(sha1buf, &buf[r], sizeof(sha1buf));
 151                         sha1buflen = sizeof(sha1buf);
 152
 153                         *tot += r;
 154                         got_hash_update(&hash, buf, r);
 155                         if (write(outfd, buf, r) == -1)
 156                                 return got_error_from_errno("write");
 157
 158                         err = load_report_progress(progress_cb, progress_arg,
 159                             rl, *tot, 0, 0, 0, 0);
 160                         if (err)
 161                                 return err;
 162
 163                         continue;
 164                 }
 165
 166                 if (sha1buflen == 0)
 167                         return got_error(GOT_ERR_BAD_PACKFILE);
 168
 169                 /* short read, we've reached EOF */
 170                 *tot += r;
 171                 got_hash_update(&hash, sha1buf, r);
 172                 if (write(outfd, sha1buf, r) == -1)
 173                         return got_error_from_errno("write");
 174
 175                 memmove(&sha1buf[0], &sha1buf[r], sizeof(sha1buf) - r);
 176                 memcpy(&sha1buf[sizeof(sha1buf) - r], buf, r);
 177                 break;
 178         }
 179
 180         if (sha1buflen == 0)
 181                 return got_error(GOT_ERR_BAD_PACKFILE);
 182
 183         got_hash_final_object_id(&hash, id);
 184
 185         /* XXX SHA256 */
 186         memset(&expected_id, 0, sizeof(expected_id));
 187         memcpy(&expected_id.sha1, sha1buf, sizeof(expected_id.sha1));
 188
 189         if (got_object_id_cmp(id, &expected_id) != 0)
 190                 return got_error(GOT_ERR_PACKIDX_CSUM);
 191
 192         /* re-add the expected hash at the end of the pack */
 193         if (write(outfd, sha1buf, sizeof(sha1buf)) == -1)
 194                 return got_error_from_errno("write");
 195
 196         *tot += sizeof(sha1buf);
 197         err = progress_cb(progress_arg, *tot, 0, 0, 0, 0);
 198         if (err)
 199                 return err;
 200
 201         return NULL;
 202 }
 203
 204 const struct got_error *
 205 got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
 206     struct got_repository *repo, int list_refs_only, int noop,
 207     got_load_progress_cb progress_cb, void *progress_arg,
 208     got_cancel_cb cancel_cb, void *cancel_arg)
 209 {
 210         const struct got_error *err = NULL;
 211         struct got_object_id id;
 212         struct got_object *obj;
 213         struct got_packfile_hdr pack_hdr;
 214         struct got_ratelimit rl;
 215         struct imsgbuf idxibuf;
 216         const char *repo_path;
 217         char *packpath = NULL, *idxpath = NULL;
 218         char *tmppackpath = NULL, *tmpidxpath = NULL;
 219         int packfd = -1, idxfd = -1;
 220         char *spc, *refname, *id_str = NULL;
 221         char *line = NULL;
 222         size_t linesize = 0;
 223         ssize_t linelen;
 224         size_t i;
 225         ssize_t n;
 226         off_t packsiz;
 227         int tmpfds[3] = {-1, -1, -1};
 228         int imsg_idxfds[2] = {-1, -1};
 229         int ch, done, nobj, idxstatus;
 230         pid_t idxpid;
 231
 232         got_ratelimit_init(&rl, 0, 500);
 233
 234         repo_path = got_repo_get_path_git_dir(repo);
 235
 236         linelen = getline(&line, &linesize, in);
 237         if (linelen == -1) {
 238                 err = got_ferror(in, GOT_ERR_IO);
 239                 goto done;
 240         }
 241
 242         if (strcmp(line, GIT_BUNDLE_SIGNATURE_V2) != 0) {
 243                 err = got_error(GOT_ERR_BUNDLE_FORMAT);
 244                 goto done;
 245         }
 246
 247         /* Parse the prerequisite */
 248         for (;;) {
 249                 ch = fgetc(in);
 250                 if (ch != '-') {
 251                         if (ch != EOF)
 252                                 ungetc(ch, in);
 253                         break;
 254                 }
 255
 256                 linelen = getline(&line, &linesize, in);
 257                 if (linelen == -1) {
 258                         err = got_ferror(in, GOT_ERR_IO);
 259                         goto done;
 260                 }
 261
 262                 if (line[linelen - 1] == '\n')
 263                         line[linelen - 1] = '\0';
 264
 265                 if (!got_parse_object_id(&id, line, GOT_HASH_SHA1)) {
 266                         err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR);
 267                         goto done;
 268                 }
 269
 270                 err = got_object_open(&obj, repo, &id);
 271                 if (err)
 272                         goto done;
 273                 got_object_close(obj);
 274         }
 275
 276         /* Read references */
 277         for (;;) {
 278                 struct got_object_id *id;
 279                 char *dup;
 280
 281                 linelen = getline(&line, &linesize, in);
 282                 if (linelen == -1) {
 283                         err = got_ferror(in, GOT_ERR_IO);
 284                         goto done;
 285                 }
 286                 if (line[linelen - 1] == '\n')
 287                         line[linelen - 1] = '\0';
 288                 if (*line == '\0')
 289                         break;
 290
 291                 spc = strchr(line, ' ');
 292                 if (spc == NULL) {
 293                         err = got_error(GOT_ERR_IO);
 294                         goto done;
 295                 }
 296                 *spc = '\0';
 297
 298                 refname = spc + 1;
 299                 if (!got_ref_name_is_valid(refname)) {
 300                         err = got_error(GOT_ERR_BAD_REF_DATA);
 301                         goto done;
 302                 }
 303
 304                 id = malloc(sizeof(*id));
 305                 if (id == NULL) {
 306                         err = got_error_from_errno("malloc");
 307                         goto done;
 308                 }
 309
 310                 if (!got_parse_object_id(id, line, GOT_HASH_SHA1)) {
 311                         free(id);
 312                         err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
 313                         goto done;
 314                 }
 315
 316                 dup = strdup(refname);
 317                 if (dup == NULL) {
 318                         free(id);
 319                         err = got_error_from_errno("strdup");
 320                         goto done;
 321                 }
 322
 323                 err = got_pathlist_append(refs_found, dup, id);
 324                 if (err) {
 325                         free(id);
 326                         free(dup);
 327                         goto done;
 328                 }
 329         }
 330
 331         if (list_refs_only)
 332                 goto done;
 333
 334         err = temp_file(&packfd, &tmppackpath, ".pack", repo);
 335         if (err)
 336                 goto done;
 337
 338         err = temp_file(&idxfd, &tmpidxpath, ".idx", repo);
 339         if (err)
 340                 goto done;
 341
 342         err = copypack(in, packfd, &packsiz, &id, &rl,
 343             progress_cb, progress_arg, cancel_cb, cancel_arg);
 344         if (err)
 345                 goto done;
 346
 347         if (lseek(packfd, 0, SEEK_SET) == -1) {
 348                 err = got_error_from_errno("lseek");
 349                 goto done;
 350         }
 351
 352         /* Safety checks on the pack' content. */
 353         if (packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 354                 err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
 355                 goto done;
 356         }
 357
 358         n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
 359         if (n == -1) {
 360                 err = got_error_from_errno("read");
 361                 goto done;
 362         }
 363         if (n != ssizeof(pack_hdr)) {
 364                 err = got_error(GOT_ERR_IO);
 365                 goto done;
 366         }
 367         if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
 368                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 369                     "bad pack file signature");
 370                 goto done;
 371         }
 372         if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
 373                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 374                     "bad pack file version");
 375                 goto done;
 376         }
 377         nobj = be32toh(pack_hdr.nobjects);
 378         if (nobj == 0 &&
 379             packsiz > ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 380                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 381                     "bad pack file with zero objects");
 382                 goto done;
 383         }
 384         if (nobj != 0 &&
 385             packsiz <= ssizeof(pack_hdr) + SHA1_DIGEST_LENGTH) {
 386                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 387                     "empty pack file with non-zero object count");
 388                 goto done;
 389         }
 390
 391         /* nothing to do if there are no objects. */
 392         if (nobj == 0)
 393                 goto done;
 394
 395         for (i = 0; i < nitems(tmpfds); i++) {
 396                 tmpfds[i] = got_opentempfd();
 397                 if (tmpfds[i] == -1) {
 398                         err = got_error_from_errno("got_opentempfd");
 399                         goto done;
 400                 }
 401         }
 402
 403         if (lseek(packfd, 0, SEEK_SET) == -1) {
 404                 err = got_error_from_errno("lseek");
 405                 goto done;
 406         }
 407
 408         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 409                 err = got_error_from_errno("socketpair");
 410                 goto done;
 411         }
 412         idxpid = fork();
 413         if (idxpid == -1) {
 414                 err= got_error_from_errno("fork");
 415                 goto done;
 416         } else if (idxpid == 0)
 417                 got_privsep_exec_child(imsg_idxfds,
 418                     GOT_PATH_PROG_INDEX_PACK, tmppackpath);
 419         if (close(imsg_idxfds[1]) == -1) {
 420                 err = got_error_from_errno("close");
 421                 goto done;
 422         }
 423         imsg_idxfds[1] = -1;
 424         imsg_init(&idxibuf, imsg_idxfds[0]);
 425
 426         err = got_privsep_send_index_pack_req(&idxibuf, id.sha1, packfd);
 427         if (err)
 428                 goto done;
 429         packfd = -1;
 430
 431         err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
 432         if (err)
 433                 goto done;
 434         idxfd = -1;
 435
 436         for (i = 0; i < nitems(tmpfds); i++) {
 437                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 438                 if (err != NULL)
 439                         goto done;
 440                 tmpfds[i] = -1;
 441         }
 442
 443         done = 0;
 444         while (!done) {
 445                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 446
 447                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 448                     &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
 449                 if (err)
 450                         goto done;
 451                 if (nobj_indexed != 0) {
 452                         err = load_report_progress(progress_cb, progress_arg,
 453                             &rl, packsiz, nobj_total, nobj_indexed,
 454                             nobj_loose, nobj_resolved);
 455                         if (err)
 456                                 goto done;
 457                 }
 458         }
 459         if (close(imsg_idxfds[0]) == -1) {
 460                 err = got_error_from_errno("close");
 461                 goto done;
 462         }
 463         imsg_idxfds[0] = -1;
 464         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 465                 err = got_error_from_errno("waitpid");
 466                 goto done;
 467         }
 468
 469         if (noop)
 470                 goto done;
 471
 472         err = got_object_id_str(&id_str, &id);
 473         if (err)
 474                 goto done;
 475
 476         if (asprintf(&packpath, "%s/%s/pack-%s.pack", repo_path,
 477             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 478                 err = got_error_from_errno("asprintf");
 479                 goto done;
 480         }
 481
 482         if (asprintf(&idxpath, "%s/%s/pack-%s.idx", repo_path,
 483             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 484                 err = got_error_from_errno("asprintf");
 485                 goto done;
 486         }
 487
 488         if (rename(tmppackpath, packpath) == -1) {
 489                 err = got_error_from_errno3("rename", tmppackpath, packpath);
 490                 goto done;
 491         }
 492         free(tmppackpath);
 493         tmppackpath = NULL;
 494
 495         if (rename(tmpidxpath, idxpath) == -1) {
 496                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 497                 goto done;
 498         }
 499         free(tmpidxpath);
 500         tmpidxpath = NULL;
 501
 502  done:
 503         free(line);
 504         free(packpath);
 505         free(idxpath);
 506         free(id_str);
 507
 508         if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
 509                 err = got_error_from_errno2("unlink", tmppackpath);
 510         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 511                 err = got_error_from_errno("close");
 512         free(tmppackpath);
 513
 514         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 515                 err = got_error_from_errno2("unlink", tmpidxpath);
 516         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 517                 err = got_error_from_errno("close");
 518         free(tmpidxpath);
 519
 520         if (imsg_idxfds[0] != -1 && close(imsg_idxfds[0]) == -1 && err == NULL)
 521                 err = got_error_from_errno("close");
 522         if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
 523                 err = got_error_from_errno("close");
 524
 525         for (i = 0; i < nitems(tmpfds); ++i)
 526                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 527                         err = got_error_from_errno("close");
 528
 529         return err;
 530 }