lib/load.c

   1 /*
   2  * Copyright (c) 2023 Omar Polo <op@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/queue.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/time.h>
  23 #include <sys/types.h>
  24 #include <sys/uio.h>
  25 #include <sys/wait.h>
  26
  27 #include <limits.h>
  28 #include <stdint.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <imsg.h>
  34
  35 #include "got_error.h"
  36 #include "got_cancel.h"
  37 #include "got_object.h"
  38 #include "got_opentemp.h"
  39 #include "got_path.h"
  40 #include "got_reference.h"
  41 #include "got_repository.h"
  42 #include "got_repository_load.h"
  43
  44 #include "got_lib_delta.h"
  45 #include "got_lib_hash.h"
  46 #include "got_lib_object.h"
  47 #include "got_lib_object_cache.h"
  48 #include "got_lib_pack.h"
  49 #include "got_lib_ratelimit.h"
  50 #include "got_lib_repository.h"
  51 #include "got_lib_privsep.h"
  52
  53 #define GIT_BUNDLE_SIGNATURE_V2 "# v2 git bundle\n"
  54 #define GIT_BUNDLE_SIGNATURE_V3 "# v3 git bundle\n"
  55
  56 #ifndef nitems
  57 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  58 #endif
  59
  60 #ifndef ssizeof
  61 #define ssizeof(_x) ((ssize_t)(sizeof(_x)))
  62 #endif
  63
  64 static const struct got_error *
  65 temp_file(int *fd, char **path, const char *ext, struct got_repository *repo)
  66 {
  67         const struct got_error *err;
  68         char p[PATH_MAX];
  69         int r;
  70
  71         *path = NULL;
  72
  73         r = snprintf(p, sizeof(p), "%s/%s/loading",
  74             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR);
  75         if (r < 0 || (size_t)r >= sizeof(p))
  76                 return got_error_from_errno("snprintf");
  77
  78         err = got_opentemp_named_fd(path, fd, p, ext);
  79         if (err)
  80                 return err;
  81
  82         if (fchmod(*fd, GOT_DEFAULT_FILE_MODE) == -1)
  83                 return got_error_from_errno("fchmod");
  84
  85         return NULL;
  86 }
  87
  88 static const struct got_error *
  89 load_report_progress(got_load_progress_cb progress_cb, void *progress_arg,
  90     struct got_ratelimit *rl, off_t packsiz, int nobj_total,
  91     int nobj_indexed, int nobj_loose, int nobj_resolved)
  92 {
  93         const struct got_error *err;
  94         int elapsed;
  95
  96         if (progress_cb == NULL)
  97                 return NULL;
  98
  99         err = got_ratelimit_check(&elapsed, rl);
 100         if (err || !elapsed)
 101                 return err;
 102
 103         return progress_cb(progress_arg, packsiz, nobj_total, nobj_indexed,
 104             nobj_loose, nobj_resolved);
 105 }
 106
 107 static const struct got_error *
 108 copypack(FILE *in, int outfd, off_t *tot, struct got_object_id *id,
 109     enum got_hash_algorithm algo, struct got_ratelimit *rl,
 110     got_load_progress_cb progress_cb, void *progress_arg,
 111     got_cancel_cb cancel_cb, void *cancel_arg)
 112 {
 113         const struct got_error *err;
 114         struct got_hash hash;
 115         struct got_object_id expected_id;
 116         char buf[BUFSIZ], hashbuf[GOT_HASH_DIGEST_MAXLEN];
 117         size_t r, digest_len, hashlen = 0;
 118
 119         *tot = 0;
 120         digest_len = got_hash_digest_length(algo);
 121         got_hash_init(&hash, algo);
 122
 123         for (;;) {
 124                 err = cancel_cb(cancel_arg);
 125                 if (err)
 126                         return err;
 127
 128                 r = fread(buf, 1, sizeof(buf), in);
 129                 if (r == 0)
 130                         break;
 131
 132                 /*
 133                  * An expected a checksum sits at the end of the pack
 134                  * file.  Since we don't know the file size ahead of
 135                  * time we have to keep digest_len bytes buffered and
 136                  * avoid mixing those bytes int our hash computation
 137                  * until we know for sure that additional pack file
 138                  * data bytes follow.
 139                  *
 140                  * We can assume that BUFSIZE is greater than
 141                  * digest_len and that a short read means that we've
 142                  * reached EOF.
 143                  */
 144
 145                 if (r >= digest_len) {
 146                         *tot += hashlen;
 147                         got_hash_update(&hash, hashbuf, hashlen);
 148                         if (write(outfd, hashbuf, hashlen) == -1)
 149                                 return got_error_from_errno("write");
 150
 151                         r -= digest_len;
 152                         memcpy(hashbuf, &buf[r], digest_len);
 153                         hashlen = digest_len;
 154
 155                         *tot += r;
 156                         got_hash_update(&hash, buf, r);
 157                         if (write(outfd, buf, r) == -1)
 158                                 return got_error_from_errno("write");
 159
 160                         err = load_report_progress(progress_cb, progress_arg,
 161                             rl, *tot, 0, 0, 0, 0);
 162                         if (err)
 163                                 return err;
 164
 165                         continue;
 166                 }
 167
 168                 if (hashlen == 0)
 169                         return got_error(GOT_ERR_BAD_PACKFILE);
 170
 171                 /* short read, we've reached EOF */
 172                 *tot += r;
 173                 got_hash_update(&hash, hashbuf, r);
 174                 if (write(outfd, hashbuf, r) == -1)
 175                         return got_error_from_errno("write");
 176
 177                 memmove(&hashbuf[0], &hashbuf[r], digest_len - r);
 178                 memcpy(&hashbuf[digest_len - r], buf, r);
 179                 break;
 180         }
 181
 182         if (hashlen == 0)
 183                 return got_error(GOT_ERR_BAD_PACKFILE);
 184
 185         got_hash_final_object_id(&hash, id);
 186
 187         memset(&expected_id, 0, sizeof(expected_id));
 188         expected_id.algo = algo;
 189         memcpy(&expected_id.hash, hashbuf, digest_len);
 190
 191         if (got_object_id_cmp(id, &expected_id) != 0)
 192                 return got_error(GOT_ERR_PACKIDX_CSUM);
 193
 194         /* re-add the expected hash at the end of the pack */
 195         if (write(outfd, hashbuf, digest_len) == -1)
 196                 return got_error_from_errno("write");
 197
 198         *tot += digest_len;
 199         err = progress_cb(progress_arg, *tot, 0, 0, 0, 0);
 200         if (err)
 201                 return err;
 202
 203         return NULL;
 204 }
 205
 206 const struct got_error *
 207 got_repo_load(FILE *in, struct got_pathlist_head *refs_found,
 208     struct got_repository *repo, int list_refs_only, int noop,
 209     got_load_progress_cb progress_cb, void *progress_arg,
 210     got_cancel_cb cancel_cb, void *cancel_arg)
 211 {
 212         const struct got_error *err = NULL;
 213         struct got_object_id id;
 214         struct got_object *obj;
 215         struct got_packfile_hdr pack_hdr;
 216         struct got_ratelimit rl;
 217         struct imsgbuf idxibuf;
 218         const char *repo_path;
 219         char *packpath = NULL, *idxpath = NULL;
 220         char *tmppackpath = NULL, *tmpidxpath = NULL;
 221         int packfd = -1, idxfd = -1;
 222         char *spc, *refname, *id_str = NULL;
 223         char *line = NULL;
 224         size_t linesize = 0;
 225         ssize_t linelen;
 226         size_t i, digest_len;
 227         ssize_t n;
 228         off_t packsiz;
 229         int tmpfds[3] = {-1, -1, -1};
 230         int imsg_idxfds[2] = {-1, -1};
 231         int ch, done, nobj, idxstatus;
 232         pid_t idxpid;
 233         enum got_hash_algorithm repo_algo, bundle_algo;
 234
 235         got_ratelimit_init(&rl, 0, 500);
 236         repo_algo = got_repo_get_object_format(repo);
 237         digest_len = got_hash_digest_length(repo_algo);
 238         repo_path = got_repo_get_path_git_dir(repo);
 239
 240         /* bundles will use v3 and a capability to advertise sha256 */
 241         bundle_algo = GOT_HASH_SHA1;
 242
 243         linelen = getline(&line, &linesize, in);
 244         if (linelen == -1) {
 245                 err = got_ferror(in, GOT_ERR_IO);
 246                 goto done;
 247         }
 248
 249         if (strcmp(line, GIT_BUNDLE_SIGNATURE_V2) != 0 &&
 250             strcmp(line, GIT_BUNDLE_SIGNATURE_V3) != 0) {
 251                 err = got_error(GOT_ERR_BUNDLE_FORMAT);
 252                 goto done;
 253         }
 254
 255         /* Parse the capabilities */
 256         for (;;) {
 257                 char *key, *val;
 258
 259                 ch = fgetc(in);
 260                 if (ch != '@') {
 261                         if (ch != EOF)
 262                                 ungetc(ch, in);
 263                         break;
 264                 }
 265
 266                 linelen = getline(&line, &linesize, in);
 267                 if (linelen == -1) {
 268                         err = got_ferror(in, GOT_ERR_IO);
 269                         goto done;
 270                 }
 271
 272                 if (line[linelen - 1] == '\n')
 273                         line[linelen - 1] = '\0';
 274
 275                 key = line;
 276                 val = strchr(key, '=');
 277                 if (val == NULL) {
 278                         err = got_error_path(key, GOT_ERR_UNKNOWN_CAPA);
 279                         goto done;
 280                 }
 281                 *val++ = '\0';
 282                 if (!strcmp(key, "object-format")) {
 283                         if (!strcmp(val, "sha1")) {
 284                                 bundle_algo = GOT_HASH_SHA1;
 285                                 continue;
 286                         }
 287                         if (!strcmp(val, "sha256")) {
 288                                 bundle_algo = GOT_HASH_SHA256;
 289                                 continue;
 290                         }
 291                 }
 292                 err = got_error_path(key, GOT_ERR_UNKNOWN_CAPA);
 293                 goto done;
 294         }
 295
 296         if (bundle_algo != repo_algo) {
 297                 err = got_error(GOT_ERR_OBJECT_FORMAT);
 298                 goto done;
 299         }
 300
 301         /* Parse the prerequisite */
 302         for (;;) {
 303                 ch = fgetc(in);
 304                 if (ch != '-') {
 305                         if (ch != EOF)
 306                                 ungetc(ch, in);
 307                         break;
 308                 }
 309
 310                 linelen = getline(&line, &linesize, in);
 311                 if (linelen == -1) {
 312                         err = got_ferror(in, GOT_ERR_IO);
 313                         goto done;
 314                 }
 315
 316                 if (line[linelen - 1] == '\n')
 317                         line[linelen - 1] = '\0';
 318
 319                 if (!got_parse_object_id(&id, line, repo_algo)) {
 320                         err = got_error_path(line, GOT_ERR_BAD_OBJ_ID_STR);
 321                         goto done;
 322                 }
 323
 324                 err = got_object_open(&obj, repo, &id);
 325                 if (err)
 326                         goto done;
 327                 got_object_close(obj);
 328         }
 329
 330         /* Read references */
 331         for (;;) {
 332                 struct got_object_id *id;
 333                 char *dup;
 334
 335                 linelen = getline(&line, &linesize, in);
 336                 if (linelen == -1) {
 337                         err = got_ferror(in, GOT_ERR_IO);
 338                         goto done;
 339                 }
 340                 if (line[linelen - 1] == '\n')
 341                         line[linelen - 1] = '\0';
 342                 if (*line == '\0')
 343                         break;
 344
 345                 spc = strchr(line, ' ');
 346                 if (spc == NULL) {
 347                         err = got_error(GOT_ERR_IO);
 348                         goto done;
 349                 }
 350                 *spc = '\0';
 351
 352                 refname = spc + 1;
 353                 if (!got_ref_name_is_valid(refname)) {
 354                         err = got_error(GOT_ERR_BAD_REF_DATA);
 355                         goto done;
 356                 }
 357
 358                 id = malloc(sizeof(*id));
 359                 if (id == NULL) {
 360                         err = got_error_from_errno("malloc");
 361                         goto done;
 362                 }
 363
 364                 if (!got_parse_object_id(id, line, repo_algo)) {
 365                         free(id);
 366                         err = got_error(GOT_ERR_BAD_OBJ_ID_STR);
 367                         goto done;
 368                 }
 369
 370                 dup = strdup(refname);
 371                 if (dup == NULL) {
 372                         free(id);
 373                         err = got_error_from_errno("strdup");
 374                         goto done;
 375                 }
 376
 377                 err = got_pathlist_append(refs_found, dup, id);
 378                 if (err) {
 379                         free(id);
 380                         free(dup);
 381                         goto done;
 382                 }
 383         }
 384
 385         if (list_refs_only)
 386                 goto done;
 387
 388         err = temp_file(&packfd, &tmppackpath, ".pack", repo);
 389         if (err)
 390                 goto done;
 391
 392         err = temp_file(&idxfd, &tmpidxpath, ".idx", repo);
 393         if (err)
 394                 goto done;
 395
 396         err = copypack(in, packfd, &packsiz, &id, repo_algo, &rl,
 397             progress_cb, progress_arg, cancel_cb, cancel_arg);
 398         if (err)
 399                 goto done;
 400
 401         if (lseek(packfd, 0, SEEK_SET) == -1) {
 402                 err = got_error_from_errno("lseek");
 403                 goto done;
 404         }
 405
 406         /* Safety checks on the pack' content. */
 407         if (packsiz <= ssizeof(pack_hdr) + digest_len) {
 408                 err = got_error_msg(GOT_ERR_BAD_PACKFILE, "short pack file");
 409                 goto done;
 410         }
 411
 412         n = read(packfd, &pack_hdr, ssizeof(pack_hdr));
 413         if (n == -1) {
 414                 err = got_error_from_errno("read");
 415                 goto done;
 416         }
 417         if (n != ssizeof(pack_hdr)) {
 418                 err = got_error(GOT_ERR_IO);
 419                 goto done;
 420         }
 421         if (pack_hdr.signature != htobe32(GOT_PACKFILE_SIGNATURE)) {
 422                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 423                     "bad pack file signature");
 424                 goto done;
 425         }
 426         if (pack_hdr.version != htobe32(GOT_PACKFILE_VERSION)) {
 427                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 428                     "bad pack file version");
 429                 goto done;
 430         }
 431         nobj = be32toh(pack_hdr.nobjects);
 432         if (nobj == 0 &&
 433             packsiz > ssizeof(pack_hdr) + digest_len) {
 434                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 435                     "bad pack file with zero objects");
 436                 goto done;
 437         }
 438         if (nobj != 0 &&
 439             packsiz <= ssizeof(pack_hdr) + digest_len) {
 440                 err = got_error_msg(GOT_ERR_BAD_PACKFILE,
 441                     "empty pack file with non-zero object count");
 442                 goto done;
 443         }
 444
 445         /* nothing to do if there are no objects. */
 446         if (nobj == 0)
 447                 goto done;
 448
 449         for (i = 0; i < nitems(tmpfds); i++) {
 450                 tmpfds[i] = got_opentempfd();
 451                 if (tmpfds[i] == -1) {
 452                         err = got_error_from_errno("got_opentempfd");
 453                         goto done;
 454                 }
 455         }
 456
 457         if (lseek(packfd, 0, SEEK_SET) == -1) {
 458                 err = got_error_from_errno("lseek");
 459                 goto done;
 460         }
 461
 462         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 463                 err = got_error_from_errno("socketpair");
 464                 goto done;
 465         }
 466         idxpid = fork();
 467         if (idxpid == -1) {
 468                 err= got_error_from_errno("fork");
 469                 goto done;
 470         } else if (idxpid == 0)
 471                 got_privsep_exec_child(imsg_idxfds,
 472                     GOT_PATH_PROG_INDEX_PACK, tmppackpath);
 473         if (close(imsg_idxfds[1]) == -1) {
 474                 err = got_error_from_errno("close");
 475                 goto done;
 476         }
 477         imsg_idxfds[1] = -1;
 478         imsg_init(&idxibuf, imsg_idxfds[0]);
 479
 480         err = got_privsep_send_index_pack_req(&idxibuf, &id, packfd);
 481         if (err)
 482                 goto done;
 483         packfd = -1;
 484
 485         err = got_privsep_send_index_pack_outfd(&idxibuf, idxfd);
 486         if (err)
 487                 goto done;
 488         idxfd = -1;
 489
 490         for (i = 0; i < nitems(tmpfds); i++) {
 491                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 492                 if (err != NULL)
 493                         goto done;
 494                 tmpfds[i] = -1;
 495         }
 496
 497         done = 0;
 498         while (!done) {
 499                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 500
 501                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 502                     &nobj_indexed, &nobj_loose, &nobj_resolved, &idxibuf);
 503                 if (err)
 504                         goto done;
 505                 if (nobj_indexed != 0) {
 506                         err = load_report_progress(progress_cb, progress_arg,
 507                             &rl, packsiz, nobj_total, nobj_indexed,
 508                             nobj_loose, nobj_resolved);
 509                         if (err)
 510                                 goto done;
 511                 }
 512         }
 513         if (close(imsg_idxfds[0]) == -1) {
 514                 err = got_error_from_errno("close");
 515                 goto done;
 516         }
 517         imsg_idxfds[0] = -1;
 518         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 519                 err = got_error_from_errno("waitpid");
 520                 goto done;
 521         }
 522
 523         if (noop)
 524                 goto done;
 525
 526         err = got_object_id_str(&id_str, &id);
 527         if (err)
 528                 goto done;
 529
 530         if (asprintf(&packpath, "%s/%s/pack-%s.pack", repo_path,
 531             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 532                 err = got_error_from_errno("asprintf");
 533                 goto done;
 534         }
 535
 536         if (asprintf(&idxpath, "%s/%s/pack-%s.idx", repo_path,
 537             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 538                 err = got_error_from_errno("asprintf");
 539                 goto done;
 540         }
 541
 542         if (rename(tmppackpath, packpath) == -1) {
 543                 err = got_error_from_errno3("rename", tmppackpath, packpath);
 544                 goto done;
 545         }
 546         free(tmppackpath);
 547         tmppackpath = NULL;
 548
 549         if (rename(tmpidxpath, idxpath) == -1) {
 550                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 551                 goto done;
 552         }
 553         free(tmpidxpath);
 554         tmpidxpath = NULL;
 555
 556  done:
 557         free(line);
 558         free(packpath);
 559         free(idxpath);
 560         free(id_str);
 561
 562         if (tmppackpath && unlink(tmppackpath) == -1 && err == NULL)
 563                 err = got_error_from_errno2("unlink", tmppackpath);
 564         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 565                 err = got_error_from_errno("close");
 566         free(tmppackpath);
 567
 568         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 569                 err = got_error_from_errno2("unlink", tmpidxpath);
 570         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 571                 err = got_error_from_errno("close");
 572         free(tmpidxpath);
 573
 574         if (imsg_idxfds[0] != -1 && close(imsg_idxfds[0]) == -1 && err == NULL)
 575                 err = got_error_from_errno("close");
 576         if (imsg_idxfds[1] != -1 && close(imsg_idxfds[1]) == -1 && err == NULL)
 577                 err = got_error_from_errno("close");
 578
 579         for (i = 0; i < nitems(tmpfds); ++i)
 580                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 581                         err = got_error_from_errno("close");
 582
 583         return err;
 584 }