lib/repository.c

   1 /*
   2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/types.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/socket.h>
  23 #include <sys/stat.h>
  24 #include <sys/mman.h>
  25 #include <sys/resource.h>
  26
  27 #include <ctype.h>
  28 #include <fcntl.h>
  29 #include <fnmatch.h>
  30 #include <limits.h>
  31 #include <dirent.h>
  32 #include <stdlib.h>
  33 #include <stdio.h>
  34 #include <string.h>
  35 #include <time.h>
  36 #include <unistd.h>
  37 #include <zlib.h>
  38 #include <errno.h>
  39 #include <libgen.h>
  40 #include <stdint.h>
  41
  42 #include "bloom.h"
  43
  44 #include "got_error.h"
  45 #include "got_reference.h"
  46 #include "got_repository.h"
  47 #include "got_path.h"
  48 #include "got_cancel.h"
  49 #include "got_object.h"
  50 #include "got_opentemp.h"
  51
  52 #include "got_lib_delta.h"
  53 #include "got_lib_delta_cache.h"
  54 #include "got_lib_inflate.h"
  55 #include "got_lib_object.h"
  56 #include "got_lib_object_parse.h"
  57 #include "got_lib_object_create.h"
  58 #include "got_lib_pack.h"
  59 #include "got_lib_privsep.h"
  60 #include "got_lib_hash.h"
  61 #include "got_lib_object_cache.h"
  62 #include "got_lib_repository.h"
  63 #include "got_lib_gotconfig.h"
  64
  65 #ifndef nitems
  66 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
  67 #endif
  68
  69 #define GOT_PACK_NUM_TEMPFILES          GOT_PACK_CACHE_SIZE * 2
  70
  71 RB_PROTOTYPE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
  72     got_packidx_bloom_filter_cmp);
  73
  74 static inline int
  75 is_boolean_val(const char *val)
  76 {
  77         return (strcasecmp(val, "true") == 0 ||
  78             strcasecmp(val, "false") == 0 ||
  79             strcasecmp(val, "on") == 0 ||
  80             strcasecmp(val, "off") == 0 ||
  81             strcasecmp(val, "yes") == 0 ||
  82             strcasecmp(val, "no") == 0 ||
  83             strcasecmp(val, "1") == 0 ||
  84             strcasecmp(val, "0") == 0);
  85 }
  86
  87 static inline int
  88 get_boolean_val(const char *val)
  89 {
  90         return (strcasecmp(val, "true") == 0 ||
  91             strcasecmp(val, "on") == 0 ||
  92             strcasecmp(val, "yes") == 0 ||
  93             strcasecmp(val, "1") == 0);
  94 }
  95
  96 const char *
  97 got_repo_get_path(struct got_repository *repo)
  98 {
  99         return repo->path;
 100 }
 101
 102 const char *
 103 got_repo_get_path_git_dir(struct got_repository *repo)
 104 {
 105         return repo->path_git_dir;
 106 }
 107
 108 int
 109 got_repo_get_fd(struct got_repository *repo)
 110 {
 111         return repo->gitdir_fd;
 112 }
 113
 114 enum got_hash_algorithm
 115 got_repo_get_object_format(struct got_repository *repo)
 116 {
 117         return repo->algo;
 118 }
 119
 120 const char *
 121 got_repo_get_gitconfig_author_name(struct got_repository *repo)
 122 {
 123         return repo->gitconfig_author_name;
 124 }
 125
 126 const char *
 127 got_repo_get_gitconfig_author_email(struct got_repository *repo)
 128 {
 129         return repo->gitconfig_author_email;
 130 }
 131
 132 const char *
 133 got_repo_get_global_gitconfig_author_name(struct got_repository *repo)
 134 {
 135         return repo->global_gitconfig_author_name;
 136 }
 137
 138 const char *
 139 got_repo_get_global_gitconfig_author_email(struct got_repository *repo)
 140 {
 141         return repo->global_gitconfig_author_email;
 142 }
 143
 144 const char *
 145 got_repo_get_gitconfig_owner(struct got_repository *repo)
 146 {
 147         return repo->gitconfig_owner;
 148 }
 149
 150 int
 151 got_repo_has_extension(struct got_repository *repo, const char *ext)
 152 {
 153         int i;
 154
 155         for (i = 0; i < repo->nextensions; ++i) {
 156                 if (!strcasecmp(ext, repo->extnames[i]))
 157                         return get_boolean_val(repo->extvals[i]);
 158         }
 159
 160         return 0;
 161 }
 162
 163 int
 164 got_repo_is_bare(struct got_repository *repo)
 165 {
 166         return (strcmp(repo->path, repo->path_git_dir) == 0);
 167 }
 168
 169 static char *
 170 get_path_git_child(struct got_repository *repo, const char *basename)
 171 {
 172         char *path_child;
 173
 174         if (asprintf(&path_child, "%s/%s", repo->path_git_dir,
 175             basename) == -1)
 176                 return NULL;
 177
 178         return path_child;
 179 }
 180
 181 char *
 182 got_repo_get_path_objects(struct got_repository *repo)
 183 {
 184         return get_path_git_child(repo, GOT_OBJECTS_DIR);
 185 }
 186
 187 char *
 188 got_repo_get_path_objects_pack(struct got_repository *repo)
 189 {
 190         return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR);
 191 }
 192
 193 char *
 194 got_repo_get_path_refs(struct got_repository *repo)
 195 {
 196         return get_path_git_child(repo, GOT_REFS_DIR);
 197 }
 198
 199 char *
 200 got_repo_get_path_packed_refs(struct got_repository *repo)
 201 {
 202         return get_path_git_child(repo, GOT_PACKED_REFS_FILE);
 203 }
 204
 205 static char *
 206 get_path_head(struct got_repository *repo)
 207 {
 208         return get_path_git_child(repo, GOT_HEAD_FILE);
 209 }
 210
 211 char *
 212 got_repo_get_path_gitconfig(struct got_repository *repo)
 213 {
 214         return get_path_git_child(repo, GOT_GITCONFIG);
 215 }
 216
 217 char *
 218 got_repo_get_path_gotconfig(struct got_repository *repo)
 219 {
 220         return get_path_git_child(repo, GOT_GOTCONFIG_FILENAME);
 221 }
 222
 223 const struct got_gotconfig *
 224 got_repo_get_gotconfig(struct got_repository *repo)
 225 {
 226         return repo->gotconfig;
 227 }
 228
 229 void
 230 got_repo_get_gitconfig_remotes(int *nremotes,
 231     const struct got_remote_repo **remotes, struct got_repository *repo)
 232 {
 233         *nremotes = repo->ngitconfig_remotes;
 234         *remotes = repo->gitconfig_remotes;
 235 }
 236
 237 static int
 238 is_git_repo(struct got_repository *repo)
 239 {
 240         const char *path_git = got_repo_get_path_git_dir(repo);
 241         char *path_objects = got_repo_get_path_objects(repo);
 242         char *path_refs = got_repo_get_path_refs(repo);
 243         char *path_head = get_path_head(repo);
 244         int ret = 0;
 245         struct stat sb;
 246         struct got_reference *head_ref;
 247
 248         if (lstat(path_git, &sb) == -1)
 249                 goto done;
 250         if (!S_ISDIR(sb.st_mode))
 251                 goto done;
 252
 253         if (lstat(path_objects, &sb) == -1)
 254                 goto done;
 255         if (!S_ISDIR(sb.st_mode))
 256                 goto done;
 257
 258         if (lstat(path_refs, &sb) == -1)
 259                 goto done;
 260         if (!S_ISDIR(sb.st_mode))
 261                 goto done;
 262
 263         if (lstat(path_head, &sb) == -1)
 264                 goto done;
 265         if (!S_ISREG(sb.st_mode))
 266                 goto done;
 267
 268         /* Check if the HEAD reference can be opened. */
 269         if (got_ref_open(&head_ref, repo, GOT_REF_HEAD, 0) != NULL)
 270                 goto done;
 271         got_ref_close(head_ref);
 272
 273         ret = 1;
 274 done:
 275         free(path_objects);
 276         free(path_refs);
 277         free(path_head);
 278         return ret;
 279
 280 }
 281
 282 static const struct got_error *
 283 close_tempfiles(int *fds, size_t nfds)
 284 {
 285         const struct got_error *err = NULL;
 286         int i;
 287
 288         for (i = 0; i < nfds; i++) {
 289                 if (fds[i] == -1)
 290                         continue;
 291                 if (close(fds[i]) == -1) {
 292                         err = got_error_from_errno("close");
 293                         break;
 294                 }
 295         }
 296         free(fds);
 297         return err;
 298 }
 299
 300 static const struct got_error *
 301 open_tempfiles(int **fds, size_t array_size, size_t nfds)
 302 {
 303         const struct got_error *err = NULL;
 304         int i;
 305
 306         *fds = calloc(array_size, sizeof(**fds));
 307         if (*fds == NULL)
 308                 return got_error_from_errno("calloc");
 309
 310         for (i = 0; i < array_size; i++)
 311                 (*fds)[i] = -1;
 312
 313         for (i = 0; i < nfds; i++) {
 314                 (*fds)[i] = got_opentempfd();
 315                 if ((*fds)[i] == -1) {
 316                         err = got_error_from_errno("got_opentempfd");
 317                         close_tempfiles(*fds, nfds);
 318                         *fds = NULL;
 319                         return err;
 320                 }
 321         }
 322
 323         return NULL;
 324 }
 325
 326 static const struct got_error *
 327 get_pack_cache_size(int *pack_cache_size)
 328 {
 329         struct rlimit rl;
 330
 331         if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
 332                 return got_error_from_errno("getrlimit");
 333
 334         *pack_cache_size = GOT_PACK_CACHE_SIZE;
 335         if (*pack_cache_size > rl.rlim_cur / 8)
 336                 *pack_cache_size = rl.rlim_cur / 8;
 337
 338         return NULL;
 339 }
 340
 341 const struct got_error *
 342 got_repo_pack_fds_open(int **pack_fds)
 343 {
 344         const struct got_error *err;
 345         int nfds;
 346
 347         err = get_pack_cache_size(&nfds);
 348         if (err)
 349                 return err;
 350
 351         /*
 352          * We need one basefd and one accumfd per cached pack.
 353          * Our constants should be set up in a way such that
 354          * this error never triggers.
 355          */
 356         if (nfds * 2 > GOT_PACK_NUM_TEMPFILES)
 357                 return got_error(GOT_ERR_NO_SPACE);
 358
 359         return open_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES, nfds * 2);
 360 }
 361
 362 const struct got_error *
 363 got_repo_pack_fds_close(int *pack_fds)
 364 {
 365         return close_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES);
 366 }
 367
 368 const struct got_error *
 369 got_repo_temp_fds_open(int **temp_fds)
 370 {
 371         return open_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES,
 372             GOT_REPO_NUM_TEMPFILES);
 373 }
 374
 375 void
 376 got_repo_temp_fds_set(struct got_repository *repo, int *temp_fds)
 377 {
 378         int i;
 379
 380         for (i = 0; i < GOT_REPO_NUM_TEMPFILES; i++)
 381                 repo->tempfiles[i] = temp_fds[i];
 382 }
 383
 384 const struct got_error *
 385 got_repo_temp_fds_get(int *fd, int *idx, struct got_repository *repo)
 386 {
 387         int i;
 388
 389         *fd = -1;
 390         *idx = -1;
 391
 392         for (i = 0; i < nitems(repo->tempfiles); i++) {
 393                 if (repo->tempfile_use_mask & (1 << i))
 394                         continue;
 395                 if (repo->tempfiles[i] != -1) {
 396                         if (ftruncate(repo->tempfiles[i], 0L) == -1)
 397                                 return got_error_from_errno("ftruncate");
 398                         *fd = repo->tempfiles[i];
 399                         *idx = i;
 400                         repo->tempfile_use_mask |= (1 << i);
 401                         return NULL;
 402                 }
 403         }
 404
 405         return got_error(GOT_ERR_REPO_TEMPFILE);
 406 }
 407
 408 void
 409 got_repo_temp_fds_put(int idx, struct got_repository *repo)
 410 {
 411         repo->tempfile_use_mask &= ~(1 << idx);
 412 }
 413
 414 const struct got_error *
 415 got_repo_temp_fds_close(int *temp_fds)
 416 {
 417         return close_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES);
 418 }
 419
 420 const struct got_error *
 421 got_repo_cache_object(struct got_repository *repo, struct got_object_id *id,
 422     struct got_object *obj)
 423 {
 424 #ifndef GOT_NO_OBJ_CACHE
 425         const struct got_error *err = NULL;
 426         err = got_object_cache_add(&repo->objcache, id, obj);
 427         if (err) {
 428                 if (err->code == GOT_ERR_OBJ_EXISTS ||
 429                     err->code == GOT_ERR_OBJ_TOO_LARGE)
 430                         err = NULL;
 431                 return err;
 432         }
 433         obj->refcnt++;
 434 #endif
 435         return NULL;
 436 }
 437
 438 struct got_object *
 439 got_repo_get_cached_object(struct got_repository *repo,
 440     struct got_object_id *id)
 441 {
 442         return (struct got_object *)got_object_cache_get(&repo->objcache, id);
 443 }
 444
 445 const struct got_error *
 446 got_repo_cache_tree(struct got_repository *repo, struct got_object_id *id,
 447     struct got_tree_object *tree)
 448 {
 449 #ifndef GOT_NO_OBJ_CACHE
 450         const struct got_error *err = NULL;
 451         err = got_object_cache_add(&repo->treecache, id, tree);
 452         if (err) {
 453                 if (err->code == GOT_ERR_OBJ_EXISTS ||
 454                     err->code == GOT_ERR_OBJ_TOO_LARGE)
 455                         err = NULL;
 456                 return err;
 457         }
 458         tree->refcnt++;
 459 #endif
 460         return NULL;
 461 }
 462
 463 struct got_tree_object *
 464 got_repo_get_cached_tree(struct got_repository *repo,
 465     struct got_object_id *id)
 466 {
 467         return (struct got_tree_object *)got_object_cache_get(
 468             &repo->treecache, id);
 469 }
 470
 471 const struct got_error *
 472 got_repo_cache_commit(struct got_repository *repo, struct got_object_id *id,
 473     struct got_commit_object *commit)
 474 {
 475 #ifndef GOT_NO_OBJ_CACHE
 476         const struct got_error *err = NULL;
 477         err = got_object_cache_add(&repo->commitcache, id, commit);
 478         if (err) {
 479                 if (err->code == GOT_ERR_OBJ_EXISTS ||
 480                     err->code == GOT_ERR_OBJ_TOO_LARGE)
 481                         err = NULL;
 482                 return err;
 483         }
 484         commit->refcnt++;
 485 #endif
 486         return NULL;
 487 }
 488
 489 struct got_commit_object *
 490 got_repo_get_cached_commit(struct got_repository *repo,
 491     struct got_object_id *id)
 492 {
 493         return (struct got_commit_object *)got_object_cache_get(
 494             &repo->commitcache, id);
 495 }
 496
 497 const struct got_error *
 498 got_repo_cache_tag(struct got_repository *repo, struct got_object_id *id,
 499     struct got_tag_object *tag)
 500 {
 501 #ifndef GOT_NO_OBJ_CACHE
 502         const struct got_error *err = NULL;
 503         err = got_object_cache_add(&repo->tagcache, id, tag);
 504         if (err) {
 505                 if (err->code == GOT_ERR_OBJ_EXISTS ||
 506                     err->code == GOT_ERR_OBJ_TOO_LARGE)
 507                         err = NULL;
 508                 return err;
 509         }
 510         tag->refcnt++;
 511 #endif
 512         return NULL;
 513 }
 514
 515 struct got_tag_object *
 516 got_repo_get_cached_tag(struct got_repository *repo, struct got_object_id *id)
 517 {
 518         return (struct got_tag_object *)got_object_cache_get(
 519             &repo->tagcache, id);
 520 }
 521
 522 const struct got_error *
 523 got_repo_cache_raw_object(struct got_repository *repo, struct got_object_id *id,
 524     struct got_raw_object *raw)
 525 {
 526 #ifndef GOT_NO_OBJ_CACHE
 527         const struct got_error *err = NULL;
 528         err = got_object_cache_add(&repo->rawcache, id, raw);
 529         if (err) {
 530                 if (err->code == GOT_ERR_OBJ_EXISTS ||
 531                     err->code == GOT_ERR_OBJ_TOO_LARGE)
 532                         err = NULL;
 533                 return err;
 534         }
 535         raw->refcnt++;
 536 #endif
 537         return NULL;
 538 }
 539
 540
 541 struct got_raw_object *
 542 got_repo_get_cached_raw_object(struct got_repository *repo,
 543     struct got_object_id *id)
 544 {
 545         return (struct got_raw_object *)got_object_cache_get(&repo->rawcache, id);
 546 }
 547
 548
 549 static const struct got_error *
 550 open_repo(struct got_repository *repo, const char *path)
 551 {
 552         const struct got_error *err = NULL;
 553
 554         repo->gitdir_fd = -1;
 555
 556         /* bare git repository? */
 557         repo->path_git_dir = strdup(path);
 558         if (repo->path_git_dir == NULL)
 559                 return got_error_from_errno("strdup");
 560         if (is_git_repo(repo)) {
 561                 repo->path = strdup(repo->path_git_dir);
 562                 if (repo->path == NULL) {
 563                         err = got_error_from_errno("strdup");
 564                         goto done;
 565                 }
 566                 repo->gitdir_fd = open(repo->path_git_dir,
 567                     O_DIRECTORY | O_CLOEXEC);
 568                 if (repo->gitdir_fd == -1) {
 569                         err = got_error_from_errno2("open",
 570                             repo->path_git_dir);
 571                         goto done;
 572                 }
 573                 return NULL;
 574         }
 575
 576         /* git repository with working tree? */
 577         free(repo->path_git_dir);
 578         repo->path_git_dir = NULL;
 579         if (asprintf(&repo->path_git_dir, "%s/%s", path, GOT_GIT_DIR) == -1) {
 580                 err = got_error_from_errno("asprintf");
 581                 goto done;
 582         }
 583         if (is_git_repo(repo)) {
 584                 repo->path = strdup(path);
 585                 if (repo->path == NULL) {
 586                         err = got_error_from_errno("strdup");
 587                         goto done;
 588                 }
 589                 repo->gitdir_fd = open(repo->path_git_dir,
 590                     O_DIRECTORY | O_CLOEXEC);
 591                 if (repo->gitdir_fd == -1) {
 592                         err = got_error_from_errno2("open",
 593                             repo->path_git_dir);
 594                         goto done;
 595                 }
 596                 return NULL;
 597         }
 598
 599         err = got_error(GOT_ERR_NOT_GIT_REPO);
 600 done:
 601         if (err) {
 602                 free(repo->path);
 603                 repo->path = NULL;
 604                 free(repo->path_git_dir);
 605                 repo->path_git_dir = NULL;
 606                 if (repo->gitdir_fd != -1)
 607                         close(repo->gitdir_fd);
 608                 repo->gitdir_fd = -1;
 609
 610         }
 611         return err;
 612 }
 613
 614 static const struct got_error *
 615 read_gitconfig(struct got_repository *repo, const char *global_gitconfig_path)
 616 {
 617         const struct got_error *err = NULL;
 618         char *repo_gitconfig_path = NULL;
 619
 620         if (global_gitconfig_path) {
 621                 /* Read settings from ~/.gitconfig. */
 622                 int dummy_repo_version;
 623                 err = got_repo_read_gitconfig(&dummy_repo_version,
 624                     &repo->global_gitconfig_author_name,
 625                     &repo->global_gitconfig_author_email,
 626                     NULL, NULL, NULL, NULL, NULL, NULL,
 627                     global_gitconfig_path);
 628                 if (err)
 629                         return err;
 630         }
 631
 632         /* Read repository's .git/config file. */
 633         repo_gitconfig_path = got_repo_get_path_gitconfig(repo);
 634         if (repo_gitconfig_path == NULL)
 635                 return got_error_from_errno("got_repo_get_path_gitconfig");
 636
 637         err = got_repo_read_gitconfig(
 638             &repo->gitconfig_repository_format_version,
 639             &repo->gitconfig_author_name, &repo->gitconfig_author_email,
 640             &repo->gitconfig_remotes, &repo->ngitconfig_remotes,
 641             &repo->gitconfig_owner, &repo->extnames, &repo->extvals,
 642             &repo->nextensions, repo_gitconfig_path);
 643         if (err)
 644                 goto done;
 645
 646         if (getenv("GOT_IGNORE_GITCONFIG") != NULL) {
 647                 int i;
 648
 649                 for (i = 0; i < repo->ngitconfig_remotes; i++) {
 650                         got_repo_free_remote_repo_data(
 651                             &repo->gitconfig_remotes[i]);
 652                 }
 653                 free(repo->gitconfig_remotes);
 654                 repo->gitconfig_remotes = NULL;
 655                 repo->ngitconfig_remotes = 0;
 656
 657                 free(repo->gitconfig_author_name);
 658                 repo->gitconfig_author_name = NULL;
 659                 free(repo->gitconfig_author_email);
 660                 repo->gitconfig_author_email = NULL;
 661
 662                 free(repo->global_gitconfig_author_name);
 663                 repo->global_gitconfig_author_name = NULL;
 664                 free(repo->global_gitconfig_author_email);
 665                 repo->global_gitconfig_author_email = NULL;
 666         }
 667
 668 done:
 669         free(repo_gitconfig_path);
 670         return err;
 671 }
 672
 673 static const struct got_error *
 674 read_gotconfig(struct got_repository *repo)
 675 {
 676         const struct got_error *err = NULL;
 677         char *gotconfig_path;
 678
 679         gotconfig_path = got_repo_get_path_gotconfig(repo);
 680         if (gotconfig_path == NULL)
 681                 return got_error_from_errno("got_repo_get_path_gotconfig");
 682
 683         err = got_gotconfig_read(&repo->gotconfig, gotconfig_path);
 684         free(gotconfig_path);
 685         return err;
 686 }
 687
 688 /* Supported repository format extensions. */
 689 static const char *const repo_extensions[] = {
 690         "noop",                 /* Got supports repository format version 1. */
 691         "preciousObjects",      /* Supported by gotadmin cleanup. */
 692         "worktreeConfig",       /* Got does not care about Git work trees. */
 693 };
 694
 695 const struct got_error *
 696 got_repo_open(struct got_repository **repop, const char *path,
 697     const char *global_gitconfig_path, int *pack_fds)
 698 {
 699         struct got_repository *repo = NULL;
 700         const struct got_error *err = NULL;
 701         char *repo_path = NULL;
 702         size_t i, j = 0;
 703
 704         *repop = NULL;
 705
 706         repo = calloc(1, sizeof(*repo));
 707         if (repo == NULL)
 708                 return got_error_from_errno("calloc");
 709
 710         RB_INIT(&repo->packidx_bloom_filters);
 711         TAILQ_INIT(&repo->packidx_paths);
 712
 713         for (i = 0; i < nitems(repo->privsep_children); i++) {
 714                 memset(&repo->privsep_children[i], 0,
 715                     sizeof(repo->privsep_children[0]));
 716                 repo->privsep_children[i].imsg_fd = -1;
 717         }
 718
 719         err = got_object_cache_init(&repo->objcache,
 720             GOT_OBJECT_CACHE_TYPE_OBJ);
 721         if (err)
 722                 goto done;
 723         err = got_object_cache_init(&repo->treecache,
 724             GOT_OBJECT_CACHE_TYPE_TREE);
 725         if (err)
 726                 goto done;
 727         err = got_object_cache_init(&repo->commitcache,
 728             GOT_OBJECT_CACHE_TYPE_COMMIT);
 729         if (err)
 730                 goto done;
 731         err = got_object_cache_init(&repo->tagcache,
 732             GOT_OBJECT_CACHE_TYPE_TAG);
 733         if (err)
 734                 goto done;
 735         err = got_object_cache_init(&repo->rawcache,
 736             GOT_OBJECT_CACHE_TYPE_RAW);
 737         if (err)
 738                 goto done;
 739
 740         err = get_pack_cache_size(&repo->pack_cache_size);
 741         if (err)
 742                 goto done;
 743         for (i = 0; i < nitems(repo->packs); i++) {
 744                 if (pack_fds != NULL && i < repo->pack_cache_size) {
 745                         repo->packs[i].basefd = pack_fds[j++];
 746                         repo->packs[i].accumfd = pack_fds[j++];
 747                 } else {
 748                         repo->packs[i].basefd = -1;
 749                         repo->packs[i].accumfd = -1;
 750                 }
 751         }
 752         for (i = 0; i < nitems(repo->tempfiles); i++)
 753                 repo->tempfiles[i] = -1;
 754         repo->pinned_pack = -1;
 755         repo->pinned_packidx = -1;
 756         repo->pinned_pid = 0;
 757
 758         repo_path = realpath(path, NULL);
 759         if (repo_path == NULL) {
 760                 err = got_error_from_errno2("realpath", path);
 761                 goto done;
 762         }
 763
 764         for (;;) {
 765                 char *parent_path;
 766
 767                 err = open_repo(repo, repo_path);
 768                 if (err == NULL)
 769                         break;
 770                 if (err->code != GOT_ERR_NOT_GIT_REPO)
 771                         goto done;
 772                 if (repo_path[0] == '/' && repo_path[1] == '\0') {
 773                         err = got_error(GOT_ERR_NOT_GIT_REPO);
 774                         goto done;
 775                 }
 776                 err = got_path_dirname(&parent_path, repo_path);
 777                 if (err)
 778                         goto done;
 779                 free(repo_path);
 780                 repo_path = parent_path;
 781         }
 782
 783         err = read_gotconfig(repo);
 784         if (err)
 785                 goto done;
 786
 787         err = read_gitconfig(repo, global_gitconfig_path);
 788         if (err)
 789                 goto done;
 790         if (repo->gitconfig_repository_format_version != 0) {
 791                 err = got_error_path(path, GOT_ERR_GIT_REPO_FORMAT);
 792                 goto done;
 793         }
 794         for (i = 0; i < repo->nextensions; i++) {
 795                 char *ext = repo->extnames[i];
 796                 char *val = repo->extvals[i];
 797                 int j, supported = 0;
 798
 799                 if (!is_boolean_val(val)) {
 800                         err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
 801                         goto done;
 802                 }
 803
 804                 if (!get_boolean_val(val))
 805                         continue;
 806
 807                 for (j = 0; j < nitems(repo_extensions); j++) {
 808                         if (strcmp(ext, repo_extensions[j]) == 0) {
 809                                 supported = 1;
 810                                 break;
 811                         }
 812                 }
 813                 if (!supported) {
 814                         err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
 815                         goto done;
 816                 }
 817         }
 818
 819         err = got_repo_list_packidx(&repo->packidx_paths, repo);
 820 done:
 821         if (err)
 822                 got_repo_close(repo);
 823         else
 824                 *repop = repo;
 825         free(repo_path);
 826         return err;
 827 }
 828
 829 const struct got_error *
 830 got_repo_close(struct got_repository *repo)
 831 {
 832         const struct got_error *err = NULL, *child_err;
 833         struct got_packidx_bloom_filter *bf;
 834         size_t i;
 835
 836         for (i = 0; i < repo->pack_cache_size; i++) {
 837                 if (repo->packidx_cache[i] == NULL)
 838                         break;
 839                 got_packidx_close(repo->packidx_cache[i]);
 840         }
 841
 842         while ((bf = RB_MIN(got_packidx_bloom_filter_tree,
 843             &repo->packidx_bloom_filters))) {
 844                 RB_REMOVE(got_packidx_bloom_filter_tree,
 845                     &repo->packidx_bloom_filters, bf);
 846                 bloom_free(bf->bloom);
 847                 free(bf->bloom);
 848                 free(bf);
 849         }
 850
 851         for (i = 0; i < repo->pack_cache_size; i++)
 852                 if (repo->packs[i].path_packfile)
 853                         if (repo->packs[i].path_packfile)
 854                                 got_pack_close(&repo->packs[i]);
 855
 856         free(repo->path);
 857         free(repo->path_git_dir);
 858
 859         got_object_cache_close(&repo->objcache);
 860         got_object_cache_close(&repo->treecache);
 861         got_object_cache_close(&repo->commitcache);
 862         got_object_cache_close(&repo->tagcache);
 863         got_object_cache_close(&repo->rawcache);
 864
 865         for (i = 0; i < nitems(repo->privsep_children); i++) {
 866                 if (repo->privsep_children[i].imsg_fd == -1)
 867                         continue;
 868                 imsg_clear(repo->privsep_children[i].ibuf);
 869                 free(repo->privsep_children[i].ibuf);
 870                 err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd);
 871                 if (err && err->code == GOT_ERR_EOF)
 872                         err = NULL;
 873                 child_err = got_privsep_wait_for_child(
 874                     repo->privsep_children[i].pid);
 875                 if (child_err && err == NULL)
 876                         err = child_err;
 877                 if (close(repo->privsep_children[i].imsg_fd) == -1 &&
 878                     err == NULL)
 879                         err = got_error_from_errno("close");
 880         }
 881
 882         if (repo->gitdir_fd != -1 && close(repo->gitdir_fd) == -1 &&
 883             err == NULL)
 884                 err = got_error_from_errno("close");
 885
 886         if (repo->gotconfig)
 887                 got_gotconfig_free(repo->gotconfig);
 888         free(repo->gitconfig_author_name);
 889         free(repo->gitconfig_author_email);
 890         for (i = 0; i < repo->ngitconfig_remotes; i++)
 891                 got_repo_free_remote_repo_data(&repo->gitconfig_remotes[i]);
 892         free(repo->gitconfig_remotes);
 893         for (i = 0; i < repo->nextensions; i++) {
 894                 free(repo->extnames[i]);
 895                 free(repo->extvals[i]);
 896         }
 897         free(repo->extnames);
 898         free(repo->extvals);
 899
 900         got_pathlist_free(&repo->packidx_paths, GOT_PATHLIST_FREE_PATH);
 901         free(repo);
 902
 903         return err;
 904 }
 905
 906 const struct got_error *
 907 got_repo_remote_repo_dup(struct got_remote_repo **newp,
 908     const struct got_remote_repo *repo)
 909 {
 910         const struct got_error *err = NULL;
 911         struct got_remote_repo *new;
 912         int i;
 913
 914         new = calloc(1, sizeof(*new));
 915         if (new == NULL)
 916                 return got_error_from_errno("calloc");
 917
 918         if (repo->name) {
 919                 new->name = strdup(repo->name);
 920                 if (new->name == NULL) {
 921                         err = got_error_from_errno("strdup");
 922                         goto done;
 923                 }
 924         }
 925
 926         if (repo->fetch_url) {
 927                 new->fetch_url = strdup(repo->fetch_url);
 928                 if (new->fetch_url == NULL) {
 929                         err = got_error_from_errno("strdup");
 930                         goto done;
 931                 }
 932         }
 933
 934         if (repo->send_url) {
 935                 new->send_url = strdup(repo->send_url);
 936                 if (new->send_url == NULL) {
 937                         err = got_error_from_errno("strdup");
 938                         goto done;
 939                 }
 940         }
 941
 942         new->mirror_references = repo->mirror_references;
 943
 944         new->fetch_all_branches = repo->fetch_all_branches;
 945
 946         new->nfetch_branches = repo->nfetch_branches;
 947         if (repo->fetch_branches) {
 948                 new->fetch_branches = calloc(repo->nfetch_branches,
 949                     sizeof(char *));
 950                 if (new->fetch_branches == NULL) {
 951                         err = got_error_from_errno("calloc");
 952                         goto done;
 953                 }
 954                 for (i = 0; i < repo->nfetch_branches; i++) {
 955                         new->fetch_branches[i] = strdup(
 956                             repo->fetch_branches[i]);
 957                         if (new->fetch_branches[i] == NULL) {
 958                                 err = got_error_from_errno("strdup");
 959                                 goto done;
 960                         }
 961                 }
 962         }
 963
 964         new->nsend_branches = repo->nsend_branches;
 965         if (repo->send_branches) {
 966                 new->send_branches = calloc(repo->nsend_branches,
 967                     sizeof(char *));
 968                 if (new->send_branches == NULL) {
 969                         err = got_error_from_errno("calloc");
 970                         goto done;
 971                 }
 972                 for (i = 0; i < repo->nsend_branches; i++) {
 973                         new->send_branches[i] = strdup(
 974                             repo->send_branches[i]);
 975                         if (new->send_branches[i] == NULL) {
 976                                 err = got_error_from_errno("strdup");
 977                                 goto done;
 978                         }
 979                 }
 980         }
 981
 982         new->nfetch_refs = repo->nfetch_refs;
 983         if (repo->fetch_refs) {
 984                 new->fetch_refs = calloc(repo->nfetch_refs,
 985                     sizeof(char *));
 986                 if (new->fetch_refs == NULL) {
 987                         err = got_error_from_errno("calloc");
 988                         goto done;
 989                 }
 990                 for (i = 0; i < repo->nfetch_refs; i++) {
 991                         new->fetch_refs[i] = strdup(
 992                             repo->fetch_refs[i]);
 993                         if (new->fetch_refs[i] == NULL) {
 994                                 err = got_error_from_errno("strdup");
 995                                 goto done;
 996                         }
 997                 }
 998         }
 999 done:
1000         if (err) {
1001                 got_repo_free_remote_repo_data(new);
1002                 free(new);
1003         } else
1004                 *newp = new;
1005
1006         return err;
1007 }
1008
1009 void
1010 got_repo_free_remote_repo_data(struct got_remote_repo *repo)
1011 {
1012         int i;
1013
1014         if (repo == NULL)
1015                 return;
1016
1017         free(repo->name);
1018         repo->name = NULL;
1019         free(repo->fetch_url);
1020         repo->fetch_url = NULL;
1021         free(repo->send_url);
1022         repo->send_url = NULL;
1023         for (i = 0; i < repo->nfetch_branches; i++)
1024                 free(repo->fetch_branches[i]);
1025         free(repo->fetch_branches);
1026         repo->fetch_branches = NULL;
1027         repo->nfetch_branches = 0;
1028         for (i = 0; i < repo->nsend_branches; i++)
1029                 free(repo->send_branches[i]);
1030         free(repo->send_branches);
1031         repo->send_branches = NULL;
1032         repo->nsend_branches = 0;
1033         for (i = 0; i < repo->nfetch_refs; i++)
1034                 free(repo->fetch_refs[i]);
1035         free(repo->fetch_refs);
1036         repo->fetch_refs = NULL;
1037         repo->nfetch_refs = 0;
1038 }
1039
1040 const struct got_error *
1041 got_repo_map_path(char **in_repo_path, struct got_repository *repo,
1042     const char *input_path)
1043 {
1044         const struct got_error *err = NULL;
1045         const char *repo_abspath = NULL;
1046         size_t repolen, len;
1047         char *canonpath, *path = NULL;
1048
1049         *in_repo_path = NULL;
1050
1051         canonpath = strdup(input_path);
1052         if (canonpath == NULL) {
1053                 err = got_error_from_errno("strdup");
1054                 goto done;
1055         }
1056         err = got_canonpath(input_path, canonpath, strlen(canonpath) + 1);
1057         if (err)
1058                 goto done;
1059
1060         repo_abspath = got_repo_get_path(repo);
1061
1062         if (canonpath[0] == '\0') {
1063                 path = strdup(canonpath);
1064                 if (path == NULL) {
1065                         err = got_error_from_errno("strdup");
1066                         goto done;
1067                 }
1068         } else {
1069                 path = realpath(canonpath, NULL);
1070                 if (path == NULL) {
1071                         if (errno != ENOENT) {
1072                                 err = got_error_from_errno2("realpath",
1073                                     canonpath);
1074                                 goto done;
1075                         }
1076                         /*
1077                          * Path is not on disk.
1078                          * Assume it is already relative to repository root.
1079                          */
1080                         path = strdup(canonpath);
1081                         if (path == NULL) {
1082                                 err = got_error_from_errno("strdup");
1083                                 goto done;
1084                         }
1085                 }
1086
1087                 repolen = strlen(repo_abspath);
1088                 len = strlen(path);
1089
1090
1091                 if (strcmp(path, repo_abspath) == 0) {
1092                         free(path);
1093                         path = strdup("");
1094                         if (path == NULL) {
1095                                 err = got_error_from_errno("strdup");
1096                                 goto done;
1097                         }
1098                 } else if (len > repolen &&
1099                     got_path_is_child(path, repo_abspath, repolen)) {
1100                         /* Matched an on-disk path inside repository. */
1101                         if (got_repo_is_bare(repo)) {
1102                                 /*
1103                                  * Matched an on-disk path inside repository
1104                                  * database. Treat input as repository-relative.
1105                                  */
1106                                 free(path);
1107                                 path = canonpath;
1108                                 canonpath = NULL;
1109                         } else {
1110                                 char *child;
1111                                 /* Strip common prefix with repository path. */
1112                                 err = got_path_skip_common_ancestor(&child,
1113                                     repo_abspath, path);
1114                                 if (err)
1115                                         goto done;
1116                                 free(path);
1117                                 path = child;
1118                         }
1119                 } else {
1120                         /*
1121                          * Matched unrelated on-disk path.
1122                          * Treat input as repository-relative.
1123                          */
1124                         free(path);
1125                         path = canonpath;
1126                         canonpath = NULL;
1127                 }
1128         }
1129
1130         /* Make in-repository path absolute */
1131         if (path[0] != '/') {
1132                 char *abspath;
1133                 if (asprintf(&abspath, "/%s", path) == -1) {
1134                         err = got_error_from_errno("asprintf");
1135                         goto done;
1136                 }
1137                 free(path);
1138                 path = abspath;
1139         }
1140
1141 done:
1142         free(canonpath);
1143         if (err)
1144                 free(path);
1145         else
1146                 *in_repo_path = path;
1147         return err;
1148 }
1149
1150 static const struct got_error *
1151 cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
1152     const char *path_packidx)
1153 {
1154         const struct got_error *err = NULL;
1155         size_t i;
1156
1157         for (i = 0; i < repo->pack_cache_size; i++) {
1158                 if (repo->packidx_cache[i] == NULL)
1159                         break;
1160                 if (strcmp(repo->packidx_cache[i]->path_packidx,
1161                     path_packidx) == 0) {
1162                         return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1163                 }
1164         }
1165         if (i == repo->pack_cache_size) {
1166                 do {
1167                         i--;
1168                 } while (i > 0 && repo->pinned_packidx >= 0 &&
1169                     i == repo->pinned_packidx);
1170                 err = got_packidx_close(repo->packidx_cache[i]);
1171                 if (err)
1172                         return err;
1173         }
1174
1175         repo->packidx_cache[i] = packidx;
1176
1177         return NULL;
1178 }
1179
1180 int
1181 got_repo_is_packidx_filename(const char *name, size_t len)
1182 {
1183         if (len != GOT_PACKIDX_NAMELEN)
1184                 return 0;
1185
1186         if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
1187                 return 0;
1188
1189         if (strcmp(name + strlen(GOT_PACK_PREFIX) +
1190             SHA1_DIGEST_STRING_LENGTH - 1, GOT_PACKIDX_SUFFIX) != 0)
1191                 return 0;
1192
1193         return 1;
1194 }
1195
1196 static struct got_packidx_bloom_filter *
1197 get_packidx_bloom_filter(struct got_repository *repo,
1198     const char *path, size_t path_len)
1199 {
1200         struct got_packidx_bloom_filter key;
1201
1202         if (strlcpy(key.path, path, sizeof(key.path)) >= sizeof(key.path))
1203                 return NULL; /* XXX */
1204         key.path_len = path_len;
1205
1206         return RB_FIND(got_packidx_bloom_filter_tree,
1207             &repo->packidx_bloom_filters, &key);
1208 }
1209
1210 int
1211 got_repo_check_packidx_bloom_filter(struct got_repository *repo,
1212     const char *path_packidx, struct got_object_id *id)
1213 {
1214         struct got_packidx_bloom_filter *bf;
1215
1216         bf = get_packidx_bloom_filter(repo, path_packidx, strlen(path_packidx));
1217         if (bf)
1218                 return bloom_check(bf->bloom, id->sha1, sizeof(id->sha1));
1219
1220         /* No bloom filter means this pack index must be searched. */
1221         return 1;
1222 }
1223
1224 static const struct got_error *
1225 add_packidx_bloom_filter(struct got_repository *repo,
1226     struct got_packidx *packidx, const char *path_packidx)
1227 {
1228         int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1229         struct got_packidx_bloom_filter *bf;
1230         size_t len;
1231
1232         /*
1233          * Don't use bloom filters for very large pack index files.
1234          * Large pack files will contain a relatively large fraction
1235          * of our objects so we will likely need to visit them anyway.
1236          * The more objects a pack file contains the higher the probability
1237          * of a false-positive match from the bloom filter. And reading
1238          * all object IDs from a large pack index file can be expensive.
1239          */
1240         if (nobjects > 100000) /* cut-off at about 2MB, at 20 bytes per ID */
1241                 return NULL;
1242
1243         /* Do we already have a filter for this pack index? */
1244         if (get_packidx_bloom_filter(repo, path_packidx,
1245             strlen(path_packidx)) != NULL)
1246                 return NULL;
1247
1248         bf = calloc(1, sizeof(*bf));
1249         if (bf == NULL)
1250                 return got_error_from_errno("calloc");
1251         bf->bloom = calloc(1, sizeof(*bf->bloom));
1252         if (bf->bloom == NULL) {
1253                 free(bf);
1254                 return got_error_from_errno("calloc");
1255         }
1256
1257         len = strlcpy(bf->path, path_packidx, sizeof(bf->path));
1258         if (len >= sizeof(bf->path)) {
1259                 free(bf->bloom);
1260                 free(bf);
1261                 return got_error(GOT_ERR_NO_SPACE);
1262         }
1263         bf->path_len = len;
1264
1265         /* Minimum size supported by our bloom filter is 1000 entries. */
1266         bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
1267         for (i = 0; i < nobjects; i++) {
1268                 struct got_packidx_object_id *id;
1269                 id = &packidx->hdr.sorted_ids[i];
1270                 bloom_add(bf->bloom, id->sha1, sizeof(id->sha1));
1271         }
1272
1273         RB_INSERT(got_packidx_bloom_filter_tree,
1274             &repo->packidx_bloom_filters, bf);
1275         return NULL;
1276 }
1277
1278 static void
1279 purge_packidx_paths(struct got_pathlist_head *packidx_paths)
1280 {
1281         struct got_pathlist_entry *pe;
1282
1283         while (!TAILQ_EMPTY(packidx_paths)) {
1284                 pe = TAILQ_FIRST(packidx_paths);
1285                 TAILQ_REMOVE(packidx_paths, pe, entry);
1286                 free((char *)pe->path);
1287                 free(pe);
1288         }
1289 }
1290
1291 static const struct got_error *
1292 refresh_packidx_paths(struct got_repository *repo)
1293 {
1294         const struct got_error *err = NULL;
1295         char *objects_pack_dir = NULL;
1296         struct stat sb;
1297
1298         objects_pack_dir = got_repo_get_path_objects_pack(repo);
1299         if (objects_pack_dir == NULL)
1300                 return got_error_from_errno("got_repo_get_path_objects_pack");
1301
1302         if (stat(objects_pack_dir, &sb) == -1) {
1303                 if (errno != ENOENT) {
1304                         err = got_error_from_errno2("stat", objects_pack_dir);
1305                         goto done;
1306                 }
1307         } else if (TAILQ_EMPTY(&repo->packidx_paths) ||
1308             sb.st_mtim.tv_sec != repo->pack_path_mtime.tv_sec ||
1309             sb.st_mtim.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1310                 purge_packidx_paths(&repo->packidx_paths);
1311                 err = got_repo_list_packidx(&repo->packidx_paths, repo);
1312                 if (err)
1313                         goto done;
1314         }
1315 done:
1316         free(objects_pack_dir);
1317         return err;
1318 }
1319
1320 const struct got_error *
1321 got_repo_search_packidx(struct got_packidx **packidx, int *idx,
1322     struct got_repository *repo, struct got_object_id *id)
1323 {
1324         const struct got_error *err;
1325         struct got_pathlist_entry *pe;
1326         size_t i;
1327
1328         /* Search pack index cache. */
1329         for (i = 0; i < repo->pack_cache_size; i++) {
1330                 if (repo->packidx_cache[i] == NULL)
1331                         break;
1332                 if (!got_repo_check_packidx_bloom_filter(repo,
1333                     repo->packidx_cache[i]->path_packidx, id))
1334                         continue; /* object will not be found in this index */
1335                 *idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
1336                 if (*idx != -1) {
1337                         *packidx = repo->packidx_cache[i];
1338                         /*
1339                          * Move this cache entry to the front. Repeatedly
1340                          * searching a wrong pack index can be expensive.
1341                          */
1342                         if (i > 0) {
1343                                 memmove(&repo->packidx_cache[1],
1344                                     &repo->packidx_cache[0],
1345                                     i * sizeof(repo->packidx_cache[0]));
1346                                 repo->packidx_cache[0] = *packidx;
1347                                 if (repo->pinned_packidx >= 0 &&
1348                                     repo->pinned_packidx < i)
1349                                         repo->pinned_packidx++;
1350                                 else if (repo->pinned_packidx == i)
1351                                         repo->pinned_packidx = 0;
1352                         }
1353                         return NULL;
1354                 }
1355         }
1356         /* No luck. Search the filesystem. */
1357
1358         err = refresh_packidx_paths(repo);
1359         if (err)
1360                 return err;
1361
1362         TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1363                 const char *path_packidx = pe->path;
1364                 int is_cached = 0;
1365
1366                 if (!got_repo_check_packidx_bloom_filter(repo,
1367                     pe->path, id))
1368                         continue; /* object will not be found in this index */
1369
1370                 for (i = 0; i < repo->pack_cache_size; i++) {
1371                         if (repo->packidx_cache[i] == NULL)
1372                                 break;
1373                         if (strcmp(repo->packidx_cache[i]->path_packidx,
1374                             path_packidx) == 0) {
1375                                 is_cached = 1;
1376                                 break;
1377                         }
1378                 }
1379                 if (is_cached)
1380                         continue; /* already searched */
1381
1382                 err = got_packidx_open(packidx, got_repo_get_fd(repo),
1383                     path_packidx, 0);
1384                 if (err)
1385                         goto done;
1386
1387                 err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1388                 if (err)
1389                         goto done;
1390
1391                 err = cache_packidx(repo, *packidx, path_packidx);
1392                 if (err)
1393                         goto done;
1394
1395                 *idx = got_packidx_get_object_idx(*packidx, id);
1396                 if (*idx != -1) {
1397                         err = NULL; /* found the object */
1398                         goto done;
1399                 }
1400         }
1401
1402         err = got_error_no_obj(id);
1403 done:
1404         return err;
1405 }
1406
1407 const struct got_error *
1408 got_repo_list_packidx(struct got_pathlist_head *packidx_paths,
1409     struct got_repository *repo)
1410 {
1411         const struct got_error *err = NULL;
1412         DIR *packdir = NULL;
1413         struct dirent *dent;
1414         char *path_packidx = NULL;
1415         int packdir_fd;
1416         struct stat sb;
1417
1418         packdir_fd = openat(got_repo_get_fd(repo),
1419             GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1420         if (packdir_fd == -1) {
1421                 return got_error_from_errno_fmt("openat: %s/%s",
1422                     got_repo_get_path_git_dir(repo),
1423                     GOT_OBJECTS_PACK_DIR);
1424         }
1425
1426         packdir = fdopendir(packdir_fd);
1427         if (packdir == NULL) {
1428                 err = got_error_from_errno("fdopendir");
1429                 goto done;
1430         }
1431
1432         if (fstat(packdir_fd, &sb) == -1) {
1433                 err = got_error_from_errno("fstat");
1434                 goto done;
1435         }
1436         repo->pack_path_mtime.tv_sec = sb.st_mtim.tv_sec;
1437         repo->pack_path_mtime.tv_nsec = sb.st_mtim.tv_nsec;
1438
1439         while ((dent = readdir(packdir)) != NULL) {
1440                 if (!got_repo_is_packidx_filename(dent->d_name,
1441                     strlen(dent->d_name)))
1442                         continue;
1443
1444                 if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
1445                     dent->d_name) == -1) {
1446                         err = got_error_from_errno("asprintf");
1447                         path_packidx = NULL;
1448                         break;
1449                 }
1450
1451                 err = got_pathlist_append(packidx_paths, path_packidx, NULL);
1452                 if (err)
1453                         break;
1454         }
1455 done:
1456         if (err)
1457                 free(path_packidx);
1458         if (packdir && closedir(packdir) != 0 && err == NULL)
1459                 err = got_error_from_errno("closedir");
1460         return err;
1461 }
1462
1463 const struct got_error *
1464 got_repo_get_packidx(struct got_packidx **packidx, const char *path_packidx,
1465     struct got_repository *repo)
1466 {
1467         const struct got_error *err;
1468         size_t i;
1469
1470         *packidx = NULL;
1471
1472         /* Search pack index cache. */
1473         for (i = 0; i < repo->pack_cache_size; i++) {
1474                 if (repo->packidx_cache[i] == NULL)
1475                         break;
1476                 if (strcmp(repo->packidx_cache[i]->path_packidx,
1477                     path_packidx) == 0) {
1478                         *packidx = repo->packidx_cache[i];
1479                         return NULL;
1480                 }
1481         }
1482         /* No luck. Search the filesystem. */
1483
1484         err = got_packidx_open(packidx, got_repo_get_fd(repo),
1485             path_packidx, 0);
1486         if (err)
1487                 return err;
1488
1489         err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1490         if (err)
1491                 goto done;
1492
1493         err = cache_packidx(repo, *packidx, path_packidx);
1494 done:
1495         if (err) {
1496                 got_packidx_close(*packidx);
1497                 *packidx = NULL;
1498         }
1499         return err;
1500 }
1501
1502 static const struct got_error *
1503 read_packfile_hdr(int fd, struct got_packidx *packidx)
1504 {
1505         const struct got_error *err = NULL;
1506         uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
1507         struct got_packfile_hdr hdr;
1508         ssize_t n;
1509
1510         n = read(fd, &hdr, sizeof(hdr));
1511         if (n < 0)
1512                 return got_error_from_errno("read");
1513         if (n != sizeof(hdr))
1514                 return got_error(GOT_ERR_BAD_PACKFILE);
1515
1516         if (be32toh(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
1517             be32toh(hdr.version) != GOT_PACKFILE_VERSION ||
1518             be32toh(hdr.nobjects) != totobj)
1519                 err = got_error(GOT_ERR_BAD_PACKFILE);
1520
1521         return err;
1522 }
1523
1524 static const struct got_error *
1525 open_packfile(int *fd, struct got_repository *repo,
1526     const char *relpath, struct got_packidx *packidx)
1527 {
1528         const struct got_error *err = NULL;
1529
1530         *fd = openat(got_repo_get_fd(repo), relpath,
1531             O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1532         if (*fd == -1)
1533                 return got_error_from_errno_fmt("openat: %s/%s",
1534                     got_repo_get_path_git_dir(repo), relpath);
1535
1536         if (packidx) {
1537                 err = read_packfile_hdr(*fd, packidx);
1538                 if (err) {
1539                         close(*fd);
1540                         *fd = -1;
1541                 }
1542         }
1543
1544         return err;
1545 }
1546
1547 const struct got_error *
1548 got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
1549     const char *path_packfile, struct got_packidx *packidx)
1550 {
1551         const struct got_error *err = NULL;
1552         struct got_pack *pack = NULL;
1553         struct stat sb;
1554         size_t i;
1555
1556         if (packp)
1557                 *packp = NULL;
1558
1559         for (i = 0; i < repo->pack_cache_size; i++) {
1560                 pack = &repo->packs[i];
1561                 if (pack->path_packfile == NULL)
1562                         break;
1563                 if (strcmp(pack->path_packfile, path_packfile) == 0)
1564                         return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1565         }
1566
1567         if (i == repo->pack_cache_size) {
1568                 struct got_pack tmp;
1569                 do {
1570                         i--;
1571                 } while (i > 0 && repo->pinned_pack >= 0 &&
1572                     i == repo->pinned_pack);
1573                 err = got_pack_close(&repo->packs[i]);
1574                 if (err)
1575                         return err;
1576                 if (ftruncate(repo->packs[i].basefd, 0L) == -1)
1577                         return got_error_from_errno("ftruncate");
1578                 if (ftruncate(repo->packs[i].accumfd, 0L) == -1)
1579                         return got_error_from_errno("ftruncate");
1580                 memcpy(&tmp, &repo->packs[i], sizeof(tmp));
1581                 memcpy(&repo->packs[i], &repo->packs[0],
1582                     sizeof(repo->packs[i]));
1583                 memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0]));
1584                 if (repo->pinned_pack == 0)
1585                         repo->pinned_pack = i;
1586                 else if (repo->pinned_pack == i)
1587                         repo->pinned_pack = 0;
1588                 i = 0;
1589         }
1590
1591         pack = &repo->packs[i];
1592
1593         pack->path_packfile = strdup(path_packfile);
1594         if (pack->path_packfile == NULL) {
1595                 err = got_error_from_errno("strdup");
1596                 goto done;
1597         }
1598
1599         err = open_packfile(&pack->fd, repo, path_packfile, packidx);
1600         if (err)
1601                 goto done;
1602
1603         if (fstat(pack->fd, &sb) != 0) {
1604                 err = got_error_from_errno("fstat");
1605                 goto done;
1606         }
1607         pack->filesize = sb.st_size;
1608
1609         pack->privsep_child = NULL;
1610
1611         err = got_delta_cache_alloc(&pack->delta_cache);
1612         if (err)
1613                 goto done;
1614
1615 #ifndef GOT_PACK_NO_MMAP
1616         if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1617                 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1618                     pack->fd, 0);
1619                 if (pack->map == MAP_FAILED) {
1620                         if (errno != ENOMEM) {
1621                                 err = got_error_from_errno("mmap");
1622                                 goto done;
1623                         }
1624                         pack->map = NULL; /* fall back to read(2) */
1625                 }
1626         }
1627 #endif
1628 done:
1629         if (err) {
1630                 if (pack)
1631                         got_pack_close(pack);
1632         } else if (packp)
1633                 *packp = pack;
1634         return err;
1635 }
1636
1637 struct got_pack *
1638 got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
1639 {
1640         struct got_pack *pack = NULL;
1641         size_t i;
1642
1643         for (i = 0; i < repo->pack_cache_size; i++) {
1644                 pack = &repo->packs[i];
1645                 if (pack->path_packfile == NULL)
1646                         break;
1647                 if (strcmp(pack->path_packfile, path_packfile) == 0)
1648                         return pack;
1649         }
1650
1651         return NULL;
1652 }
1653
1654 const struct got_error *
1655 got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx,
1656     struct got_pack *pack)
1657 {
1658         size_t i;
1659         int pinned_pack = -1, pinned_packidx = -1;
1660
1661         for (i = 0; i < repo->pack_cache_size; i++) {
1662                 if (repo->packidx_cache[i] &&
1663                     strcmp(repo->packidx_cache[i]->path_packidx,
1664                     packidx->path_packidx) == 0)
1665                         pinned_packidx = i;
1666                 if (repo->packs[i].path_packfile &&
1667                     strcmp(repo->packs[i].path_packfile,
1668                     pack->path_packfile) == 0)
1669                         pinned_pack = i;
1670         }
1671
1672         if (pinned_packidx == -1 || pinned_pack == -1)
1673                 return got_error(GOT_ERR_PIN_PACK);
1674
1675         repo->pinned_pack = pinned_pack;
1676         repo->pinned_packidx = pinned_packidx;
1677         if (repo->packs[pinned_pack].privsep_child)
1678                 repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid;
1679         return NULL;
1680 }
1681
1682 struct got_pack *
1683 got_repo_get_pinned_pack(struct got_repository *repo)
1684 {
1685         if (repo->pinned_pack >= 0 &&
1686             repo->pinned_pack < repo->pack_cache_size)
1687                 return &repo->packs[repo->pinned_pack];
1688
1689         return NULL;
1690 }
1691
1692 void
1693 got_repo_unpin_pack(struct got_repository *repo)
1694 {
1695         repo->pinned_packidx = -1;
1696         repo->pinned_pack = -1;
1697         repo->pinned_pid = 0;
1698 }
1699
1700 const struct got_error *
1701 got_repo_init(const char *repo_path, const char *head_name)
1702 {
1703         const struct got_error *err = NULL;
1704         const char *dirnames[] = {
1705                 GOT_OBJECTS_DIR,
1706                 GOT_OBJECTS_PACK_DIR,
1707                 GOT_REFS_DIR,
1708         };
1709         const char *description_str = "Unnamed repository; "
1710             "edit this file 'description' to name the repository.";
1711         const char *headref = "ref: refs/heads/";
1712         const char *gitconfig_str = "[core]\n"
1713             "\trepositoryformatversion = 0\n"
1714             "\tfilemode = true\n"
1715             "\tbare = true\n";
1716         char *headref_str, *path;
1717         size_t i;
1718
1719         if (!got_path_dir_is_empty(repo_path))
1720                 return got_error(GOT_ERR_DIR_NOT_EMPTY);
1721
1722         for (i = 0; i < nitems(dirnames); i++) {
1723                 if (asprintf(&path, "%s/%s", repo_path, dirnames[i]) == -1) {
1724                         return got_error_from_errno("asprintf");
1725                 }
1726                 err = got_path_mkdir(path);
1727                 free(path);
1728                 if (err)
1729                         return err;
1730         }
1731
1732         if (asprintf(&path, "%s/%s", repo_path, "description") == -1)
1733                 return got_error_from_errno("asprintf");
1734         err = got_path_create_file(path, description_str);
1735         free(path);
1736         if (err)
1737                 return err;
1738
1739         if (asprintf(&path, "%s/%s", repo_path, GOT_HEAD_FILE) == -1)
1740                 return got_error_from_errno("asprintf");
1741         if (asprintf(&headref_str, "%s%s", headref,
1742             head_name ? head_name : "main") == -1) {
1743                 free(path);
1744                 return got_error_from_errno("asprintf");
1745         }
1746         err = got_path_create_file(path, headref_str);
1747         free(headref_str);
1748         free(path);
1749         if (err)
1750                 return err;
1751
1752         if (asprintf(&path, "%s/%s", repo_path, "config") == -1)
1753                 return got_error_from_errno("asprintf");
1754         err = got_path_create_file(path, gitconfig_str);
1755         free(path);
1756         if (err)
1757                 return err;
1758
1759         return NULL;
1760 }
1761
1762 static const struct got_error *
1763 match_packed_object(struct got_object_id **unique_id,
1764     struct got_repository *repo, const char *id_str_prefix, int obj_type)
1765 {
1766         const struct got_error *err = NULL;
1767         struct got_object_id_queue matched_ids;
1768         struct got_pathlist_entry *pe;
1769
1770         STAILQ_INIT(&matched_ids);
1771
1772         err = refresh_packidx_paths(repo);
1773         if (err)
1774                 return err;
1775
1776         TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1777                 const char *path_packidx = pe->path;
1778                 struct got_packidx *packidx;
1779                 struct got_object_qid *qid;
1780
1781                 err = got_packidx_open(&packidx, got_repo_get_fd(repo),
1782                     path_packidx, 0);
1783                 if (err)
1784                         break;
1785
1786                 err = got_packidx_match_id_str_prefix(&matched_ids,
1787                     packidx, id_str_prefix);
1788                 if (err) {
1789                         got_packidx_close(packidx);
1790                         break;
1791                 }
1792                 err = got_packidx_close(packidx);
1793                 if (err)
1794                         break;
1795
1796                 STAILQ_FOREACH(qid, &matched_ids, entry) {
1797                         if (obj_type != GOT_OBJ_TYPE_ANY) {
1798                                 int matched_type;
1799                                 err = got_object_get_type(&matched_type, repo,
1800                                     &qid->id);
1801                                 if (err)
1802                                         goto done;
1803                                 if (matched_type != obj_type)
1804                                         continue;
1805                         }
1806                         if (*unique_id == NULL) {
1807                                 *unique_id = got_object_id_dup(&qid->id);
1808                                 if (*unique_id == NULL) {
1809                                         err = got_error_from_errno("malloc");
1810                                         goto done;
1811                                 }
1812                         } else {
1813                                 if (got_object_id_cmp(*unique_id,
1814                                     &qid->id) == 0)
1815                                         continue; /* packed multiple times */
1816                                 err = got_error(GOT_ERR_AMBIGUOUS_ID);
1817                                 goto done;
1818                         }
1819                 }
1820         }
1821 done:
1822         got_object_id_queue_free(&matched_ids);
1823         if (err) {
1824                 free(*unique_id);
1825                 *unique_id = NULL;
1826         }
1827         return err;
1828 }
1829
1830 static const struct got_error *
1831 match_loose_object(struct got_object_id **unique_id, const char *path_objects,
1832     const char *object_dir, const char *id_str_prefix, int obj_type,
1833     struct got_repository *repo)
1834 {
1835         const struct got_error *err = NULL;
1836         char *path, *id_str = NULL;
1837         DIR *dir = NULL;
1838         struct dirent *dent;
1839         struct got_object_id id;
1840
1841         if (asprintf(&path, "%s/%s", path_objects, object_dir) == -1) {
1842                 err = got_error_from_errno("asprintf");
1843                 goto done;
1844         }
1845
1846         dir = opendir(path);
1847         if (dir == NULL) {
1848                 if (errno == ENOENT) {
1849                         err = NULL;
1850                         goto done;
1851                 }
1852                 err = got_error_from_errno2("opendir", path);
1853                 goto done;
1854         }
1855         while ((dent = readdir(dir)) != NULL) {
1856                 int cmp;
1857
1858                 free(id_str);
1859                 id_str = NULL;
1860
1861                 if (strcmp(dent->d_name, ".") == 0 ||
1862                     strcmp(dent->d_name, "..") == 0)
1863                         continue;
1864
1865                 if (asprintf(&id_str, "%s%s", object_dir, dent->d_name) == -1) {
1866                         err = got_error_from_errno("asprintf");
1867                         goto done;
1868                 }
1869
1870                 if (!got_parse_object_id(&id, id_str, repo->algo))
1871                         continue;
1872
1873                 /*
1874                  * Directory entries do not necessarily appear in
1875                  * sorted order, so we must iterate over all of them.
1876                  */
1877                 cmp = strncmp(id_str, id_str_prefix, strlen(id_str_prefix));
1878                 if (cmp != 0)
1879                         continue;
1880
1881                 if (*unique_id == NULL) {
1882                         if (obj_type != GOT_OBJ_TYPE_ANY) {
1883                                 int matched_type;
1884                                 err = got_object_get_type(&matched_type, repo,
1885                                     &id);
1886                                 if (err)
1887                                         goto done;
1888                                 if (matched_type != obj_type)
1889                                         continue;
1890                         }
1891                         *unique_id = got_object_id_dup(&id);
1892                         if (*unique_id == NULL) {
1893                                 err = got_error_from_errno("got_object_id_dup");
1894                                 goto done;
1895                         }
1896                 } else {
1897                         if (got_object_id_cmp(*unique_id, &id) == 0)
1898                                 continue; /* both packed and loose */
1899                         err = got_error(GOT_ERR_AMBIGUOUS_ID);
1900                         goto done;
1901                 }
1902         }
1903 done:
1904         if (dir && closedir(dir) != 0 && err == NULL)
1905                 err = got_error_from_errno("closedir");
1906         if (err) {
1907                 free(*unique_id);
1908                 *unique_id = NULL;
1909         }
1910         free(id_str);
1911         free(path);
1912         return err;
1913 }
1914
1915 const struct got_error *
1916 got_repo_match_object_id_prefix(struct got_object_id **id,
1917     const char *id_str_prefix, int obj_type, struct got_repository *repo)
1918 {
1919         const struct got_error *err = NULL;
1920         char *path_objects = NULL, *object_dir = NULL;
1921         size_t len;
1922         int i;
1923
1924         *id = NULL;
1925
1926         path_objects = got_repo_get_path_objects(repo);
1927
1928         len = strlen(id_str_prefix);
1929         if (len > SHA1_DIGEST_STRING_LENGTH - 1) {
1930                 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1931                 goto done;
1932         }
1933
1934         for (i = 0; i < len; i++) {
1935                 if (isxdigit((unsigned char)id_str_prefix[i]))
1936                         continue;
1937                 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1938                 goto done;
1939         }
1940
1941         if (len >= 2) {
1942                 err = match_packed_object(id, repo, id_str_prefix, obj_type);
1943                 if (err)
1944                         goto done;
1945                 object_dir = strndup(id_str_prefix, 2);
1946                 if (object_dir == NULL) {
1947                         err = got_error_from_errno("strdup");
1948                         goto done;
1949                 }
1950                 err = match_loose_object(id, path_objects, object_dir,
1951                     id_str_prefix, obj_type, repo);
1952         } else if (len == 1) {
1953                 int i;
1954                 for (i = 0; i < 0xf; i++) {
1955                         if (asprintf(&object_dir, "%s%.1x", id_str_prefix, i)
1956                             == -1) {
1957                                 err = got_error_from_errno("asprintf");
1958                                 goto done;
1959                         }
1960                         err = match_packed_object(id, repo, object_dir,
1961                             obj_type);
1962                         if (err)
1963                                 goto done;
1964                         err = match_loose_object(id, path_objects, object_dir,
1965                             id_str_prefix, obj_type, repo);
1966                         if (err)
1967                                 goto done;
1968                 }
1969         } else {
1970                 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
1971                 goto done;
1972         }
1973 done:
1974         free(path_objects);
1975         free(object_dir);
1976         if (err) {
1977                 free(*id);
1978                 *id = NULL;
1979         } else if (*id == NULL) {
1980                 switch (obj_type) {
1981                 case GOT_OBJ_TYPE_BLOB:
1982                         err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1983                             GOT_OBJ_LABEL_BLOB, id_str_prefix);
1984                         break;
1985                 case GOT_OBJ_TYPE_TREE:
1986                         err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1987                             GOT_OBJ_LABEL_TREE, id_str_prefix);
1988                         break;
1989                 case GOT_OBJ_TYPE_COMMIT:
1990                         err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1991                             GOT_OBJ_LABEL_COMMIT, id_str_prefix);
1992                         break;
1993                 case GOT_OBJ_TYPE_TAG:
1994                         err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
1995                             GOT_OBJ_LABEL_TAG, id_str_prefix);
1996                         break;
1997                 default:
1998                         err = got_error_path(id_str_prefix, GOT_ERR_NO_OBJ);
1999                         break;
2000                 }
2001         }
2002
2003         return err;
2004 }
2005
2006 const struct got_error *
2007 got_repo_match_object_id(struct got_object_id **id, char **label,
2008     const char *id_str, int obj_type, struct got_reflist_head *refs,
2009     struct got_repository *repo)
2010 {
2011         const struct got_error *err;
2012         struct got_tag_object *tag;
2013         struct got_reference *ref = NULL;
2014
2015         *id = NULL;
2016         if (label)
2017                 *label = NULL;
2018
2019         if (refs) {
2020                 err = got_repo_object_match_tag(&tag, id_str, obj_type,
2021                     refs, repo);
2022                 if (err == NULL) {
2023                         *id = got_object_id_dup(
2024                             got_object_tag_get_object_id(tag));
2025                         if (*id == NULL)
2026                                 err = got_error_from_errno("got_object_id_dup");
2027                         else if (label && asprintf(label, "refs/tags/%s",
2028                             got_object_tag_get_name(tag)) == -1) {
2029                                 err = got_error_from_errno("asprintf");
2030                                 free(*id);
2031                                 *id = NULL;
2032                         }
2033                         got_object_tag_close(tag);
2034                         return err;
2035                 } else if (err->code != GOT_ERR_OBJ_TYPE &&
2036                     err->code != GOT_ERR_NO_OBJ)
2037                         return err;
2038         }
2039
2040         err = got_ref_open(&ref, repo, id_str, 0);
2041         if (err == NULL) {
2042                 err = got_ref_resolve(id, repo, ref);
2043                 if (err)
2044                         goto done;
2045                 if (label) {
2046                         *label = strdup(got_ref_get_name(ref));
2047                         if (*label == NULL) {
2048                                 err = got_error_from_errno("strdup");
2049                                 goto done;
2050                         }
2051                 }
2052         } else {
2053                 if (err->code != GOT_ERR_NOT_REF &&
2054                     err->code != GOT_ERR_BAD_REF_NAME)
2055                         goto done;
2056                 err = got_repo_match_object_id_prefix(id, id_str,
2057                     obj_type, repo);
2058                 if (err) {
2059                         if (err->code == GOT_ERR_BAD_OBJ_ID_STR)
2060                                 err = got_error_not_ref(id_str);
2061                         goto done;
2062                 }
2063                 if (label) {
2064                         err = got_object_id_str(label, *id);
2065                         if (*label == NULL) {
2066                                 err = got_error_from_errno("strdup");
2067                                 goto done;
2068                         }
2069                 }
2070         }
2071 done:
2072         if (ref)
2073                 got_ref_close(ref);
2074         return err;
2075 }
2076
2077 const struct got_error *
2078 got_repo_object_match_tag(struct got_tag_object **tag, const char *name,
2079     int obj_type, struct got_reflist_head *refs, struct got_repository *repo)
2080 {
2081         const struct got_error *err = NULL;
2082         struct got_reflist_entry *re;
2083         struct got_object_id *tag_id;
2084         int name_is_absolute = (strncmp(name, "refs/", 5) == 0);
2085
2086         *tag = NULL;
2087
2088         TAILQ_FOREACH(re, refs, entry) {
2089                 const char *refname;
2090                 refname = got_ref_get_name(re->ref);
2091                 if (got_ref_is_symbolic(re->ref))
2092                         continue;
2093                 if (strncmp(refname, "refs/tags/", 10) != 0)
2094                         continue;
2095                 if (!name_is_absolute)
2096                         refname += strlen("refs/tags/");
2097                 if (strcmp(refname, name) != 0)
2098                         continue;
2099                 err = got_ref_resolve(&tag_id, repo, re->ref);
2100                 if (err)
2101                         break;
2102                 err = got_object_open_as_tag(tag, repo, tag_id);
2103                 free(tag_id);
2104                 if (err)
2105                         break;
2106                 if (obj_type == GOT_OBJ_TYPE_ANY ||
2107                     got_object_tag_get_object_type(*tag) == obj_type)
2108                         break;
2109                 got_object_tag_close(*tag);
2110                 *tag = NULL;
2111         }
2112
2113         if (err == NULL && *tag == NULL)
2114                 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2115                     GOT_OBJ_LABEL_TAG, name);
2116         return err;
2117 }
2118
2119 static const struct got_error *
2120 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
2121     const char *name, mode_t mode, struct got_object_id *blob_id)
2122 {
2123         const struct got_error *err = NULL;
2124
2125          *new_te = NULL;
2126
2127         *new_te = calloc(1, sizeof(**new_te));
2128         if (*new_te == NULL)
2129                 return got_error_from_errno("calloc");
2130
2131         if (strlcpy((*new_te)->name, name, sizeof((*new_te)->name)) >=
2132             sizeof((*new_te)->name)) {
2133                 err = got_error(GOT_ERR_NO_SPACE);
2134                 goto done;
2135         }
2136
2137         if (S_ISLNK(mode)) {
2138                 (*new_te)->mode = S_IFLNK;
2139         } else {
2140                 (*new_te)->mode = S_IFREG;
2141                 (*new_te)->mode |= (mode & (S_IRWXU | S_IRWXG | S_IRWXO));
2142         }
2143         memcpy(&(*new_te)->id, blob_id, sizeof((*new_te)->id));
2144 done:
2145         if (err && *new_te) {
2146                 free(*new_te);
2147                 *new_te = NULL;
2148         }
2149         return err;
2150 }
2151
2152 static const struct got_error *
2153 import_file(struct got_tree_entry **new_te, struct dirent *de,
2154     const char *path, struct got_repository *repo)
2155 {
2156         const struct got_error *err;
2157         struct got_object_id *blob_id = NULL;
2158         char *filepath;
2159         struct stat sb;
2160
2161         if (asprintf(&filepath, "%s%s%s", path,
2162             path[0] == '\0' ? "" : "/", de->d_name) == -1)
2163                 return got_error_from_errno("asprintf");
2164
2165         if (lstat(filepath, &sb) != 0) {
2166                 err = got_error_from_errno2("lstat", path);
2167                 goto done;
2168         }
2169
2170         err = got_object_blob_create(&blob_id, filepath, repo);
2171         if (err)
2172                 goto done;
2173
2174         err = alloc_added_blob_tree_entry(new_te, de->d_name, sb.st_mode,
2175             blob_id);
2176 done:
2177         free(filepath);
2178         if (err)
2179                 free(blob_id);
2180         return err;
2181 }
2182
2183 static const struct got_error *
2184 insert_tree_entry(struct got_tree_entry *new_te,
2185     struct got_pathlist_head *paths)
2186 {
2187         const struct got_error *err = NULL;
2188         struct got_pathlist_entry *new_pe;
2189
2190         err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
2191         if (err)
2192                 return err;
2193         if (new_pe == NULL)
2194                 return got_error(GOT_ERR_TREE_DUP_ENTRY);
2195         return NULL;
2196 }
2197
2198 static const struct got_error *write_tree(struct got_object_id **,
2199     const char *, struct got_pathlist_head *, struct got_repository *,
2200     got_repo_import_cb progress_cb, void *progress_arg);
2201
2202 static const struct got_error *
2203 import_subdir(struct got_tree_entry **new_te, struct dirent *de,
2204     const char *path, struct got_pathlist_head *ignores,
2205     struct got_repository *repo,
2206     got_repo_import_cb progress_cb, void *progress_arg)
2207 {
2208         const struct got_error *err;
2209         struct got_object_id *id = NULL;
2210         char *subdirpath;
2211
2212         if (asprintf(&subdirpath, "%s%s%s", path,
2213             path[0] == '\0' ? "" : "/", de->d_name) == -1)
2214                 return got_error_from_errno("asprintf");
2215
2216         (*new_te) = calloc(1, sizeof(**new_te));
2217         if (*new_te == NULL)
2218                 return got_error_from_errno("calloc");
2219         (*new_te)->mode = S_IFDIR;
2220         if (strlcpy((*new_te)->name, de->d_name, sizeof((*new_te)->name)) >=
2221             sizeof((*new_te)->name)) {
2222                 err = got_error(GOT_ERR_NO_SPACE);
2223                 goto done;
2224         }
2225         err = write_tree(&id, subdirpath, ignores,  repo,
2226             progress_cb, progress_arg);
2227         if (err)
2228                 goto done;
2229         memcpy(&(*new_te)->id, id, sizeof((*new_te)->id));
2230
2231 done:
2232         free(id);
2233         free(subdirpath);
2234         if (err) {
2235                 free(*new_te);
2236                 *new_te = NULL;
2237         }
2238         return err;
2239 }
2240
2241 static const struct got_error *
2242 write_tree(struct got_object_id **new_tree_id, const char *path_dir,
2243     struct got_pathlist_head *ignores, struct got_repository *repo,
2244     got_repo_import_cb progress_cb, void *progress_arg)
2245 {
2246         const struct got_error *err = NULL;
2247         DIR *dir;
2248         struct dirent *de;
2249         int nentries;
2250         struct got_tree_entry *new_te = NULL;
2251         struct got_pathlist_head paths;
2252         struct got_pathlist_entry *pe;
2253
2254         *new_tree_id = NULL;
2255
2256         TAILQ_INIT(&paths);
2257
2258         dir = opendir(path_dir);
2259         if (dir == NULL) {
2260                 err = got_error_from_errno2("opendir", path_dir);
2261                 goto done;
2262         }
2263
2264         nentries = 0;
2265         while ((de = readdir(dir)) != NULL) {
2266                 int ignore = 0;
2267                 int type;
2268
2269                 if (strcmp(de->d_name, ".") == 0 ||
2270                     strcmp(de->d_name, "..") == 0)
2271                         continue;
2272
2273                 err = got_path_dirent_type(&type, path_dir, de);
2274                 if (err)
2275                         goto done;
2276
2277                 TAILQ_FOREACH(pe, ignores, entry) {
2278                         if (type == DT_DIR && pe->path_len > 0 &&
2279                             pe->path[pe->path_len - 1] == '/') {
2280                                 char stripped[PATH_MAX];
2281
2282                                 if (strlcpy(stripped, pe->path,
2283                                     sizeof(stripped)) >= sizeof(stripped)) {
2284                                         err = got_error(GOT_ERR_NO_SPACE);
2285                                         goto done;
2286                                 }
2287                                 got_path_strip_trailing_slashes(stripped);
2288                                 if (fnmatch(stripped, de->d_name, 0) == 0) {
2289                                         ignore = 1;
2290                                         break;
2291                                 }
2292                         } else if (fnmatch(pe->path, de->d_name, 0) == 0) {
2293                                 ignore = 1;
2294                                 break;
2295                         }
2296                 }
2297                 if (ignore)
2298                         continue;
2299
2300                 if (type == DT_DIR) {
2301                         err = import_subdir(&new_te, de, path_dir,
2302                             ignores, repo, progress_cb, progress_arg);
2303                         if (err) {
2304                                 if (err->code != GOT_ERR_NO_TREE_ENTRY)
2305                                         goto done;
2306                                 err = NULL;
2307                                 continue;
2308                         }
2309                 } else if (type == DT_REG || type == DT_LNK) {
2310                         err = import_file(&new_te, de, path_dir, repo);
2311                         if (err)
2312                                 goto done;
2313                 } else
2314                         continue;
2315
2316                 err = insert_tree_entry(new_te, &paths);
2317                 if (err)
2318                         goto done;
2319                 nentries++;
2320         }
2321
2322         if (TAILQ_EMPTY(&paths)) {
2323                 err = got_error_msg(GOT_ERR_NO_TREE_ENTRY,
2324                     "cannot create tree without any entries");
2325                 goto done;
2326         }
2327
2328         TAILQ_FOREACH(pe, &paths, entry) {
2329                 struct got_tree_entry *te = pe->data;
2330                 char *path;
2331                 if (!S_ISREG(te->mode) && !S_ISLNK(te->mode))
2332                         continue;
2333                 if (asprintf(&path, "%s/%s", path_dir, pe->path) == -1) {
2334                         err = got_error_from_errno("asprintf");
2335                         goto done;
2336                 }
2337                 err = (*progress_cb)(progress_arg, path);
2338                 free(path);
2339                 if (err)
2340                         goto done;
2341         }
2342
2343         err = got_object_tree_create(new_tree_id, &paths, nentries, repo);
2344 done:
2345         if (dir)
2346                 closedir(dir);
2347         got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2348         return err;
2349 }
2350
2351 const struct got_error *
2352 got_repo_import(struct got_object_id **new_commit_id, const char *path_dir,
2353     const char *logmsg, const char *author, struct got_pathlist_head *ignores,
2354     struct got_repository *repo, got_repo_import_cb progress_cb,
2355     void *progress_arg)
2356 {
2357         const struct got_error *err;
2358         struct got_object_id *new_tree_id;
2359
2360         err = write_tree(&new_tree_id, path_dir, ignores, repo,
2361             progress_cb, progress_arg);
2362         if (err)
2363                 return err;
2364
2365         err = got_object_commit_create(new_commit_id, new_tree_id, NULL, 0,
2366             author, time(NULL), author, time(NULL), logmsg, repo);
2367         free(new_tree_id);
2368         return err;
2369 }
2370
2371 const struct got_error *
2372 got_repo_get_loose_object_info(int *nobjects, off_t *ondisk_size,
2373     struct got_repository *repo)
2374 {
2375         const struct got_error *err = NULL;
2376         char *path_objects = NULL, *path = NULL;
2377         DIR *dir = NULL;
2378         struct got_object_id id;
2379         int i;
2380
2381         *nobjects = 0;
2382         *ondisk_size = 0;
2383
2384         path_objects = got_repo_get_path_objects(repo);
2385         if (path_objects == NULL)
2386                 return got_error_from_errno("got_repo_get_path_objects");
2387
2388         for (i = 0; i <= 0xff; i++) {
2389                 struct dirent *dent;
2390
2391                 if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
2392                         err = got_error_from_errno("asprintf");
2393                         break;
2394                 }
2395
2396                 dir = opendir(path);
2397                 if (dir == NULL) {
2398                         if (errno == ENOENT) {
2399                                 err = NULL;
2400                                 continue;
2401                         }
2402                         err = got_error_from_errno2("opendir", path);
2403                         break;
2404                 }
2405
2406                 while ((dent = readdir(dir)) != NULL) {
2407                         char *id_str;
2408                         int fd;
2409                         struct stat sb;
2410
2411                         if (strcmp(dent->d_name, ".") == 0 ||
2412                             strcmp(dent->d_name, "..") == 0)
2413                                 continue;
2414
2415                         if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
2416                                 err = got_error_from_errno("asprintf");
2417                                 goto done;
2418                         }
2419
2420                         if (!got_parse_object_id(&id, id_str, repo->algo)) {
2421                                 free(id_str);
2422                                 continue;
2423                         }
2424                         free(id_str);
2425
2426                         err = got_object_open_loose_fd(&fd, &id, repo);
2427                         if (err)
2428                                 goto done;
2429
2430                         if (fstat(fd, &sb) == -1) {
2431                                 err = got_error_from_errno("fstat");
2432                                 close(fd);
2433                                 goto done;
2434                         }
2435                         (*nobjects)++;
2436                         (*ondisk_size) += sb.st_size;
2437
2438                         if (close(fd) == -1) {
2439                                 err = got_error_from_errno("close");
2440                                 goto done;
2441                         }
2442                 }
2443
2444                 if (closedir(dir) != 0) {
2445                         err = got_error_from_errno("closedir");
2446                         goto done;
2447                 }
2448                 dir = NULL;
2449
2450                 free(path);
2451                 path = NULL;
2452         }
2453 done:
2454         if (dir && closedir(dir) != 0 && err == NULL)
2455                 err = got_error_from_errno("closedir");
2456
2457         if (err) {
2458                 *nobjects = 0;
2459                 *ondisk_size = 0;
2460         }
2461         free(path_objects);
2462         free(path);
2463         return err;
2464 }
2465
2466 const struct got_error *
2467 got_repo_get_packfile_info(int *npackfiles, int *nobjects,
2468     off_t *total_packsize, struct got_repository *repo)
2469 {
2470         const struct got_error *err = NULL;
2471         DIR *packdir = NULL;
2472         struct dirent *dent;
2473         struct got_packidx *packidx = NULL;
2474         char *path_packidx;
2475         char *path_packfile;
2476         int packdir_fd;
2477         struct stat sb;
2478
2479         *npackfiles = 0;
2480         *nobjects = 0;
2481         *total_packsize = 0;
2482
2483         packdir_fd = openat(got_repo_get_fd(repo),
2484             GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
2485         if (packdir_fd == -1) {
2486                 return got_error_from_errno_fmt("openat: %s/%s",
2487                     got_repo_get_path_git_dir(repo),
2488                     GOT_OBJECTS_PACK_DIR);
2489         }
2490
2491         packdir = fdopendir(packdir_fd);
2492         if (packdir == NULL) {
2493                 err = got_error_from_errno("fdopendir");
2494                 goto done;
2495         }
2496
2497         while ((dent = readdir(packdir)) != NULL) {
2498                 if (!got_repo_is_packidx_filename(dent->d_name,
2499                     strlen(dent->d_name)))
2500                         continue;
2501
2502                 if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
2503                     dent->d_name) == -1) {
2504                         err = got_error_from_errno("asprintf");
2505                         goto done;
2506                 }
2507
2508                 err = got_packidx_open(&packidx, got_repo_get_fd(repo),
2509                     path_packidx, 0);
2510                 free(path_packidx);
2511                 if (err)
2512                         goto done;
2513
2514                 if (fstat(packidx->fd, &sb) == -1)
2515                         goto done;
2516                 *total_packsize += sb.st_size;
2517
2518                 err = got_packidx_get_packfile_path(&path_packfile,
2519                     packidx->path_packidx);
2520                 if (err)
2521                         goto done;
2522
2523                 if (fstatat(got_repo_get_fd(repo), path_packfile, &sb,
2524                     0) == -1) {
2525                         free(path_packfile);
2526                         goto done;
2527                 }
2528                 free(path_packfile);
2529                 *total_packsize += sb.st_size;
2530
2531                 *nobjects += be32toh(packidx->hdr.fanout_table[0xff]);
2532
2533                 (*npackfiles)++;
2534
2535                 got_packidx_close(packidx);
2536                 packidx = NULL;
2537         }
2538 done:
2539         if (packidx)
2540                 got_packidx_close(packidx);
2541         if (packdir && closedir(packdir) != 0 && err == NULL)
2542                 err = got_error_from_errno("closedir");
2543         if (err) {
2544                 *npackfiles = 0;
2545                 *nobjects = 0;
2546                 *total_packsize = 0;
2547         }
2548         return err;
2549 }
2550
2551 RB_GENERATE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
2552     got_packidx_bloom_filter_cmp);