lib/repository_admin.c

   1 /*
   2  * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/types.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/stat.h>
  23 #include <sys/socket.h>
  24 #include <sys/wait.h>
  25
  26 #include <dirent.h>
  27 #include <errno.h>
  28 #include <fcntl.h>
  29 #include <stdint.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <limits.h>
  34 #include <unistd.h>
  35
  36 #include "got_error.h"
  37 #include "got_cancel.h"
  38 #include "got_object.h"
  39 #include "got_reference.h"
  40 #include "got_repository.h"
  41 #include "got_repository_admin.h"
  42 #include "got_opentemp.h"
  43 #include "got_path.h"
  44
  45 #include "got_lib_delta.h"
  46 #include "got_lib_hash.h"
  47 #include "got_lib_object.h"
  48 #include "got_lib_object_idset.h"
  49 #include "got_lib_object_cache.h"
  50 #include "got_lib_pack.h"
  51 #include "got_lib_privsep.h"
  52 #include "got_lib_repository.h"
  53 #include "got_lib_ratelimit.h"
  54 #include "got_lib_pack_create.h"
  55 #include "got_lib_lockfile.h"
  56
  57 #ifndef nitems
  58 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  59 #endif
  60
  61 static const struct got_error *
  62 get_reflist_object_ids(struct got_object_id ***ids, int *nobjects,
  63     unsigned int wanted_obj_type_mask, struct got_reflist_head *refs,
  64     struct got_repository *repo,
  65     got_cancel_cb cancel_cb, void *cancel_arg)
  66 {
  67         const struct got_error *err = NULL;
  68         const size_t alloc_chunksz = 256;
  69         size_t nalloc;
  70         struct got_reflist_entry *re;
  71         int i;
  72
  73         *ids = NULL;
  74         *nobjects = 0;
  75
  76         err = got_reflist_sort(refs,
  77             got_ref_cmp_by_commit_timestamp_descending, repo);
  78         if (err)
  79                 return err;
  80
  81         *ids = reallocarray(NULL, alloc_chunksz, sizeof(struct got_object_id *));
  82         if (*ids == NULL)
  83                 return got_error_from_errno("reallocarray");
  84         nalloc = alloc_chunksz;
  85
  86         TAILQ_FOREACH(re, refs, entry) {
  87                 struct got_object_id *id;
  88
  89                 if (cancel_cb) {
  90                         err = cancel_cb(cancel_arg);
  91                         if (err)
  92                                 goto done;
  93                 }
  94
  95                 err = got_ref_resolve(&id, repo, re->ref);
  96                 if (err)
  97                         goto done;
  98
  99                 if (wanted_obj_type_mask != GOT_OBJ_TYPE_ANY) {
 100                         int obj_type;
 101                         err = got_object_get_type(&obj_type, repo, id);
 102                         if (err)
 103                                 goto done;
 104                         if ((wanted_obj_type_mask & (1 << obj_type)) == 0) {
 105                                 free(id);
 106                                 id = NULL;
 107                                 continue;
 108                         }
 109                 }
 110
 111                 if (nalloc <= *nobjects) {
 112                         struct got_object_id **new;
 113                         new = recallocarray(*ids, nalloc,
 114                             nalloc + alloc_chunksz,
 115                             sizeof(struct got_object_id *));
 116                         if (new == NULL) {
 117                                 err = got_error_from_errno(
 118                                     "recallocarray");
 119                                 goto done;
 120                         }
 121                         *ids = new;
 122                         nalloc += alloc_chunksz;
 123                 }
 124                 (*ids)[*nobjects] = id;
 125                 if ((*ids)[*nobjects] == NULL) {
 126                         err = got_error_from_errno("got_object_id_dup");
 127                         goto done;
 128                 }
 129                 (*nobjects)++;
 130         }
 131 done:
 132         if (err) {
 133                 for (i = 0; i < *nobjects; i++)
 134                         free((*ids)[i]);
 135                 free(*ids);
 136                 *ids = NULL;
 137                 *nobjects = 0;
 138         }
 139         return err;
 140 }
 141
 142 const struct got_error *
 143 got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash,
 144     struct got_reflist_head *include_refs,
 145     struct got_reflist_head *exclude_refs, struct got_repository *repo,
 146     int loose_obj_only, int force_refdelta,
 147     got_pack_progress_cb progress_cb, void *progress_arg,
 148     got_cancel_cb cancel_cb, void *cancel_arg)
 149 {
 150         const struct got_error *err = NULL;
 151         struct got_object_id **ours = NULL, **theirs = NULL;
 152         int nours = 0, ntheirs = 0, packfd = -1, i;
 153         char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL;
 154         char *hash_str = NULL;
 155         FILE *delta_cache = NULL;
 156         struct got_ratelimit rl;
 157
 158         *packfile = NULL;
 159         *pack_hash = NULL;
 160
 161         got_ratelimit_init(&rl, 0, 500);
 162
 163         if (asprintf(&path, "%s/%s/packing.pack",
 164             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) {
 165                 err = got_error_from_errno("asprintf");
 166                 goto done;
 167         }
 168         err = got_opentemp_named_fd(&tmpfile_path, &packfd, path, "");
 169         if (err)
 170                 goto done;
 171
 172         if (fchmod(packfd, GOT_DEFAULT_PACK_MODE) == -1) {
 173                 err = got_error_from_errno2("fchmod", tmpfile_path);
 174                 goto done;
 175         }
 176
 177         delta_cache = got_opentemp();
 178         if (delta_cache == NULL) {
 179                 err = got_error_from_errno("got_opentemp");
 180                 goto done;
 181         }
 182
 183         err = get_reflist_object_ids(&ours, &nours,
 184             (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
 185             include_refs, repo, cancel_cb, cancel_arg);
 186         if (err)
 187                 goto done;
 188
 189         if (nours == 0) {
 190                 err = got_error(GOT_ERR_CANNOT_PACK);
 191                 goto done;
 192         }
 193
 194         if (!TAILQ_EMPTY(exclude_refs)) {
 195                 err = get_reflist_object_ids(&theirs, &ntheirs,
 196                     (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
 197                     exclude_refs, repo,
 198                     cancel_cb, cancel_arg);
 199                 if (err)
 200                         goto done;
 201         }
 202
 203         *pack_hash = calloc(1, sizeof(**pack_hash));
 204         if (*pack_hash == NULL) {
 205                 err = got_error_from_errno("calloc");
 206                 goto done;
 207         }
 208         err = got_pack_create(*pack_hash, packfd, delta_cache,
 209             theirs, ntheirs, ours, nours, repo, loose_obj_only,
 210             0, force_refdelta, progress_cb, progress_arg, &rl,
 211             cancel_cb, cancel_arg);
 212         if (err)
 213                 goto done;
 214
 215         err = got_object_id_str(&hash_str, *pack_hash);
 216         if (err)
 217                 goto done;
 218         if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
 219             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR,
 220             hash_str) == -1) {
 221                 err = got_error_from_errno("asprintf");
 222                 goto done;
 223         }
 224
 225         if (lseek(packfd, 0L, SEEK_SET) == -1) {
 226                 err = got_error_from_errno("lseek");
 227                 goto done;
 228         }
 229         if (rename(tmpfile_path, packfile_path) == -1) {
 230                 err = got_error_from_errno3("rename", tmpfile_path,
 231                     packfile_path);
 232                 goto done;
 233         }
 234         free(tmpfile_path);
 235         tmpfile_path = NULL;
 236
 237         *packfile = fdopen(packfd, "w");
 238         if (*packfile == NULL) {
 239                 err = got_error_from_errno2("fdopen", tmpfile_path);
 240                 goto done;
 241         }
 242         packfd = -1;
 243 done:
 244         for (i = 0; i < nours; i++)
 245                 free(ours[i]);
 246         free(ours);
 247         for (i = 0; i < ntheirs; i++)
 248                 free(theirs[i]);
 249         free(theirs);
 250         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 251                 err = got_error_from_errno2("close", packfile_path);
 252         if (delta_cache && fclose(delta_cache) == EOF && err == NULL)
 253                 err = got_error_from_errno("fclose");
 254         if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL)
 255                 err = got_error_from_errno2("unlink", tmpfile_path);
 256         free(tmpfile_path);
 257         free(packfile_path);
 258         free(hash_str);
 259         free(path);
 260         if (err) {
 261                 free(*pack_hash);
 262                 *pack_hash = NULL;
 263                 if (*packfile)
 264                         fclose(*packfile);
 265                 *packfile = NULL;
 266         }
 267         return err;
 268 }
 269
 270 const struct got_error *
 271 got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
 272     struct got_repository *repo,
 273     got_pack_index_progress_cb progress_cb, void *progress_arg,
 274     got_cancel_cb cancel_cb, void *cancel_arg)
 275 {
 276         size_t i;
 277         char *path;
 278         int imsg_idxfds[2];
 279         int npackfd = -1, idxfd = -1, nidxfd = -1;
 280         int tmpfds[3];
 281         int idxstatus, done = 0;
 282         const struct got_error *err;
 283         struct imsgbuf idxibuf;
 284         pid_t idxpid;
 285         char *tmpidxpath = NULL;
 286         char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL;
 287         const char *repo_path = got_repo_get_path_git_dir(repo);
 288         struct stat sb;
 289
 290         for (i = 0; i < nitems(tmpfds); i++)
 291                 tmpfds[i] = -1;
 292
 293         if (asprintf(&path, "%s/%s/indexing.idx",
 294             repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
 295                 err = got_error_from_errno("asprintf");
 296                 goto done;
 297         }
 298         err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path, "");
 299         free(path);
 300         if (err)
 301                 goto done;
 302         if (fchmod(idxfd, GOT_DEFAULT_PACK_MODE) == -1) {
 303                 err = got_error_from_errno2("fchmod", tmpidxpath);
 304                 goto done;
 305         }
 306
 307         nidxfd = dup(idxfd);
 308         if (nidxfd == -1) {
 309                 err = got_error_from_errno("dup");
 310                 goto done;
 311         }
 312
 313         for (i = 0; i < nitems(tmpfds); i++) {
 314                 tmpfds[i] = got_opentempfd();
 315                 if (tmpfds[i] == -1) {
 316                         err = got_error_from_errno("got_opentempfd");
 317                         goto done;
 318                 }
 319         }
 320
 321         err = got_object_id_str(&id_str, pack_hash);
 322         if (err)
 323                 goto done;
 324
 325         if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
 326             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 327                 err = got_error_from_errno("asprintf");
 328                 goto done;
 329         }
 330
 331         if (fstat(fileno(packfile), &sb) == -1) {
 332                 err = got_error_from_errno2("fstat", packfile_path);
 333                 goto done;
 334         }
 335
 336         if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
 337             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 338                 err = got_error_from_errno("asprintf");
 339                 goto done;
 340         }
 341
 342         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 343                 err = got_error_from_errno("socketpair");
 344                 goto done;
 345         }
 346         idxpid = fork();
 347         if (idxpid == -1) {
 348                 err= got_error_from_errno("fork");
 349                 goto done;
 350         } else if (idxpid == 0)
 351                 got_privsep_exec_child(imsg_idxfds,
 352                     GOT_PATH_PROG_INDEX_PACK, packfile_path);
 353         if (close(imsg_idxfds[1]) == -1) {
 354                 err = got_error_from_errno("close");
 355                 goto done;
 356         }
 357         if (imsgbuf_init(&idxibuf, imsg_idxfds[0]) == -1) {
 358                 err = got_error_from_errno("imsgbuf_init");
 359                 goto done;
 360         }
 361         imsgbuf_allow_fdpass(&idxibuf);
 362
 363         npackfd = dup(fileno(packfile));
 364         if (npackfd == -1) {
 365                 err = got_error_from_errno("dup");
 366                 goto done;
 367         }
 368         err = got_privsep_send_index_pack_req(&idxibuf, pack_hash, npackfd);
 369         if (err != NULL)
 370                 goto done;
 371         npackfd = -1;
 372         err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
 373         if (err != NULL)
 374                 goto done;
 375         nidxfd = -1;
 376         for (i = 0; i < nitems(tmpfds); i++) {
 377                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 378                 if (err != NULL)
 379                         goto done;
 380                 tmpfds[i] = -1;
 381         }
 382         done = 0;
 383         while (!done) {
 384                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 385
 386                 if (cancel_cb) {
 387                         err = cancel_cb(cancel_arg);
 388                         if (err)
 389                                 goto done;
 390                 }
 391
 392                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 393                     &nobj_indexed, &nobj_loose, &nobj_resolved,
 394                     &idxibuf);
 395                 if (err != NULL)
 396                         goto done;
 397                 if (nobj_indexed != 0) {
 398                         err = progress_cb(progress_arg, sb.st_size,
 399                             nobj_total, nobj_indexed, nobj_loose,
 400                             nobj_resolved);
 401                         if (err)
 402                                 break;
 403                 }
 404         }
 405         if (close(imsg_idxfds[0]) == -1) {
 406                 err = got_error_from_errno("close");
 407                 goto done;
 408         }
 409         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 410                 err = got_error_from_errno("waitpid");
 411                 goto done;
 412         }
 413
 414         if (rename(tmpidxpath, idxpath) == -1) {
 415                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 416                 goto done;
 417         }
 418         free(tmpidxpath);
 419         tmpidxpath = NULL;
 420
 421 done:
 422         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 423                 err = got_error_from_errno2("unlink", tmpidxpath);
 424         if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
 425                 err = got_error_from_errno("close");
 426         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 427                 err = got_error_from_errno("close");
 428         for (i = 0; i < nitems(tmpfds); i++) {
 429                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 430                         err = got_error_from_errno("close");
 431         }
 432         free(tmpidxpath);
 433         free(idxpath);
 434         free(packfile_path);
 435         return err;
 436 }
 437
 438 const struct got_error *
 439 got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash,
 440     struct got_repository *repo, const char *packfile_path)
 441 {
 442         const struct got_error *err = NULL;
 443         const char *packdir_path = NULL;
 444         char *packfile_name = NULL, *p, *dot;
 445         struct got_object_id id;
 446         int packfd = -1;
 447
 448         *packfile = NULL;
 449         *pack_hash = NULL;
 450
 451         packdir_path = got_repo_get_path_objects_pack(repo);
 452         if (packdir_path == NULL)
 453                 return got_error_from_errno("got_repo_get_path_objects_pack");
 454
 455         if (!got_path_is_child(packfile_path, packdir_path,
 456             strlen(packdir_path))) {
 457                 err = got_error_path(packfile_path, GOT_ERR_BAD_PATH);
 458                 goto done;
 459
 460         }
 461
 462         err = got_path_basename(&packfile_name, packfile_path);
 463         if (err)
 464                 goto done;
 465         p = packfile_name;
 466
 467         if (strncmp(p, "pack-", 5) != 0) {
 468                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 469                     "'%s' is not a valid pack file name",
 470                     packfile_name);
 471                 goto done;
 472         }
 473         p += 5;
 474         dot = strchr(p, '.');
 475         if (dot == NULL) {
 476                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 477                     "'%s' is not a valid pack file name",
 478                     packfile_name);
 479                 goto done;
 480         }
 481         if (strcmp(dot + 1, "pack") != 0) {
 482                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 483                     "'%s' is not a valid pack file name",
 484                     packfile_name);
 485                 goto done;
 486         }
 487         *dot = '\0';
 488         if (!got_parse_object_id(&id, p, repo->algo)) {
 489                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 490                     "'%s' is not a valid pack file name",
 491                     packfile_name);
 492                 goto done;
 493         }
 494
 495         *pack_hash = got_object_id_dup(&id);
 496         if (*pack_hash == NULL) {
 497                 err = got_error_from_errno("got_object_id_dup");
 498                 goto done;
 499         }
 500
 501         packfd = open(packfile_path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
 502         if (packfd == -1) {
 503                 err = got_error_from_errno2("open", packfile_path);
 504                 goto done;
 505         }
 506
 507         *packfile = fdopen(packfd, "r");
 508         if (*packfile == NULL) {
 509                 err = got_error_from_errno2("fdopen", packfile_path);
 510                 goto done;
 511         }
 512         packfd = -1;
 513 done:
 514         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 515                 err = got_error_from_errno2("close", packfile_path);
 516         free(packfile_name);
 517         if (err) {
 518                 free(*pack_hash);
 519                 *pack_hash = NULL;
 520         }
 521         return err;
 522 }
 523
 524 const struct got_error *
 525 got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash,
 526     struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg,
 527     got_cancel_cb cancel_cb, void *cancel_arg)
 528 {
 529         const struct got_error *err = NULL;
 530         char *id_str = NULL, *idxpath = NULL, *packpath = NULL;
 531         struct got_packidx *packidx = NULL;
 532         struct got_pack *pack = NULL;
 533         uint32_t nobj, i;
 534         size_t digest_len = got_hash_digest_length(repo->algo);
 535
 536         err = got_object_id_str(&id_str, pack_hash);
 537         if (err)
 538                 goto done;
 539
 540         if (asprintf(&packpath, "%s/pack-%s.pack",
 541             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 542                 err = got_error_from_errno("asprintf");
 543                 goto done;
 544         }
 545         if (asprintf(&idxpath, "%s/pack-%s.idx",
 546             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 547                 err = got_error_from_errno("asprintf");
 548                 goto done;
 549         }
 550
 551         err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1,
 552             repo->algo);
 553         if (err)
 554                 goto done;
 555
 556         err = got_repo_cache_pack(&pack, repo, packpath, packidx);
 557         if (err)
 558                 goto done;
 559
 560         nobj = be32toh(packidx->hdr.fanout_table[0xff]);
 561         for (i = 0; i < nobj; i++) {
 562                 uint8_t *oid;
 563                 struct got_object_id id, base_id;
 564                 off_t offset, base_offset = 0;
 565                 uint8_t type;
 566                 uint64_t size;
 567                 size_t tslen, len;
 568
 569                 if (cancel_cb) {
 570                         err = cancel_cb(cancel_arg);
 571                         if (err)
 572                                 break;
 573                 }
 574                 oid = packidx->hdr.sorted_ids + i * digest_len;
 575                 id.algo = repo->algo;
 576                 memcpy(id.hash, oid, digest_len);
 577
 578                 offset = got_packidx_get_object_offset(packidx, i);
 579                 if (offset == -1) {
 580                         err = got_error(GOT_ERR_BAD_PACKIDX);
 581                         goto done;
 582                 }
 583
 584                 err = got_pack_parse_object_type_and_size(&type, &size, &tslen,
 585                     pack, offset);
 586                 if (err)
 587                         goto done;
 588
 589                 switch (type) {
 590                 case GOT_OBJ_TYPE_OFFSET_DELTA:
 591                         err = got_pack_parse_offset_delta(&base_offset, &len,
 592                             pack, offset, tslen);
 593                         if (err)
 594                                 goto done;
 595                         break;
 596                 case GOT_OBJ_TYPE_REF_DELTA:
 597                         err = got_pack_parse_ref_delta(&base_id,
 598                             pack, offset, tslen);
 599                         if (err)
 600                                 goto done;
 601                         break;
 602                 }
 603                 err = (*list_cb)(list_arg, &id, type, offset, size,
 604                     base_offset, &base_id);
 605                 if (err)
 606                         goto done;
 607         }
 608
 609 done:
 610         free(id_str);
 611         free(idxpath);
 612         free(packpath);
 613         if (packidx)
 614                 got_packidx_close(packidx);
 615         return err;
 616 }
 617
 618 static const struct got_error *
 619 repo_cleanup_lock(struct got_repository *repo, struct got_lockfile **lk)
 620 {
 621         const struct got_error *err;
 622         char myname[_POSIX_HOST_NAME_MAX + 1];
 623
 624         if (gethostname(myname, sizeof(myname)) == -1)
 625                 return got_error_from_errno("gethostname");
 626
 627         err = got_lockfile_lock(lk, "gc.pid", got_repo_get_fd(repo));
 628         if (err)
 629                 return err;
 630
 631         /*
 632          * Git uses these info to provide some verbiage when finds a
 633          * lock during `git gc --force' so don't try too hard to avoid
 634          * short writes and don't care if a race happens between the
 635          * lockfile creation and the write itself.
 636          */
 637         if (dprintf((*lk)->fd, "%d %s", getpid(), myname) < 0)
 638                 return got_error_from_errno("dprintf");
 639
 640         return NULL;
 641 }
 642
 643 static const struct got_error *
 644 report_cleanup_progress(got_cleanup_progress_cb progress_cb,
 645     void *progress_arg, struct got_ratelimit *rl,
 646     int ncommits, int nloose, int npurged, int nredundant)
 647 {
 648         const struct got_error *err;
 649         int elapsed;
 650
 651         if (progress_cb == NULL)
 652                 return NULL;
 653
 654         err = got_ratelimit_check(&elapsed, rl);
 655         if (err || !elapsed)
 656                 return err;
 657
 658         return progress_cb(progress_arg, ncommits, nloose, npurged,
 659             nredundant);
 660 }
 661
 662 static const struct got_error *
 663 get_loose_object_ids(struct got_object_idset **loose_ids,
 664     off_t *ondisk_size, int ncommits,
 665     got_cleanup_progress_cb progress_cb, void *progress_arg,
 666     struct got_ratelimit *rl, struct got_repository *repo)
 667 {
 668         const struct got_error *err = NULL;
 669         char *path_objects = NULL, *path = NULL;
 670         DIR *dir = NULL;
 671         struct got_object *obj = NULL;
 672         struct got_object_id id;
 673         int i, fd = -1;
 674         struct stat sb;
 675
 676         *ondisk_size = 0;
 677         *loose_ids = got_object_idset_alloc();
 678         if (*loose_ids == NULL)
 679                 return got_error_from_errno("got_object_idset_alloc");
 680
 681         path_objects = got_repo_get_path_objects(repo);
 682         if (path_objects == NULL) {
 683                 err = got_error_from_errno("got_repo_get_path_objects");
 684                 goto done;
 685         }
 686
 687         for (i = 0; i <= 0xff; i++) {
 688                 struct dirent *dent;
 689
 690                 if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
 691                         err = got_error_from_errno("asprintf");
 692                         break;
 693                 }
 694
 695                 dir = opendir(path);
 696                 if (dir == NULL) {
 697                         if (errno == ENOENT) {
 698                                 err = NULL;
 699                                 continue;
 700                         }
 701                         err = got_error_from_errno2("opendir", path);
 702                         break;
 703                 }
 704
 705                 while ((dent = readdir(dir)) != NULL) {
 706                         char *id_str;
 707
 708                         if (strcmp(dent->d_name, ".") == 0 ||
 709                             strcmp(dent->d_name, "..") == 0)
 710                                 continue;
 711
 712                         if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
 713                                 err = got_error_from_errno("asprintf");
 714                                 goto done;
 715                         }
 716
 717                         if (!got_parse_object_id(&id, id_str, repo->algo)) {
 718                                 free(id_str);
 719                                 continue;
 720                         }
 721                         free(id_str);
 722
 723                         err = got_object_open_loose_fd(&fd, &id, repo);
 724                         if (err)
 725                                 goto done;
 726                         if (fstat(fd, &sb) == -1) {
 727                                 err = got_error_from_errno("fstat");
 728                                 goto done;
 729                         }
 730                         err = got_object_read_header_privsep(&obj, &id, repo,
 731                             fd);
 732                         if (err)
 733                                 goto done;
 734                         fd = -1; /* already closed */
 735
 736                         switch (obj->type) {
 737                         case GOT_OBJ_TYPE_COMMIT:
 738                         case GOT_OBJ_TYPE_TREE:
 739                         case GOT_OBJ_TYPE_BLOB:
 740                         case GOT_OBJ_TYPE_TAG:
 741                                 break;
 742                         default:
 743                                 err = got_error_fmt(GOT_ERR_OBJ_TYPE,
 744                                     "%d", obj->type);
 745                                 goto done;
 746                         }
 747                         got_object_close(obj);
 748                         obj = NULL;
 749                         (*ondisk_size) += sb.st_size;
 750                         err = got_object_idset_add(*loose_ids, &id, NULL);
 751                         if (err)
 752                                 goto done;
 753                         err = report_cleanup_progress(progress_cb,
 754                             progress_arg, rl, ncommits,
 755                             got_object_idset_num_elements(*loose_ids),
 756                             -1, -1);
 757                         if (err)
 758                                 goto done;
 759                 }
 760
 761                 if (closedir(dir) != 0) {
 762                         err = got_error_from_errno("closedir");
 763                         goto done;
 764                 }
 765                 dir = NULL;
 766
 767                 free(path);
 768                 path = NULL;
 769         }
 770 done:
 771         if (dir && closedir(dir) != 0 && err == NULL)
 772                 err = got_error_from_errno("closedir");
 773         if (fd != -1 && close(fd) == -1 && err == NULL)
 774                 err = got_error_from_errno("close");
 775         if (err) {
 776                 got_object_idset_free(*loose_ids);
 777                 *loose_ids = NULL;
 778         }
 779         if (obj)
 780                 got_object_close(obj);
 781         free(path_objects);
 782         free(path);
 783         return err;
 784 }
 785
 786 static const struct got_error *
 787 load_tree_entries(struct got_object_id_queue *ids,
 788     struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
 789     const char *dpath, struct got_repository *repo,
 790     got_cancel_cb cancel_cb, void *cancel_arg)
 791 {
 792         const struct got_error *err;
 793         struct got_tree_object *tree;
 794         char *p = NULL;
 795         int i;
 796
 797         err = got_object_open_as_tree(&tree, repo, tree_id);
 798         if (err)
 799                 return err;
 800
 801         for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
 802                 struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
 803                 struct got_object_id *id = got_tree_entry_get_id(e);
 804                 mode_t mode = got_tree_entry_get_mode(e);
 805
 806                 if (cancel_cb) {
 807                         err = (*cancel_cb)(cancel_arg);
 808                         if (err)
 809                                 break;
 810                 }
 811
 812                 if (got_object_tree_entry_is_symlink(e) ||
 813                     got_object_tree_entry_is_submodule(e) ||
 814                     got_object_idset_contains(traversed_ids, id))
 815                         continue;
 816
 817                 if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "",
 818                     got_tree_entry_get_name(e)) == -1) {
 819                         err = got_error_from_errno("asprintf");
 820                         break;
 821                 }
 822
 823                 if (S_ISDIR(mode)) {
 824                         struct got_object_qid *qid;
 825                         err = got_object_qid_alloc(&qid, id);
 826                         if (err)
 827                                 break;
 828                         STAILQ_INSERT_TAIL(ids, qid, entry);
 829                 } else if (S_ISREG(mode)) {
 830                         /* This blob is referenced. */
 831                         err = got_object_idset_add(traversed_ids, id, NULL);
 832                         if (err)
 833                                 break;
 834                 }
 835                 free(p);
 836                 p = NULL;
 837         }
 838
 839         got_object_tree_close(tree);
 840         free(p);
 841         return err;
 842 }
 843
 844 static const struct got_error *
 845 load_tree(struct got_object_idset *traversed_ids,
 846     struct got_object_id *tree_id,
 847     const char *dpath, struct got_repository *repo,
 848     got_cancel_cb cancel_cb, void *cancel_arg)
 849 {
 850         const struct got_error *err = NULL;
 851         struct got_object_id_queue tree_ids;
 852         struct got_object_qid *qid;
 853
 854         err = got_object_qid_alloc(&qid, tree_id);
 855         if (err)
 856                 return err;
 857
 858         STAILQ_INIT(&tree_ids);
 859         STAILQ_INSERT_TAIL(&tree_ids, qid, entry);
 860
 861         while (!STAILQ_EMPTY(&tree_ids)) {
 862                 if (cancel_cb) {
 863                         err = (*cancel_cb)(cancel_arg);
 864                         if (err)
 865                                 break;
 866                 }
 867
 868                 qid = STAILQ_FIRST(&tree_ids);
 869                 STAILQ_REMOVE_HEAD(&tree_ids, entry);
 870
 871                 if (got_object_idset_contains(traversed_ids, &qid->id)) {
 872                         got_object_qid_free(qid);
 873                         continue;
 874                 }
 875
 876                 err = got_object_idset_add(traversed_ids, &qid->id, NULL);
 877                 if (err) {
 878                         got_object_qid_free(qid);
 879                         break;
 880                 }
 881
 882                 err = load_tree_entries(&tree_ids, traversed_ids,
 883                     &qid->id, dpath, repo, cancel_cb, cancel_arg);
 884                 got_object_qid_free(qid);
 885                 if (err)
 886                         break;
 887         }
 888
 889         got_object_id_queue_free(&tree_ids);
 890         return err;
 891 }
 892
 893 static const struct got_error *
 894 load_commit_or_tag(int *ncommits, struct got_object_idset *traversed_ids,
 895     struct got_object_id *id, struct got_repository *repo,
 896     got_cleanup_progress_cb progress_cb, void *progress_arg,
 897     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 898 {
 899         const struct got_error *err;
 900         struct got_commit_object *commit = NULL;
 901         struct got_tag_object *tag = NULL;
 902         struct got_object_id *tree_id = NULL;
 903         struct got_object_id_queue ids;
 904         struct got_object_qid *qid;
 905         int obj_type;
 906
 907         err = got_object_qid_alloc(&qid, id);
 908         if (err)
 909                 return err;
 910
 911         STAILQ_INIT(&ids);
 912         STAILQ_INSERT_TAIL(&ids, qid, entry);
 913
 914         while (!STAILQ_EMPTY(&ids)) {
 915                 if (cancel_cb) {
 916                         err = (*cancel_cb)(cancel_arg);
 917                         if (err)
 918                                 break;
 919                 }
 920
 921                 qid = STAILQ_FIRST(&ids);
 922                 STAILQ_REMOVE_HEAD(&ids, entry);
 923
 924                 if (got_object_idset_contains(traversed_ids, &qid->id)) {
 925                         got_object_qid_free(qid);
 926                         qid = NULL;
 927                         continue;
 928                 }
 929
 930                 err = got_object_idset_add(traversed_ids, &qid->id, NULL);
 931                 if (err)
 932                         break;
 933
 934                 err = got_object_get_type(&obj_type, repo, &qid->id);
 935                 if (err)
 936                         break;
 937                 switch (obj_type) {
 938                 case GOT_OBJ_TYPE_COMMIT:
 939                         err = got_object_open_as_commit(&commit, repo,
 940                             &qid->id);
 941                         if (err)
 942                                 goto done;
 943                         break;
 944                 case GOT_OBJ_TYPE_TAG:
 945                         err = got_object_open_as_tag(&tag, repo, &qid->id);
 946                         if (err)
 947                                 goto done;
 948                         break;
 949                 default:
 950                         /* should not happen */
 951                         err = got_error(GOT_ERR_OBJ_TYPE);
 952                         goto done;
 953                 }
 954
 955                 /* Find a tree object to scan. */
 956                 if (commit) {
 957                         tree_id = got_object_commit_get_tree_id(commit);
 958                 } else if (tag) {
 959                         obj_type = got_object_tag_get_object_type(tag);
 960                         switch (obj_type) {
 961                         case GOT_OBJ_TYPE_COMMIT:
 962                                 err = got_object_open_as_commit(&commit, repo,
 963                                     got_object_tag_get_object_id(tag));
 964                                 if (err)
 965                                         goto done;
 966                                 tree_id = got_object_commit_get_tree_id(commit);
 967                                 break;
 968                         case GOT_OBJ_TYPE_TREE:
 969                                 tree_id = got_object_tag_get_object_id(tag);
 970                                 break;
 971                         default:
 972                                 /*
 973                                  * Tag points at something other than a
 974                                  * commit or tree. Leave this weird tag object
 975                                  * and the object it points to.
 976                                  */
 977                                 if (got_object_idset_contains(traversed_ids,
 978                                     got_object_tag_get_object_id(tag)))
 979                                         break;
 980                                 err = got_object_idset_add(traversed_ids,
 981                                     got_object_tag_get_object_id(tag), NULL);
 982                                 if (err)
 983                                         goto done;
 984                                 break;
 985                         }
 986                 }
 987
 988                 if (tree_id) {
 989                         err = load_tree(traversed_ids, tree_id, "",
 990                             repo, cancel_cb, cancel_arg);
 991                         if (err)
 992                                 break;
 993                 }
 994
 995                 if (commit || tag)
 996                         (*ncommits)++; /* scanned tags are counted as commits */
 997
 998                 err = report_cleanup_progress(progress_cb, progress_arg, rl,
 999                     *ncommits, -1, -1, -1);
1000                 if (err)
1001                         break;
1002
1003                 if (commit) {
1004                         /* Find parent commits to scan. */
1005                         const struct got_object_id_queue *parent_ids;
1006                         parent_ids = got_object_commit_get_parent_ids(commit);
1007                         err = got_object_id_queue_copy(parent_ids, &ids);
1008                         if (err)
1009                                 break;
1010                         got_object_commit_close(commit);
1011                         commit = NULL;
1012                 }
1013                 if (tag) {
1014                         got_object_tag_close(tag);
1015                         tag = NULL;
1016                 }
1017                 got_object_qid_free(qid);
1018                 qid = NULL;
1019         }
1020 done:
1021         if (qid)
1022                 got_object_qid_free(qid);
1023         if (commit)
1024                 got_object_commit_close(commit);
1025         if (tag)
1026                 got_object_tag_close(tag);
1027         got_object_id_queue_free(&ids);
1028         return err;
1029 }
1030
1031 static const struct got_error *
1032 is_object_packed(int *packed, struct got_repository *repo,
1033     struct got_object_id *id)
1034 {
1035         const struct got_error *err;
1036         struct got_object *obj;
1037
1038         *packed = 0;
1039
1040         err = got_object_open_packed(&obj, id, repo);
1041         if (err) {
1042                 if (err->code == GOT_ERR_NO_OBJ)
1043                         err = NULL;
1044                 return err;
1045         }
1046         got_object_close(obj);
1047         *packed = 1;
1048         return NULL;
1049 }
1050
1051 struct purge_loose_object_arg {
1052         struct got_repository *repo;
1053         got_cleanup_progress_cb progress_cb;
1054         void *progress_arg;
1055         struct got_ratelimit *rl;
1056         struct got_object_idset *traversed_ids;
1057         int nloose;
1058         int ncommits;
1059         int npacked;
1060         int npurged;
1061         off_t size_purged;
1062         int dry_run;
1063         time_t max_mtime;
1064         int ignore_mtime;
1065 };
1066
1067 static const struct got_error *
1068 purge_loose_object(struct got_object_id *id, void *data, void *arg)
1069 {
1070         struct purge_loose_object_arg *a = arg;
1071         const struct got_error *err, *unlock_err = NULL;
1072         char *path = NULL;
1073         int packed, fd = -1;
1074         struct stat sb;
1075         struct got_lockfile *lf = NULL;
1076
1077         err = is_object_packed(&packed, a->repo, id);
1078         if (err)
1079                 return err;
1080
1081         if (!packed && got_object_idset_contains(a->traversed_ids, id))
1082                 return NULL;
1083
1084         if (packed)
1085                 a->npacked++;
1086
1087         err = got_object_get_path(&path, id, a->repo);
1088         if (err)
1089                 return err;
1090
1091         err = got_object_open_loose_fd(&fd, id, a->repo);
1092         if (err)
1093                 goto done;
1094
1095         if (fstat(fd, &sb) == -1) {
1096                 err = got_error_from_errno("fstat");
1097                 goto done;
1098         }
1099
1100         /*
1101          * Do not delete objects which are younger than our maximum
1102          * modification time threshold. This prevents a race where
1103          * new objects which are being added to the repository
1104          * concurrently would be deleted.
1105          */
1106         if (a->ignore_mtime || sb.st_mtime <= a->max_mtime) {
1107                 if (!a->dry_run) {
1108                         err = got_lockfile_lock(&lf, path, -1);
1109                         if (err)
1110                                 goto done;
1111                         if (unlink(path) == -1) {
1112                                 err = got_error_from_errno2("unlink", path);
1113                                 goto done;
1114                         }
1115                 }
1116
1117                 a->npurged++;
1118                 a->size_purged += sb.st_size;
1119                 err = report_cleanup_progress(a->progress_cb, a->progress_arg,
1120                     a->rl, a->ncommits, a->nloose, a->npurged, -1);
1121                 if (err)
1122                         goto done;
1123         }
1124 done:
1125         if (fd != -1 && close(fd) == -1 && err == NULL)
1126                 err = got_error_from_errno("close");
1127         free(path);
1128         if (lf)
1129                 unlock_err = got_lockfile_unlock(lf, -1);
1130         return err ? err : unlock_err;
1131 }
1132
1133 static const struct got_error *
1134 repo_purge_unreferenced_loose_objects(struct got_repository *repo,
1135     struct got_object_idset *traversed_ids,
1136     off_t *size_before, off_t *size_after, int ncommits, int *nloose,
1137     int *npacked, int *npurged, int dry_run, int ignore_mtime,
1138     time_t max_mtime, struct got_ratelimit *rl,
1139     got_cleanup_progress_cb progress_cb, void *progress_arg,
1140     got_cancel_cb cancel_cb, void *cancel_arg)
1141 {
1142         const struct got_error *err;
1143         struct got_object_idset *loose_ids;
1144         struct purge_loose_object_arg arg;
1145
1146         err = get_loose_object_ids(&loose_ids, size_before, ncommits,
1147             progress_cb, progress_arg, rl, repo);
1148         if (err)
1149                 return err;
1150         *nloose = got_object_idset_num_elements(loose_ids);
1151         if (*nloose == 0) {
1152                 got_object_idset_free(loose_ids);
1153                 if (progress_cb) {
1154                         err = progress_cb(progress_arg, 0, 0, 0, -1);
1155                         if (err)
1156                                 return err;
1157                 }
1158                 return NULL;
1159         }
1160
1161         memset(&arg, 0, sizeof(arg));
1162         arg.repo = repo;
1163         arg.progress_arg = progress_arg;
1164         arg.progress_cb = progress_cb;
1165         arg.rl = rl;
1166         arg.traversed_ids = traversed_ids;
1167         arg.nloose = *nloose;
1168         arg.npacked = 0;
1169         arg.npurged = 0;
1170         arg.size_purged = 0;
1171         arg.dry_run = dry_run;
1172         arg.max_mtime = max_mtime;
1173         arg.ignore_mtime = ignore_mtime;
1174         err = got_object_idset_for_each(loose_ids, purge_loose_object, &arg);
1175         if (err)
1176                 goto done;
1177
1178         *size_after = *size_before - arg.size_purged;
1179         *npacked = arg.npacked;
1180         *npurged = arg.npurged;
1181
1182         /* Produce a final progress report. */
1183         if (progress_cb) {
1184                 err = progress_cb(progress_arg, ncommits, *nloose,
1185                     arg.npurged, -1);
1186                 if (err)
1187                         goto done;
1188         }
1189 done:
1190         got_object_idset_free(loose_ids);
1191         return err;
1192 }
1193
1194 static const struct got_error *
1195 purge_redundant_pack(struct got_repository *repo, const char *packidx_path,
1196     int dry_run, int ignore_mtime, time_t max_mtime,
1197     int *remove, off_t *size_before, off_t *size_after)
1198 {
1199         static const char *ext[] = {".idx", ".pack", ".rev", ".bitmap",
1200             ".promisor", ".mtimes"};
1201         struct stat sb;
1202         char *dot, path[PATH_MAX];
1203         size_t i;
1204
1205         if (strlcpy(path, packidx_path, sizeof(path)) >= sizeof(path))
1206                 return got_error(GOT_ERR_NO_SPACE);
1207
1208         /*
1209          * Do not delete pack files which are younger than our maximum
1210          * modification time threshold.  This prevents a race where a
1211          * new pack file which is being added to the repository
1212          * concurrently would be deleted.
1213          */
1214         if (fstatat(got_repo_get_fd(repo), path, &sb, 0) == -1) {
1215                 if (errno == ENOENT)
1216                         return NULL;
1217                 return got_error_from_errno2("fstatat", path);
1218         }
1219         if (!ignore_mtime && sb.st_mtime > max_mtime)
1220                 *remove = 0;
1221
1222         /*
1223          * For compatibility with Git, if a matching .keep file exist
1224          * don't delete the packfile.
1225          */
1226         dot = strrchr(path, '.');
1227         *dot = '\0';
1228         if (strlcat(path, ".keep", sizeof(path)) >= sizeof(path))
1229                 return got_error(GOT_ERR_NO_SPACE);
1230         if (faccessat(got_repo_get_fd(repo), path, F_OK, 0) == 0)
1231                 *remove = 0;
1232
1233         for (i = 0; i < nitems(ext); ++i) {
1234                 *dot = '\0';
1235
1236                 if (strlcat(path, ext[i], sizeof(path)) >=
1237                     sizeof(path))
1238                         return got_error(GOT_ERR_NO_SPACE);
1239
1240                 if (fstatat(got_repo_get_fd(repo), path, &sb, 0) ==
1241                     -1) {
1242                         if (errno == ENOENT)
1243                                 continue;
1244                         return got_error_from_errno2("fstatat", path);
1245                 }
1246
1247                 *size_before += sb.st_size;
1248                 if (!*remove) {
1249                         *size_after += sb.st_size;
1250                         continue;
1251                 }
1252
1253                 if (dry_run)
1254                         continue;
1255
1256                 if (unlinkat(got_repo_get_fd(repo), path, 0) == -1) {
1257                         if (errno == ENOENT)
1258                                 continue;
1259                         return got_error_from_errno2("unlinkat",
1260                             path);
1261                 }
1262         }
1263
1264         return NULL;
1265 }
1266
1267 static const struct got_error *
1268 pack_is_redundant(int *redundant, struct got_repository *repo,
1269     struct got_object_idset *traversed_ids,
1270     const char *packidx_path, struct got_object_idset *idset)
1271 {
1272         const struct got_error *err;
1273         struct got_packidx *packidx;
1274         uint8_t *pid;
1275         struct got_object_id id;
1276         size_t i, nobjects;
1277         size_t digest_len = got_hash_digest_length(repo->algo);
1278
1279         *redundant = 1;
1280
1281         err = got_repo_get_packidx(&packidx, packidx_path, repo);
1282         if (err)
1283                 return err;
1284
1285         nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1286         for (i = 0; i < nobjects; ++i) {
1287                 pid = packidx->hdr.sorted_ids + i * digest_len;
1288
1289                 memset(&id, 0, sizeof(id));
1290                 memcpy(&id.hash, pid, digest_len);
1291                 id.algo = repo->algo;
1292
1293                 if (got_object_idset_contains(idset, &id))
1294                         continue;
1295
1296                 if (!got_object_idset_contains(traversed_ids, &id))
1297                         continue;
1298
1299                 *redundant = 0;
1300                 err = got_object_idset_add(idset, &id, NULL);
1301                 if (err)
1302                         return err;
1303         }
1304
1305         return NULL;
1306 }
1307
1308 struct pack_info {
1309         const char      *path;
1310         size_t           nobjects;
1311 };
1312
1313 static int
1314 pack_info_cmp(const void *a, const void *b)
1315 {
1316         const struct pack_info  *pa, *pb;
1317
1318         pa = a;
1319         pb = b;
1320         if (pa->nobjects == pb->nobjects)
1321                 return strcmp(pa->path, pb->path);
1322         if (pa->nobjects > pb->nobjects)
1323                 return -1;
1324         return 1;
1325 }
1326
1327 static const struct got_error *
1328 repo_purge_redundant_packfiles(struct got_repository *repo,
1329     struct got_object_idset *traversed_ids,
1330     off_t *size_before, off_t *size_after, int dry_run, int ignore_mtime,
1331     time_t max_mtime, int nloose, int ncommits, int npurged,
1332     struct got_ratelimit *rl,
1333     got_cleanup_progress_cb progress_cb, void *progress_arg,
1334     got_cancel_cb cancel_cb, void *cancel_arg)
1335 {
1336         const struct got_error *err;
1337         struct pack_info *pinfo, *sorted = NULL;
1338         struct got_packidx *packidx;
1339         struct got_object_idset *idset = NULL;
1340         struct got_pathlist_entry *pe;
1341         size_t i, npacks;
1342         int remove, redundant_packs = 0;
1343
1344         npacks = 0;
1345         TAILQ_FOREACH(pe, &repo->packidx_paths, entry)
1346                 npacks++;
1347
1348         if (npacks == 0)
1349                 return NULL;
1350
1351         sorted = calloc(npacks, sizeof(*sorted));
1352         if (sorted == NULL)
1353                 return got_error_from_errno("calloc");
1354
1355         i = 0;
1356         TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1357                 err = got_repo_get_packidx(&packidx, pe->path, repo);
1358                 if (err)
1359                         goto done;
1360
1361                 pinfo = &sorted[i++];
1362                 pinfo->path = pe->path;
1363                 pinfo->nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1364         }
1365         qsort(sorted, npacks, sizeof(*sorted), pack_info_cmp);
1366
1367         idset = got_object_idset_alloc();
1368         if (idset == NULL) {
1369                 err = got_error_from_errno("got_object_idset_alloc");
1370                 goto done;
1371         }
1372
1373         for (i = 0; i < npacks; ++i) {
1374                 if (cancel_cb) {
1375                         err = (*cancel_cb)(cancel_arg);
1376                         if (err)
1377                                 break;
1378                 }
1379
1380                 err = pack_is_redundant(&remove, repo, traversed_ids,
1381                     sorted[i].path, idset);
1382                 if (err)
1383                         goto done;
1384                 err = purge_redundant_pack(repo, sorted[i].path, dry_run,
1385                     ignore_mtime, max_mtime, &remove, size_before, size_after);
1386                 if (err)
1387                         goto done;
1388                 if (!remove)
1389                         continue;
1390                 err = report_cleanup_progress(progress_cb, progress_arg,
1391                     rl, ncommits, nloose, npurged, ++redundant_packs);
1392                 if (err)
1393                         goto done;
1394         }
1395
1396         /* Produce a final progress report. */
1397         if (progress_cb) {
1398                 err = progress_cb(progress_arg, ncommits, nloose, npurged,
1399                     redundant_packs);
1400                 if (err)
1401                         goto done;
1402         }
1403  done:
1404         free(sorted);
1405         if (idset)
1406                 got_object_idset_free(idset);
1407         return err;
1408 }
1409
1410 const struct got_error *
1411 got_repo_cleanup(struct got_repository *repo,
1412     off_t *loose_before, off_t *loose_after,
1413     off_t *pack_before, off_t *pack_after,
1414     int *ncommits, int *nloose, int *npacked, int dry_run, int ignore_mtime,
1415     got_cleanup_progress_cb progress_cb, void *progress_arg,
1416     got_cancel_cb cancel_cb, void *cancel_arg)
1417 {
1418         const struct got_error *unlock_err, *err = NULL;
1419         struct got_lockfile *lk = NULL;
1420         struct got_ratelimit rl;
1421         struct got_reflist_head refs;
1422         struct got_object_idset *traversed_ids = NULL;
1423         struct got_reflist_entry *re;
1424         struct got_object_id **referenced_ids;
1425         int i, nreferenced;
1426         int npurged = 0;
1427         time_t max_mtime = 0;
1428
1429         TAILQ_INIT(&refs);
1430         got_ratelimit_init(&rl, 0, 500);
1431
1432         *loose_before = 0;
1433         *loose_after = 0;
1434         *pack_before = 0;
1435         *pack_after = 0;
1436         *ncommits = 0;
1437         *nloose = 0;
1438         *npacked = 0;
1439
1440         err = repo_cleanup_lock(repo, &lk);
1441         if (err)
1442                 return err;
1443
1444         traversed_ids = got_object_idset_alloc();
1445         if (traversed_ids == NULL) {
1446                 err = got_error_from_errno("got_object_idset_alloc");
1447                 goto done;
1448         }
1449
1450         err = got_ref_list(&refs, repo, "", got_ref_cmp_by_name, NULL);
1451         if (err)
1452                 goto done;
1453         if (!ignore_mtime) {
1454                 TAILQ_FOREACH(re, &refs, entry) {
1455                         time_t mtime = got_ref_get_mtime(re->ref);
1456                         if (mtime > max_mtime)
1457                                 max_mtime = mtime;
1458                 }
1459                 /*
1460                  * For safety, keep objects created within 10 minutes
1461                  * before the youngest reference was created.
1462                  */
1463                 if (max_mtime >= 600)
1464                         max_mtime -= 600;
1465         }
1466
1467         err = get_reflist_object_ids(&referenced_ids, &nreferenced,
1468             (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
1469             &refs, repo, cancel_cb, cancel_arg);
1470         if (err)
1471                 goto done;
1472
1473         for (i = 0; i < nreferenced; i++) {
1474                 struct got_object_id *id = referenced_ids[i];
1475                 err = load_commit_or_tag(ncommits, traversed_ids,
1476                     id, repo, progress_cb, progress_arg, &rl,
1477                     cancel_cb, cancel_arg);
1478                 if (err)
1479                         goto done;
1480         }
1481
1482         err = repo_purge_unreferenced_loose_objects(repo, traversed_ids,
1483             loose_before, loose_after, *ncommits, nloose, npacked, &npurged,
1484             dry_run, ignore_mtime, max_mtime, &rl, progress_cb, progress_arg,
1485             cancel_cb, cancel_arg);
1486         if (err)
1487                 goto done;
1488
1489         err = repo_purge_redundant_packfiles(repo, traversed_ids,
1490             pack_before, pack_after, dry_run, ignore_mtime, max_mtime,
1491             *nloose, *ncommits, npurged, &rl, progress_cb, progress_arg,
1492             cancel_cb, cancel_arg);
1493         if (err)
1494                 goto done;
1495
1496  done:
1497         if (lk) {
1498                 unlock_err = got_lockfile_unlock(lk, got_repo_get_fd(repo));
1499                 if (err == NULL)
1500                         err = unlock_err;
1501         }
1502         if (traversed_ids)
1503                 got_object_idset_free(traversed_ids);
1504         return err;
1505 }
1506
1507 const struct got_error *
1508 got_repo_remove_lonely_packidx(struct got_repository *repo, int dry_run,
1509     got_lonely_packidx_progress_cb progress_cb, void *progress_arg,
1510     got_cancel_cb cancel_cb, void *cancel_arg)
1511 {
1512         const struct got_error *err = NULL;
1513         DIR *packdir = NULL;
1514         struct dirent *dent;
1515         char *pack_relpath = NULL;
1516         int packdir_fd;
1517         struct stat sb;
1518
1519         packdir_fd = openat(got_repo_get_fd(repo),
1520             GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1521         if (packdir_fd == -1) {
1522                 if (errno == ENOENT)
1523                         return NULL;
1524                 return got_error_from_errno_fmt("openat: %s/%s",
1525                     got_repo_get_path_git_dir(repo),
1526                     GOT_OBJECTS_PACK_DIR);
1527         }
1528
1529         packdir = fdopendir(packdir_fd);
1530         if (packdir == NULL) {
1531                 err = got_error_from_errno("fdopendir");
1532                 close(packdir_fd);
1533                 goto done;
1534         }
1535
1536         while ((dent = readdir(packdir)) != NULL) {
1537                 if (cancel_cb) {
1538                         err = cancel_cb(cancel_arg);
1539                         if (err)
1540                                 goto done;
1541                 }
1542
1543                 if (!got_repo_is_packidx_filename(dent->d_name,
1544                     strlen(dent->d_name),
1545                     got_repo_get_object_format(repo)))
1546                         continue;
1547
1548                 err = got_packidx_get_packfile_path(&pack_relpath,
1549                     dent->d_name);
1550                 if (err)
1551                         goto done;
1552
1553                 if (fstatat(packdir_fd, pack_relpath, &sb, 0) != -1) {
1554                         free(pack_relpath);
1555                         pack_relpath = NULL;
1556                         continue;
1557                 }
1558                 if (errno != ENOENT) {
1559                         err = got_error_from_errno_fmt("fstatat: %s/%s/%s",
1560                             got_repo_get_path_git_dir(repo),
1561                             GOT_OBJECTS_PACK_DIR,
1562                             pack_relpath);
1563                         goto done;
1564                 }
1565
1566                 if (!dry_run) {
1567                         if (unlinkat(packdir_fd, dent->d_name, 0) == -1) {
1568                                 err = got_error_from_errno("unlinkat");
1569                                 goto done;
1570                         }
1571                 }
1572                 if (progress_cb) {
1573                         char *path;
1574                         if (asprintf(&path, "%s/%s/%s",
1575                             got_repo_get_path_git_dir(repo),
1576                             GOT_OBJECTS_PACK_DIR,
1577                             dent->d_name) == -1) {
1578                                 err = got_error_from_errno("asprintf");
1579                                 goto done;
1580                         }
1581                         err = progress_cb(progress_arg, path);
1582                         free(path);
1583                         if (err)
1584                                 goto done;
1585                 }
1586                 free(pack_relpath);
1587                 pack_relpath = NULL;
1588         }
1589 done:
1590         if (packdir && closedir(packdir) != 0 && err == NULL)
1591                 err = got_error_from_errno("closedir");
1592         free(pack_relpath);
1593         return err;
1594 }