lib/repository_admin.c

   1 /*
   2  * Copyright (c) 2020 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include "got_compat.h"
  18
  19 #include <sys/types.h>
  20 #include <sys/queue.h>
  21 #include <sys/uio.h>
  22 #include <sys/stat.h>
  23 #include <sys/socket.h>
  24 #include <sys/wait.h>
  25
  26 #include <dirent.h>
  27 #include <errno.h>
  28 #include <fcntl.h>
  29 #include <stdint.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <limits.h>
  34 #include <unistd.h>
  35
  36 #include "got_error.h"
  37 #include "got_cancel.h"
  38 #include "got_object.h"
  39 #include "got_reference.h"
  40 #include "got_repository.h"
  41 #include "got_repository_admin.h"
  42 #include "got_opentemp.h"
  43 #include "got_path.h"
  44
  45 #include "got_lib_delta.h"
  46 #include "got_lib_hash.h"
  47 #include "got_lib_object.h"
  48 #include "got_lib_object_idset.h"
  49 #include "got_lib_object_cache.h"
  50 #include "got_lib_pack.h"
  51 #include "got_lib_privsep.h"
  52 #include "got_lib_repository.h"
  53 #include "got_lib_ratelimit.h"
  54 #include "got_lib_pack_create.h"
  55 #include "got_lib_lockfile.h"
  56
  57 #ifndef nitems
  58 #define nitems(_a)      (sizeof((_a)) / sizeof((_a)[0]))
  59 #endif
  60
  61 static const struct got_error *
  62 get_reflist_object_ids(struct got_object_id ***ids, int *nobjects,
  63     unsigned int wanted_obj_type_mask, struct got_reflist_head *refs,
  64     struct got_repository *repo,
  65     got_cancel_cb cancel_cb, void *cancel_arg)
  66 {
  67         const struct got_error *err = NULL;
  68         const size_t alloc_chunksz = 256;
  69         size_t nalloc;
  70         struct got_reflist_entry *re;
  71         int i;
  72
  73         *ids = NULL;
  74         *nobjects = 0;
  75
  76         err = got_reflist_sort(refs,
  77             got_ref_cmp_by_commit_timestamp_descending, repo);
  78         if (err)
  79                 return err;
  80
  81         *ids = reallocarray(NULL, alloc_chunksz, sizeof(struct got_object_id *));
  82         if (*ids == NULL)
  83                 return got_error_from_errno("reallocarray");
  84         nalloc = alloc_chunksz;
  85
  86         TAILQ_FOREACH(re, refs, entry) {
  87                 struct got_object_id *id;
  88
  89                 if (cancel_cb) {
  90                         err = cancel_cb(cancel_arg);
  91                         if (err)
  92                                 goto done;
  93                 }
  94
  95                 err = got_ref_resolve(&id, repo, re->ref);
  96                 if (err)
  97                         goto done;
  98
  99                 if (wanted_obj_type_mask != GOT_OBJ_TYPE_ANY) {
 100                         int obj_type;
 101                         err = got_object_get_type(&obj_type, repo, id);
 102                         if (err)
 103                                 goto done;
 104                         if ((wanted_obj_type_mask & (1 << obj_type)) == 0) {
 105                                 free(id);
 106                                 id = NULL;
 107                                 continue;
 108                         }
 109                 }
 110
 111                 if (nalloc <= *nobjects) {
 112                         struct got_object_id **new;
 113                         new = recallocarray(*ids, nalloc,
 114                             nalloc + alloc_chunksz,
 115                             sizeof(struct got_object_id *));
 116                         if (new == NULL) {
 117                                 err = got_error_from_errno(
 118                                     "recallocarray");
 119                                 goto done;
 120                         }
 121                         *ids = new;
 122                         nalloc += alloc_chunksz;
 123                 }
 124                 (*ids)[*nobjects] = id;
 125                 if ((*ids)[*nobjects] == NULL) {
 126                         err = got_error_from_errno("got_object_id_dup");
 127                         goto done;
 128                 }
 129                 (*nobjects)++;
 130         }
 131 done:
 132         if (err) {
 133                 for (i = 0; i < *nobjects; i++)
 134                         free((*ids)[i]);
 135                 free(*ids);
 136                 *ids = NULL;
 137                 *nobjects = 0;
 138         }
 139         return err;
 140 }
 141
 142 const struct got_error *
 143 got_repo_pack_objects(FILE **packfile, struct got_object_id **pack_hash,
 144     struct got_reflist_head *include_refs,
 145     struct got_reflist_head *exclude_refs, struct got_repository *repo,
 146     int loose_obj_only, int force_refdelta,
 147     got_pack_progress_cb progress_cb, void *progress_arg,
 148     got_cancel_cb cancel_cb, void *cancel_arg)
 149 {
 150         const struct got_error *err = NULL;
 151         struct got_object_id **ours = NULL, **theirs = NULL;
 152         int nours = 0, ntheirs = 0, packfd = -1, i;
 153         char *tmpfile_path = NULL, *path = NULL, *packfile_path = NULL;
 154         char *hash_str = NULL;
 155         FILE *delta_cache = NULL;
 156         struct got_ratelimit rl;
 157
 158         *packfile = NULL;
 159         *pack_hash = NULL;
 160
 161         got_ratelimit_init(&rl, 0, 500);
 162
 163         if (asprintf(&path, "%s/%s/packing.pack",
 164             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR) == -1) {
 165                 err = got_error_from_errno("asprintf");
 166                 goto done;
 167         }
 168         err = got_opentemp_named_fd(&tmpfile_path, &packfd, path, "");
 169         if (err)
 170                 goto done;
 171
 172         if (fchmod(packfd, GOT_DEFAULT_PACK_MODE) == -1) {
 173                 err = got_error_from_errno2("fchmod", tmpfile_path);
 174                 goto done;
 175         }
 176
 177         delta_cache = got_opentemp();
 178         if (delta_cache == NULL) {
 179                 err = got_error_from_errno("got_opentemp");
 180                 goto done;
 181         }
 182
 183         err = get_reflist_object_ids(&ours, &nours,
 184             (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
 185             include_refs, repo, cancel_cb, cancel_arg);
 186         if (err)
 187                 goto done;
 188
 189         if (nours == 0) {
 190                 err = got_error(GOT_ERR_CANNOT_PACK);
 191                 goto done;
 192         }
 193
 194         if (!TAILQ_EMPTY(exclude_refs)) {
 195                 err = get_reflist_object_ids(&theirs, &ntheirs,
 196                     (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
 197                     exclude_refs, repo,
 198                     cancel_cb, cancel_arg);
 199                 if (err)
 200                         goto done;
 201         }
 202
 203         *pack_hash = calloc(1, sizeof(**pack_hash));
 204         if (*pack_hash == NULL) {
 205                 err = got_error_from_errno("calloc");
 206                 goto done;
 207         }
 208         err = got_pack_create(*pack_hash, packfd, delta_cache,
 209             theirs, ntheirs, ours, nours, repo, loose_obj_only,
 210             0, force_refdelta, progress_cb, progress_arg, &rl,
 211             cancel_cb, cancel_arg);
 212         if (err)
 213                 goto done;
 214
 215         err = got_object_id_str(&hash_str, *pack_hash);
 216         if (err)
 217                 goto done;
 218         if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
 219             got_repo_get_path_git_dir(repo), GOT_OBJECTS_PACK_DIR,
 220             hash_str) == -1) {
 221                 err = got_error_from_errno("asprintf");
 222                 goto done;
 223         }
 224
 225         if (lseek(packfd, 0L, SEEK_SET) == -1) {
 226                 err = got_error_from_errno("lseek");
 227                 goto done;
 228         }
 229         if (rename(tmpfile_path, packfile_path) == -1) {
 230                 err = got_error_from_errno3("rename", tmpfile_path,
 231                     packfile_path);
 232                 goto done;
 233         }
 234         free(tmpfile_path);
 235         tmpfile_path = NULL;
 236
 237         *packfile = fdopen(packfd, "w");
 238         if (*packfile == NULL) {
 239                 err = got_error_from_errno2("fdopen", tmpfile_path);
 240                 goto done;
 241         }
 242         packfd = -1;
 243 done:
 244         for (i = 0; i < nours; i++)
 245                 free(ours[i]);
 246         free(ours);
 247         for (i = 0; i < ntheirs; i++)
 248                 free(theirs[i]);
 249         free(theirs);
 250         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 251                 err = got_error_from_errno2("close", packfile_path);
 252         if (delta_cache && fclose(delta_cache) == EOF && err == NULL)
 253                 err = got_error_from_errno("fclose");
 254         if (tmpfile_path && unlink(tmpfile_path) == -1 && err == NULL)
 255                 err = got_error_from_errno2("unlink", tmpfile_path);
 256         free(tmpfile_path);
 257         free(packfile_path);
 258         free(hash_str);
 259         free(path);
 260         if (err) {
 261                 free(*pack_hash);
 262                 *pack_hash = NULL;
 263                 if (*packfile)
 264                         fclose(*packfile);
 265                 *packfile = NULL;
 266         }
 267         return err;
 268 }
 269
 270 const struct got_error *
 271 got_repo_index_pack(FILE *packfile, struct got_object_id *pack_hash,
 272     struct got_repository *repo,
 273     got_pack_index_progress_cb progress_cb, void *progress_arg,
 274     got_cancel_cb cancel_cb, void *cancel_arg)
 275 {
 276         size_t i;
 277         char *path;
 278         int imsg_idxfds[2];
 279         int npackfd = -1, idxfd = -1, nidxfd = -1;
 280         int tmpfds[3];
 281         int idxstatus, done = 0;
 282         const struct got_error *err;
 283         struct imsgbuf idxibuf;
 284         pid_t idxpid;
 285         char *tmpidxpath = NULL;
 286         char *packfile_path = NULL, *idxpath = NULL, *id_str = NULL;
 287         const char *repo_path = got_repo_get_path_git_dir(repo);
 288         struct stat sb;
 289
 290         for (i = 0; i < nitems(tmpfds); i++)
 291                 tmpfds[i] = -1;
 292
 293         if (asprintf(&path, "%s/%s/indexing.idx",
 294             repo_path, GOT_OBJECTS_PACK_DIR) == -1) {
 295                 err = got_error_from_errno("asprintf");
 296                 goto done;
 297         }
 298         err = got_opentemp_named_fd(&tmpidxpath, &idxfd, path, "");
 299         free(path);
 300         if (err)
 301                 goto done;
 302         if (fchmod(idxfd, GOT_DEFAULT_PACK_MODE) == -1) {
 303                 err = got_error_from_errno2("fchmod", tmpidxpath);
 304                 goto done;
 305         }
 306
 307         nidxfd = dup(idxfd);
 308         if (nidxfd == -1) {
 309                 err = got_error_from_errno("dup");
 310                 goto done;
 311         }
 312
 313         for (i = 0; i < nitems(tmpfds); i++) {
 314                 tmpfds[i] = got_opentempfd();
 315                 if (tmpfds[i] == -1) {
 316                         err = got_error_from_errno("got_opentempfd");
 317                         goto done;
 318                 }
 319         }
 320
 321         err = got_object_id_str(&id_str, pack_hash);
 322         if (err)
 323                 goto done;
 324
 325         if (asprintf(&packfile_path, "%s/%s/pack-%s.pack",
 326             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 327                 err = got_error_from_errno("asprintf");
 328                 goto done;
 329         }
 330
 331         if (fstat(fileno(packfile), &sb) == -1) {
 332                 err = got_error_from_errno2("fstat", packfile_path);
 333                 goto done;
 334         }
 335
 336         if (asprintf(&idxpath, "%s/%s/pack-%s.idx",
 337             repo_path, GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 338                 err = got_error_from_errno("asprintf");
 339                 goto done;
 340         }
 341
 342         if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, imsg_idxfds) == -1) {
 343                 err = got_error_from_errno("socketpair");
 344                 goto done;
 345         }
 346         idxpid = fork();
 347         if (idxpid == -1) {
 348                 err= got_error_from_errno("fork");
 349                 goto done;
 350         } else if (idxpid == 0)
 351                 got_privsep_exec_child(imsg_idxfds,
 352                     GOT_PATH_PROG_INDEX_PACK, packfile_path);
 353         if (close(imsg_idxfds[1]) == -1) {
 354                 err = got_error_from_errno("close");
 355                 goto done;
 356         }
 357         imsg_init(&idxibuf, imsg_idxfds[0]);
 358
 359         npackfd = dup(fileno(packfile));
 360         if (npackfd == -1) {
 361                 err = got_error_from_errno("dup");
 362                 goto done;
 363         }
 364         err = got_privsep_send_index_pack_req(&idxibuf, pack_hash, npackfd);
 365         if (err != NULL)
 366                 goto done;
 367         npackfd = -1;
 368         err = got_privsep_send_index_pack_outfd(&idxibuf, nidxfd);
 369         if (err != NULL)
 370                 goto done;
 371         nidxfd = -1;
 372         for (i = 0; i < nitems(tmpfds); i++) {
 373                 err = got_privsep_send_tmpfd(&idxibuf, tmpfds[i]);
 374                 if (err != NULL)
 375                         goto done;
 376                 tmpfds[i] = -1;
 377         }
 378         done = 0;
 379         while (!done) {
 380                 int nobj_total, nobj_indexed, nobj_loose, nobj_resolved;
 381
 382                 if (cancel_cb) {
 383                         err = cancel_cb(cancel_arg);
 384                         if (err)
 385                                 goto done;
 386                 }
 387
 388                 err = got_privsep_recv_index_progress(&done, &nobj_total,
 389                     &nobj_indexed, &nobj_loose, &nobj_resolved,
 390                     &idxibuf);
 391                 if (err != NULL)
 392                         goto done;
 393                 if (nobj_indexed != 0) {
 394                         err = progress_cb(progress_arg, sb.st_size,
 395                             nobj_total, nobj_indexed, nobj_loose,
 396                             nobj_resolved);
 397                         if (err)
 398                                 break;
 399                 }
 400         }
 401         if (close(imsg_idxfds[0]) == -1) {
 402                 err = got_error_from_errno("close");
 403                 goto done;
 404         }
 405         if (waitpid(idxpid, &idxstatus, 0) == -1) {
 406                 err = got_error_from_errno("waitpid");
 407                 goto done;
 408         }
 409
 410         if (rename(tmpidxpath, idxpath) == -1) {
 411                 err = got_error_from_errno3("rename", tmpidxpath, idxpath);
 412                 goto done;
 413         }
 414         free(tmpidxpath);
 415         tmpidxpath = NULL;
 416
 417 done:
 418         if (tmpidxpath && unlink(tmpidxpath) == -1 && err == NULL)
 419                 err = got_error_from_errno2("unlink", tmpidxpath);
 420         if (npackfd != -1 && close(npackfd) == -1 && err == NULL)
 421                 err = got_error_from_errno("close");
 422         if (idxfd != -1 && close(idxfd) == -1 && err == NULL)
 423                 err = got_error_from_errno("close");
 424         for (i = 0; i < nitems(tmpfds); i++) {
 425                 if (tmpfds[i] != -1 && close(tmpfds[i]) == -1 && err == NULL)
 426                         err = got_error_from_errno("close");
 427         }
 428         free(tmpidxpath);
 429         free(idxpath);
 430         free(packfile_path);
 431         return err;
 432 }
 433
 434 const struct got_error *
 435 got_repo_find_pack(FILE **packfile, struct got_object_id **pack_hash,
 436     struct got_repository *repo, const char *packfile_path)
 437 {
 438         const struct got_error *err = NULL;
 439         const char *packdir_path = NULL;
 440         char *packfile_name = NULL, *p, *dot;
 441         struct got_object_id id;
 442         int packfd = -1;
 443
 444         *packfile = NULL;
 445         *pack_hash = NULL;
 446
 447         packdir_path = got_repo_get_path_objects_pack(repo);
 448         if (packdir_path == NULL)
 449                 return got_error_from_errno("got_repo_get_path_objects_pack");
 450
 451         if (!got_path_is_child(packfile_path, packdir_path,
 452             strlen(packdir_path))) {
 453                 err = got_error_path(packfile_path, GOT_ERR_BAD_PATH);
 454                 goto done;
 455
 456         }
 457
 458         err = got_path_basename(&packfile_name, packfile_path);
 459         if (err)
 460                 goto done;
 461         p = packfile_name;
 462
 463         if (strncmp(p, "pack-", 5) != 0) {
 464                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 465                     "'%s' is not a valid pack file name",
 466                     packfile_name);
 467                 goto done;
 468         }
 469         p += 5;
 470         dot = strchr(p, '.');
 471         if (dot == NULL) {
 472                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 473                     "'%s' is not a valid pack file name",
 474                     packfile_name);
 475                 goto done;
 476         }
 477         if (strcmp(dot + 1, "pack") != 0) {
 478                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 479                     "'%s' is not a valid pack file name",
 480                     packfile_name);
 481                 goto done;
 482         }
 483         *dot = '\0';
 484         if (!got_parse_object_id(&id, p, repo->algo)) {
 485                 err = got_error_fmt(GOT_ERR_BAD_PATH,
 486                     "'%s' is not a valid pack file name",
 487                     packfile_name);
 488                 goto done;
 489         }
 490
 491         *pack_hash = got_object_id_dup(&id);
 492         if (*pack_hash == NULL) {
 493                 err = got_error_from_errno("got_object_id_dup");
 494                 goto done;
 495         }
 496
 497         packfd = open(packfile_path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
 498         if (packfd == -1) {
 499                 err = got_error_from_errno2("open", packfile_path);
 500                 goto done;
 501         }
 502
 503         *packfile = fdopen(packfd, "r");
 504         if (*packfile == NULL) {
 505                 err = got_error_from_errno2("fdopen", packfile_path);
 506                 goto done;
 507         }
 508         packfd = -1;
 509 done:
 510         if (packfd != -1 && close(packfd) == -1 && err == NULL)
 511                 err = got_error_from_errno2("close", packfile_path);
 512         free(packfile_name);
 513         if (err) {
 514                 free(*pack_hash);
 515                 *pack_hash = NULL;
 516         }
 517         return err;
 518 }
 519
 520 const struct got_error *
 521 got_repo_list_pack(FILE *packfile, struct got_object_id *pack_hash,
 522     struct got_repository *repo, got_pack_list_cb list_cb, void *list_arg,
 523     got_cancel_cb cancel_cb, void *cancel_arg)
 524 {
 525         const struct got_error *err = NULL;
 526         char *id_str = NULL, *idxpath = NULL, *packpath = NULL;
 527         struct got_packidx *packidx = NULL;
 528         struct got_pack *pack = NULL;
 529         uint32_t nobj, i;
 530         size_t digest_len = got_hash_digest_length(repo->algo);
 531
 532         err = got_object_id_str(&id_str, pack_hash);
 533         if (err)
 534                 goto done;
 535
 536         if (asprintf(&packpath, "%s/pack-%s.pack",
 537             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 538                 err = got_error_from_errno("asprintf");
 539                 goto done;
 540         }
 541         if (asprintf(&idxpath, "%s/pack-%s.idx",
 542             GOT_OBJECTS_PACK_DIR, id_str) == -1) {
 543                 err = got_error_from_errno("asprintf");
 544                 goto done;
 545         }
 546
 547         err = got_packidx_open(&packidx, got_repo_get_fd(repo), idxpath, 1,
 548             repo->algo);
 549         if (err)
 550                 goto done;
 551
 552         err = got_repo_cache_pack(&pack, repo, packpath, packidx);
 553         if (err)
 554                 goto done;
 555
 556         nobj = be32toh(packidx->hdr.fanout_table[0xff]);
 557         for (i = 0; i < nobj; i++) {
 558                 uint8_t *oid;
 559                 struct got_object_id id, base_id;
 560                 off_t offset, base_offset = 0;
 561                 uint8_t type;
 562                 uint64_t size;
 563                 size_t tslen, len;
 564
 565                 if (cancel_cb) {
 566                         err = cancel_cb(cancel_arg);
 567                         if (err)
 568                                 break;
 569                 }
 570                 oid = packidx->hdr.sorted_ids + i * digest_len;
 571                 id.algo = repo->algo;
 572                 memcpy(id.hash, oid, digest_len);
 573
 574                 offset = got_packidx_get_object_offset(packidx, i);
 575                 if (offset == -1) {
 576                         err = got_error(GOT_ERR_BAD_PACKIDX);
 577                         goto done;
 578                 }
 579
 580                 err = got_pack_parse_object_type_and_size(&type, &size, &tslen,
 581                     pack, offset);
 582                 if (err)
 583                         goto done;
 584
 585                 switch (type) {
 586                 case GOT_OBJ_TYPE_OFFSET_DELTA:
 587                         err = got_pack_parse_offset_delta(&base_offset, &len,
 588                             pack, offset, tslen);
 589                         if (err)
 590                                 goto done;
 591                         break;
 592                 case GOT_OBJ_TYPE_REF_DELTA:
 593                         err = got_pack_parse_ref_delta(&base_id,
 594                             pack, offset, tslen);
 595                         if (err)
 596                                 goto done;
 597                         break;
 598                 }
 599                 err = (*list_cb)(list_arg, &id, type, offset, size,
 600                     base_offset, &base_id);
 601                 if (err)
 602                         goto done;
 603         }
 604
 605 done:
 606         free(id_str);
 607         free(idxpath);
 608         free(packpath);
 609         if (packidx)
 610                 got_packidx_close(packidx);
 611         return err;
 612 }
 613
 614 static const struct got_error *
 615 repo_cleanup_lock(struct got_repository *repo, struct got_lockfile **lk)
 616 {
 617         const struct got_error *err;
 618         char myname[_POSIX_HOST_NAME_MAX + 1];
 619
 620         if (gethostname(myname, sizeof(myname)) == -1)
 621                 return got_error_from_errno("gethostname");
 622
 623         err = got_lockfile_lock(lk, "gc.pid", got_repo_get_fd(repo));
 624         if (err)
 625                 return err;
 626
 627         /*
 628          * Git uses these info to provide some verbiage when finds a
 629          * lock during `git gc --force' so don't try too hard to avoid
 630          * short writes and don't care if a race happens between the
 631          * lockfile creation and the write itself.
 632          */
 633         if (dprintf((*lk)->fd, "%d %s", getpid(), myname) < 0)
 634                 return got_error_from_errno("dprintf");
 635
 636         return NULL;
 637 }
 638
 639 static const struct got_error *
 640 report_cleanup_progress(got_cleanup_progress_cb progress_cb,
 641     void *progress_arg, struct got_ratelimit *rl,
 642     int ncommits, int nloose, int npurged, int nredundant)
 643 {
 644         const struct got_error *err;
 645         int elapsed;
 646
 647         if (progress_cb == NULL)
 648                 return NULL;
 649
 650         err = got_ratelimit_check(&elapsed, rl);
 651         if (err || !elapsed)
 652                 return err;
 653
 654         return progress_cb(progress_arg, ncommits, nloose, npurged,
 655             nredundant);
 656 }
 657
 658 static const struct got_error *
 659 get_loose_object_ids(struct got_object_idset **loose_ids,
 660     off_t *ondisk_size, int ncommits,
 661     got_cleanup_progress_cb progress_cb, void *progress_arg,
 662     struct got_ratelimit *rl, struct got_repository *repo)
 663 {
 664         const struct got_error *err = NULL;
 665         char *path_objects = NULL, *path = NULL;
 666         DIR *dir = NULL;
 667         struct got_object *obj = NULL;
 668         struct got_object_id id;
 669         int i, fd = -1;
 670         struct stat sb;
 671
 672         *ondisk_size = 0;
 673         *loose_ids = got_object_idset_alloc();
 674         if (*loose_ids == NULL)
 675                 return got_error_from_errno("got_object_idset_alloc");
 676
 677         path_objects = got_repo_get_path_objects(repo);
 678         if (path_objects == NULL) {
 679                 err = got_error_from_errno("got_repo_get_path_objects");
 680                 goto done;
 681         }
 682
 683         for (i = 0; i <= 0xff; i++) {
 684                 struct dirent *dent;
 685
 686                 if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
 687                         err = got_error_from_errno("asprintf");
 688                         break;
 689                 }
 690
 691                 dir = opendir(path);
 692                 if (dir == NULL) {
 693                         if (errno == ENOENT) {
 694                                 err = NULL;
 695                                 continue;
 696                         }
 697                         err = got_error_from_errno2("opendir", path);
 698                         break;
 699                 }
 700
 701                 while ((dent = readdir(dir)) != NULL) {
 702                         char *id_str;
 703
 704                         if (strcmp(dent->d_name, ".") == 0 ||
 705                             strcmp(dent->d_name, "..") == 0)
 706                                 continue;
 707
 708                         if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
 709                                 err = got_error_from_errno("asprintf");
 710                                 goto done;
 711                         }
 712
 713                         if (!got_parse_object_id(&id, id_str, repo->algo)) {
 714                                 free(id_str);
 715                                 continue;
 716                         }
 717                         free(id_str);
 718
 719                         err = got_object_open_loose_fd(&fd, &id, repo);
 720                         if (err)
 721                                 goto done;
 722                         if (fstat(fd, &sb) == -1) {
 723                                 err = got_error_from_errno("fstat");
 724                                 goto done;
 725                         }
 726                         err = got_object_read_header_privsep(&obj, &id, repo,
 727                             fd);
 728                         if (err)
 729                                 goto done;
 730                         fd = -1; /* already closed */
 731
 732                         switch (obj->type) {
 733                         case GOT_OBJ_TYPE_COMMIT:
 734                         case GOT_OBJ_TYPE_TREE:
 735                         case GOT_OBJ_TYPE_BLOB:
 736                         case GOT_OBJ_TYPE_TAG:
 737                                 break;
 738                         default:
 739                                 err = got_error_fmt(GOT_ERR_OBJ_TYPE,
 740                                     "%d", obj->type);
 741                                 goto done;
 742                         }
 743                         got_object_close(obj);
 744                         obj = NULL;
 745                         (*ondisk_size) += sb.st_size;
 746                         err = got_object_idset_add(*loose_ids, &id, NULL);
 747                         if (err)
 748                                 goto done;
 749                         err = report_cleanup_progress(progress_cb,
 750                             progress_arg, rl, ncommits,
 751                             got_object_idset_num_elements(*loose_ids),
 752                             -1, -1);
 753                         if (err)
 754                                 goto done;
 755                 }
 756
 757                 if (closedir(dir) != 0) {
 758                         err = got_error_from_errno("closedir");
 759                         goto done;
 760                 }
 761                 dir = NULL;
 762
 763                 free(path);
 764                 path = NULL;
 765         }
 766 done:
 767         if (dir && closedir(dir) != 0 && err == NULL)
 768                 err = got_error_from_errno("closedir");
 769         if (fd != -1 && close(fd) == -1 && err == NULL)
 770                 err = got_error_from_errno("close");
 771         if (err) {
 772                 got_object_idset_free(*loose_ids);
 773                 *loose_ids = NULL;
 774         }
 775         if (obj)
 776                 got_object_close(obj);
 777         free(path_objects);
 778         free(path);
 779         return err;
 780 }
 781
 782 static const struct got_error *
 783 load_tree_entries(struct got_object_id_queue *ids,
 784     struct got_object_idset *traversed_ids, struct got_object_id *tree_id,
 785     const char *dpath, struct got_repository *repo,
 786     got_cancel_cb cancel_cb, void *cancel_arg)
 787 {
 788         const struct got_error *err;
 789         struct got_tree_object *tree;
 790         char *p = NULL;
 791         int i;
 792
 793         err = got_object_open_as_tree(&tree, repo, tree_id);
 794         if (err)
 795                 return err;
 796
 797         for (i = 0; i < got_object_tree_get_nentries(tree); i++) {
 798                 struct got_tree_entry *e = got_object_tree_get_entry(tree, i);
 799                 struct got_object_id *id = got_tree_entry_get_id(e);
 800                 mode_t mode = got_tree_entry_get_mode(e);
 801
 802                 if (cancel_cb) {
 803                         err = (*cancel_cb)(cancel_arg);
 804                         if (err)
 805                                 break;
 806                 }
 807
 808                 if (got_object_tree_entry_is_symlink(e) ||
 809                     got_object_tree_entry_is_submodule(e) ||
 810                     got_object_idset_contains(traversed_ids, id))
 811                         continue;
 812
 813                 if (asprintf(&p, "%s%s%s", dpath, dpath[0] != '\0' ? "/" : "",
 814                     got_tree_entry_get_name(e)) == -1) {
 815                         err = got_error_from_errno("asprintf");
 816                         break;
 817                 }
 818
 819                 if (S_ISDIR(mode)) {
 820                         struct got_object_qid *qid;
 821                         err = got_object_qid_alloc(&qid, id);
 822                         if (err)
 823                                 break;
 824                         STAILQ_INSERT_TAIL(ids, qid, entry);
 825                 } else if (S_ISREG(mode)) {
 826                         /* This blob is referenced. */
 827                         err = got_object_idset_add(traversed_ids, id, NULL);
 828                         if (err)
 829                                 break;
 830                 }
 831                 free(p);
 832                 p = NULL;
 833         }
 834
 835         got_object_tree_close(tree);
 836         free(p);
 837         return err;
 838 }
 839
 840 static const struct got_error *
 841 load_tree(struct got_object_idset *traversed_ids,
 842     struct got_object_id *tree_id,
 843     const char *dpath, struct got_repository *repo,
 844     got_cancel_cb cancel_cb, void *cancel_arg)
 845 {
 846         const struct got_error *err = NULL;
 847         struct got_object_id_queue tree_ids;
 848         struct got_object_qid *qid;
 849
 850         err = got_object_qid_alloc(&qid, tree_id);
 851         if (err)
 852                 return err;
 853
 854         STAILQ_INIT(&tree_ids);
 855         STAILQ_INSERT_TAIL(&tree_ids, qid, entry);
 856
 857         while (!STAILQ_EMPTY(&tree_ids)) {
 858                 if (cancel_cb) {
 859                         err = (*cancel_cb)(cancel_arg);
 860                         if (err)
 861                                 break;
 862                 }
 863
 864                 qid = STAILQ_FIRST(&tree_ids);
 865                 STAILQ_REMOVE_HEAD(&tree_ids, entry);
 866
 867                 if (got_object_idset_contains(traversed_ids, &qid->id)) {
 868                         got_object_qid_free(qid);
 869                         continue;
 870                 }
 871
 872                 err = got_object_idset_add(traversed_ids, &qid->id, NULL);
 873                 if (err) {
 874                         got_object_qid_free(qid);
 875                         break;
 876                 }
 877
 878                 err = load_tree_entries(&tree_ids, traversed_ids,
 879                     &qid->id, dpath, repo, cancel_cb, cancel_arg);
 880                 got_object_qid_free(qid);
 881                 if (err)
 882                         break;
 883         }
 884
 885         got_object_id_queue_free(&tree_ids);
 886         return err;
 887 }
 888
 889 static const struct got_error *
 890 load_commit_or_tag(int *ncommits, struct got_object_idset *traversed_ids,
 891     struct got_object_id *id, struct got_repository *repo,
 892     got_cleanup_progress_cb progress_cb, void *progress_arg,
 893     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 894 {
 895         const struct got_error *err;
 896         struct got_commit_object *commit = NULL;
 897         struct got_tag_object *tag = NULL;
 898         struct got_object_id *tree_id = NULL;
 899         struct got_object_id_queue ids;
 900         struct got_object_qid *qid;
 901         int obj_type;
 902
 903         err = got_object_qid_alloc(&qid, id);
 904         if (err)
 905                 return err;
 906
 907         STAILQ_INIT(&ids);
 908         STAILQ_INSERT_TAIL(&ids, qid, entry);
 909
 910         while (!STAILQ_EMPTY(&ids)) {
 911                 if (cancel_cb) {
 912                         err = (*cancel_cb)(cancel_arg);
 913                         if (err)
 914                                 break;
 915                 }
 916
 917                 qid = STAILQ_FIRST(&ids);
 918                 STAILQ_REMOVE_HEAD(&ids, entry);
 919
 920                 if (got_object_idset_contains(traversed_ids, &qid->id)) {
 921                         got_object_qid_free(qid);
 922                         qid = NULL;
 923                         continue;
 924                 }
 925
 926                 err = got_object_idset_add(traversed_ids, &qid->id, NULL);
 927                 if (err)
 928                         break;
 929
 930                 err = got_object_get_type(&obj_type, repo, &qid->id);
 931                 if (err)
 932                         break;
 933                 switch (obj_type) {
 934                 case GOT_OBJ_TYPE_COMMIT:
 935                         err = got_object_open_as_commit(&commit, repo,
 936                             &qid->id);
 937                         if (err)
 938                                 goto done;
 939                         break;
 940                 case GOT_OBJ_TYPE_TAG:
 941                         err = got_object_open_as_tag(&tag, repo, &qid->id);
 942                         if (err)
 943                                 goto done;
 944                         break;
 945                 default:
 946                         /* should not happen */
 947                         err = got_error(GOT_ERR_OBJ_TYPE);
 948                         goto done;
 949                 }
 950
 951                 /* Find a tree object to scan. */
 952                 if (commit) {
 953                         tree_id = got_object_commit_get_tree_id(commit);
 954                 } else if (tag) {
 955                         obj_type = got_object_tag_get_object_type(tag);
 956                         switch (obj_type) {
 957                         case GOT_OBJ_TYPE_COMMIT:
 958                                 err = got_object_open_as_commit(&commit, repo,
 959                                     got_object_tag_get_object_id(tag));
 960                                 if (err)
 961                                         goto done;
 962                                 tree_id = got_object_commit_get_tree_id(commit);
 963                                 break;
 964                         case GOT_OBJ_TYPE_TREE:
 965                                 tree_id = got_object_tag_get_object_id(tag);
 966                                 break;
 967                         default:
 968                                 /*
 969                                  * Tag points at something other than a
 970                                  * commit or tree. Leave this weird tag object
 971                                  * and the object it points to.
 972                                  */
 973                                 if (got_object_idset_contains(traversed_ids,
 974                                     got_object_tag_get_object_id(tag)))
 975                                         break;
 976                                 err = got_object_idset_add(traversed_ids,
 977                                     got_object_tag_get_object_id(tag), NULL);
 978                                 if (err)
 979                                         goto done;
 980                                 break;
 981                         }
 982                 }
 983
 984                 if (tree_id) {
 985                         err = load_tree(traversed_ids, tree_id, "",
 986                             repo, cancel_cb, cancel_arg);
 987                         if (err)
 988                                 break;
 989                 }
 990
 991                 if (commit || tag)
 992                         (*ncommits)++; /* scanned tags are counted as commits */
 993
 994                 err = report_cleanup_progress(progress_cb, progress_arg, rl,
 995                     *ncommits, -1, -1, -1);
 996                 if (err)
 997                         break;
 998
 999                 if (commit) {
1000                         /* Find parent commits to scan. */
1001                         const struct got_object_id_queue *parent_ids;
1002                         parent_ids = got_object_commit_get_parent_ids(commit);
1003                         err = got_object_id_queue_copy(parent_ids, &ids);
1004                         if (err)
1005                                 break;
1006                         got_object_commit_close(commit);
1007                         commit = NULL;
1008                 }
1009                 if (tag) {
1010                         got_object_tag_close(tag);
1011                         tag = NULL;
1012                 }
1013                 got_object_qid_free(qid);
1014                 qid = NULL;
1015         }
1016 done:
1017         if (qid)
1018                 got_object_qid_free(qid);
1019         if (commit)
1020                 got_object_commit_close(commit);
1021         if (tag)
1022                 got_object_tag_close(tag);
1023         got_object_id_queue_free(&ids);
1024         return err;
1025 }
1026
1027 static const struct got_error *
1028 is_object_packed(int *packed, struct got_repository *repo,
1029     struct got_object_id *id)
1030 {
1031         const struct got_error *err;
1032         struct got_object *obj;
1033
1034         *packed = 0;
1035
1036         err = got_object_open_packed(&obj, id, repo);
1037         if (err) {
1038                 if (err->code == GOT_ERR_NO_OBJ)
1039                         err = NULL;
1040                 return err;
1041         }
1042         got_object_close(obj);
1043         *packed = 1;
1044         return NULL;
1045 }
1046
1047 struct purge_loose_object_arg {
1048         struct got_repository *repo;
1049         got_cleanup_progress_cb progress_cb;
1050         void *progress_arg;
1051         struct got_ratelimit *rl;
1052         struct got_object_idset *traversed_ids;
1053         int nloose;
1054         int ncommits;
1055         int npacked;
1056         int npurged;
1057         off_t size_purged;
1058         int dry_run;
1059         time_t max_mtime;
1060         int ignore_mtime;
1061 };
1062
1063 static const struct got_error *
1064 purge_loose_object(struct got_object_id *id, void *data, void *arg)
1065 {
1066         struct purge_loose_object_arg *a = arg;
1067         const struct got_error *err, *unlock_err = NULL;
1068         char *path = NULL;
1069         int packed, fd = -1;
1070         struct stat sb;
1071         struct got_lockfile *lf = NULL;
1072
1073         err = is_object_packed(&packed, a->repo, id);
1074         if (err)
1075                 return err;
1076
1077         if (!packed && got_object_idset_contains(a->traversed_ids, id))
1078                 return NULL;
1079
1080         if (packed)
1081                 a->npacked++;
1082
1083         err = got_object_get_path(&path, id, a->repo);
1084         if (err)
1085                 return err;
1086
1087         err = got_object_open_loose_fd(&fd, id, a->repo);
1088         if (err)
1089                 goto done;
1090
1091         if (fstat(fd, &sb) == -1) {
1092                 err = got_error_from_errno("fstat");
1093                 goto done;
1094         }
1095
1096         /*
1097          * Do not delete objects which are younger than our maximum
1098          * modification time threshold. This prevents a race where
1099          * new objects which are being added to the repository
1100          * concurrently would be deleted.
1101          */
1102         if (a->ignore_mtime || sb.st_mtime <= a->max_mtime) {
1103                 if (!a->dry_run) {
1104                         err = got_lockfile_lock(&lf, path, -1);
1105                         if (err)
1106                                 goto done;
1107                         if (unlink(path) == -1) {
1108                                 err = got_error_from_errno2("unlink", path);
1109                                 goto done;
1110                         }
1111                 }
1112
1113                 a->npurged++;
1114                 a->size_purged += sb.st_size;
1115                 err = report_cleanup_progress(a->progress_cb, a->progress_arg,
1116                     a->rl, a->ncommits, a->nloose, a->npurged, -1);
1117                 if (err)
1118                         goto done;
1119         }
1120 done:
1121         if (fd != -1 && close(fd) == -1 && err == NULL)
1122                 err = got_error_from_errno("close");
1123         free(path);
1124         if (lf)
1125                 unlock_err = got_lockfile_unlock(lf, -1);
1126         return err ? err : unlock_err;
1127 }
1128
1129 static const struct got_error *
1130 repo_purge_unreferenced_loose_objects(struct got_repository *repo,
1131     struct got_object_idset *traversed_ids,
1132     off_t *size_before, off_t *size_after, int ncommits, int *nloose,
1133     int *npacked, int *npurged, int dry_run, int ignore_mtime,
1134     time_t max_mtime, struct got_ratelimit *rl,
1135     got_cleanup_progress_cb progress_cb, void *progress_arg,
1136     got_cancel_cb cancel_cb, void *cancel_arg)
1137 {
1138         const struct got_error *err;
1139         struct got_object_idset *loose_ids;
1140         struct purge_loose_object_arg arg;
1141
1142         err = get_loose_object_ids(&loose_ids, size_before, ncommits,
1143             progress_cb, progress_arg, rl, repo);
1144         if (err)
1145                 return err;
1146         *nloose = got_object_idset_num_elements(loose_ids);
1147         if (*nloose == 0) {
1148                 got_object_idset_free(loose_ids);
1149                 if (progress_cb) {
1150                         err = progress_cb(progress_arg, 0, 0, 0, -1);
1151                         if (err)
1152                                 return err;
1153                 }
1154                 return NULL;
1155         }
1156
1157         memset(&arg, 0, sizeof(arg));
1158         arg.repo = repo;
1159         arg.progress_arg = progress_arg;
1160         arg.progress_cb = progress_cb;
1161         arg.rl = rl;
1162         arg.traversed_ids = traversed_ids;
1163         arg.nloose = *nloose;
1164         arg.npacked = 0;
1165         arg.npurged = 0;
1166         arg.size_purged = 0;
1167         arg.dry_run = dry_run;
1168         arg.max_mtime = max_mtime;
1169         arg.ignore_mtime = ignore_mtime;
1170         err = got_object_idset_for_each(loose_ids, purge_loose_object, &arg);
1171         if (err)
1172                 goto done;
1173
1174         *size_after = *size_before - arg.size_purged;
1175         *npacked = arg.npacked;
1176         *npurged = arg.npurged;
1177
1178         /* Produce a final progress report. */
1179         if (progress_cb) {
1180                 err = progress_cb(progress_arg, ncommits, *nloose,
1181                     arg.npurged, -1);
1182                 if (err)
1183                         goto done;
1184         }
1185 done:
1186         got_object_idset_free(loose_ids);
1187         return err;
1188 }
1189
1190 static const struct got_error *
1191 purge_redundant_pack(struct got_repository *repo, const char *packidx_path,
1192     int dry_run, int ignore_mtime, time_t max_mtime,
1193     int *remove, off_t *size_before, off_t *size_after)
1194 {
1195         static const char *ext[] = {".idx", ".pack", ".rev", ".bitmap",
1196             ".promisor", ".mtimes"};
1197         struct stat sb;
1198         char *dot, path[PATH_MAX];
1199         size_t i;
1200
1201         if (strlcpy(path, packidx_path, sizeof(path)) >= sizeof(path))
1202                 return got_error(GOT_ERR_NO_SPACE);
1203
1204         /*
1205          * Do not delete pack files which are younger than our maximum
1206          * modification time threshold.  This prevents a race where a
1207          * new pack file which is being added to the repository
1208          * concurrently would be deleted.
1209          */
1210         if (fstatat(got_repo_get_fd(repo), path, &sb, 0) == -1) {
1211                 if (errno == ENOENT)
1212                         return NULL;
1213                 return got_error_from_errno2("fstatat", path);
1214         }
1215         if (!ignore_mtime && sb.st_mtime > max_mtime)
1216                 *remove = 0;
1217
1218         /*
1219          * For compatibility with Git, if a matching .keep file exist
1220          * don't delete the packfile.
1221          */
1222         dot = strrchr(path, '.');
1223         *dot = '\0';
1224         if (strlcat(path, ".keep", sizeof(path)) >= sizeof(path))
1225                 return got_error(GOT_ERR_NO_SPACE);
1226         if (faccessat(got_repo_get_fd(repo), path, F_OK, 0) == 0)
1227                 *remove = 0;
1228
1229         for (i = 0; i < nitems(ext); ++i) {
1230                 *dot = '\0';
1231
1232                 if (strlcat(path, ext[i], sizeof(path)) >=
1233                     sizeof(path))
1234                         return got_error(GOT_ERR_NO_SPACE);
1235
1236                 if (fstatat(got_repo_get_fd(repo), path, &sb, 0) ==
1237                     -1) {
1238                         if (errno == ENOENT)
1239                                 continue;
1240                         return got_error_from_errno2("fstatat", path);
1241                 }
1242
1243                 *size_before += sb.st_size;
1244                 if (!*remove) {
1245                         *size_after += sb.st_size;
1246                         continue;
1247                 }
1248
1249                 if (dry_run)
1250                         continue;
1251
1252                 if (unlinkat(got_repo_get_fd(repo), path, 0) == -1) {
1253                         if (errno == ENOENT)
1254                                 continue;
1255                         return got_error_from_errno2("unlinkat",
1256                             path);
1257                 }
1258         }
1259
1260         return NULL;
1261 }
1262
1263 static const struct got_error *
1264 pack_is_redundant(int *redundant, struct got_repository *repo,
1265     struct got_object_idset *traversed_ids,
1266     const char *packidx_path, struct got_object_idset *idset)
1267 {
1268         const struct got_error *err;
1269         struct got_packidx *packidx;
1270         uint8_t *pid;
1271         struct got_object_id id;
1272         size_t i, nobjects;
1273         size_t digest_len = got_hash_digest_length(repo->algo);
1274
1275         *redundant = 1;
1276
1277         err = got_repo_get_packidx(&packidx, packidx_path, repo);
1278         if (err)
1279                 return err;
1280
1281         nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1282         for (i = 0; i < nobjects; ++i) {
1283                 pid = packidx->hdr.sorted_ids + i * digest_len;
1284
1285                 memset(&id, 0, sizeof(id));
1286                 memcpy(&id.hash, pid, digest_len);
1287                 id.algo = repo->algo;
1288
1289                 if (got_object_idset_contains(idset, &id))
1290                         continue;
1291
1292                 if (!got_object_idset_contains(traversed_ids, &id))
1293                         continue;
1294
1295                 *redundant = 0;
1296                 err = got_object_idset_add(idset, &id, NULL);
1297                 if (err)
1298                         return err;
1299         }
1300
1301         return NULL;
1302 }
1303
1304 struct pack_info {
1305         const char      *path;
1306         size_t           nobjects;
1307 };
1308
1309 static int
1310 pack_info_cmp(const void *a, const void *b)
1311 {
1312         const struct pack_info  *pa, *pb;
1313
1314         pa = a;
1315         pb = b;
1316         if (pa->nobjects == pb->nobjects)
1317                 return strcmp(pa->path, pb->path);
1318         if (pa->nobjects > pb->nobjects)
1319                 return -1;
1320         return 1;
1321 }
1322
1323 static const struct got_error *
1324 repo_purge_redundant_packfiles(struct got_repository *repo,
1325     struct got_object_idset *traversed_ids,
1326     off_t *size_before, off_t *size_after, int dry_run, int ignore_mtime,
1327     time_t max_mtime, int nloose, int ncommits, int npurged,
1328     struct got_ratelimit *rl,
1329     got_cleanup_progress_cb progress_cb, void *progress_arg,
1330     got_cancel_cb cancel_cb, void *cancel_arg)
1331 {
1332         const struct got_error *err;
1333         struct pack_info *pinfo, *sorted = NULL;
1334         struct got_packidx *packidx;
1335         struct got_object_idset *idset = NULL;
1336         struct got_pathlist_entry *pe;
1337         size_t i, npacks;
1338         int remove, redundant_packs = 0;
1339
1340         npacks = 0;
1341         TAILQ_FOREACH(pe, &repo->packidx_paths, entry)
1342                 npacks++;
1343
1344         if (npacks == 0)
1345                 return NULL;
1346
1347         sorted = calloc(npacks, sizeof(*sorted));
1348         if (sorted == NULL)
1349                 return got_error_from_errno("calloc");
1350
1351         i = 0;
1352         TAILQ_FOREACH(pe, &repo->packidx_paths, entry) {
1353                 err = got_repo_get_packidx(&packidx, pe->path, repo);
1354                 if (err)
1355                         goto done;
1356
1357                 pinfo = &sorted[i++];
1358                 pinfo->path = pe->path;
1359                 pinfo->nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1360         }
1361         qsort(sorted, npacks, sizeof(*sorted), pack_info_cmp);
1362
1363         idset = got_object_idset_alloc();
1364         if (idset == NULL) {
1365                 err = got_error_from_errno("got_object_idset_alloc");
1366                 goto done;
1367         }
1368
1369         for (i = 0; i < npacks; ++i) {
1370                 if (cancel_cb) {
1371                         err = (*cancel_cb)(cancel_arg);
1372                         if (err)
1373                                 break;
1374                 }
1375
1376                 err = pack_is_redundant(&remove, repo, traversed_ids,
1377                     sorted[i].path, idset);
1378                 if (err)
1379                         goto done;
1380                 err = purge_redundant_pack(repo, sorted[i].path, dry_run,
1381                     ignore_mtime, max_mtime, &remove, size_before, size_after);
1382                 if (err)
1383                         goto done;
1384                 if (!remove)
1385                         continue;
1386                 err = report_cleanup_progress(progress_cb, progress_arg,
1387                     rl, ncommits, nloose, npurged, ++redundant_packs);
1388                 if (err)
1389                         goto done;
1390         }
1391
1392         /* Produce a final progress report. */
1393         if (progress_cb) {
1394                 err = progress_cb(progress_arg, ncommits, nloose, npurged,
1395                     redundant_packs);
1396                 if (err)
1397                         goto done;
1398         }
1399  done:
1400         free(sorted);
1401         if (idset)
1402                 got_object_idset_free(idset);
1403         return err;
1404 }
1405
1406 const struct got_error *
1407 got_repo_cleanup(struct got_repository *repo,
1408     off_t *loose_before, off_t *loose_after,
1409     off_t *pack_before, off_t *pack_after,
1410     int *ncommits, int *nloose, int *npacked, int dry_run, int ignore_mtime,
1411     got_cleanup_progress_cb progress_cb, void *progress_arg,
1412     got_cancel_cb cancel_cb, void *cancel_arg)
1413 {
1414         const struct got_error *unlock_err, *err = NULL;
1415         struct got_lockfile *lk = NULL;
1416         struct got_ratelimit rl;
1417         struct got_reflist_head refs;
1418         struct got_object_idset *traversed_ids = NULL;
1419         struct got_reflist_entry *re;
1420         struct got_object_id **referenced_ids;
1421         int i, nreferenced;
1422         int npurged = 0;
1423         time_t max_mtime = 0;
1424
1425         TAILQ_INIT(&refs);
1426         got_ratelimit_init(&rl, 0, 500);
1427
1428         *loose_before = 0;
1429         *loose_after = 0;
1430         *pack_before = 0;
1431         *pack_after = 0;
1432         *ncommits = 0;
1433         *nloose = 0;
1434         *npacked = 0;
1435
1436         err = repo_cleanup_lock(repo, &lk);
1437         if (err)
1438                 return err;
1439
1440         traversed_ids = got_object_idset_alloc();
1441         if (traversed_ids == NULL) {
1442                 err = got_error_from_errno("got_object_idset_alloc");
1443                 goto done;
1444         }
1445
1446         err = got_ref_list(&refs, repo, "", got_ref_cmp_by_name, NULL);
1447         if (err)
1448                 goto done;
1449         if (!ignore_mtime) {
1450                 TAILQ_FOREACH(re, &refs, entry) {
1451                         time_t mtime = got_ref_get_mtime(re->ref);
1452                         if (mtime > max_mtime)
1453                                 max_mtime = mtime;
1454                 }
1455                 /*
1456                  * For safety, keep objects created within 10 minutes
1457                  * before the youngest reference was created.
1458                  */
1459                 if (max_mtime >= 600)
1460                         max_mtime -= 600;
1461         }
1462
1463         err = get_reflist_object_ids(&referenced_ids, &nreferenced,
1464             (1 << GOT_OBJ_TYPE_COMMIT) | (1 << GOT_OBJ_TYPE_TAG),
1465             &refs, repo, cancel_cb, cancel_arg);
1466         if (err)
1467                 goto done;
1468
1469         for (i = 0; i < nreferenced; i++) {
1470                 struct got_object_id *id = referenced_ids[i];
1471                 err = load_commit_or_tag(ncommits, traversed_ids,
1472                     id, repo, progress_cb, progress_arg, &rl,
1473                     cancel_cb, cancel_arg);
1474                 if (err)
1475                         goto done;
1476         }
1477
1478         err = repo_purge_unreferenced_loose_objects(repo, traversed_ids,
1479             loose_before, loose_after, *ncommits, nloose, npacked, &npurged,
1480             dry_run, ignore_mtime, max_mtime, &rl, progress_cb, progress_arg,
1481             cancel_cb, cancel_arg);
1482         if (err)
1483                 goto done;
1484
1485         err = repo_purge_redundant_packfiles(repo, traversed_ids,
1486             pack_before, pack_after, dry_run, ignore_mtime, max_mtime,
1487             *nloose, *ncommits, npurged, &rl, progress_cb, progress_arg,
1488             cancel_cb, cancel_arg);
1489         if (err)
1490                 goto done;
1491
1492  done:
1493         if (lk) {
1494                 unlock_err = got_lockfile_unlock(lk, got_repo_get_fd(repo));
1495                 if (err == NULL)
1496                         err = unlock_err;
1497         }
1498         if (traversed_ids)
1499                 got_object_idset_free(traversed_ids);
1500         return err;
1501 }
1502
1503 const struct got_error *
1504 got_repo_remove_lonely_packidx(struct got_repository *repo, int dry_run,
1505     got_lonely_packidx_progress_cb progress_cb, void *progress_arg,
1506     got_cancel_cb cancel_cb, void *cancel_arg)
1507 {
1508         const struct got_error *err = NULL;
1509         DIR *packdir = NULL;
1510         struct dirent *dent;
1511         char *pack_relpath = NULL;
1512         int packdir_fd;
1513         struct stat sb;
1514
1515         packdir_fd = openat(got_repo_get_fd(repo),
1516             GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1517         if (packdir_fd == -1) {
1518                 if (errno == ENOENT)
1519                         return NULL;
1520                 return got_error_from_errno_fmt("openat: %s/%s",
1521                     got_repo_get_path_git_dir(repo),
1522                     GOT_OBJECTS_PACK_DIR);
1523         }
1524
1525         packdir = fdopendir(packdir_fd);
1526         if (packdir == NULL) {
1527                 err = got_error_from_errno("fdopendir");
1528                 close(packdir_fd);
1529                 goto done;
1530         }
1531
1532         while ((dent = readdir(packdir)) != NULL) {
1533                 if (cancel_cb) {
1534                         err = cancel_cb(cancel_arg);
1535                         if (err)
1536                                 goto done;
1537                 }
1538
1539                 if (!got_repo_is_packidx_filename(dent->d_name,
1540                     strlen(dent->d_name),
1541                     got_repo_get_object_format(repo)))
1542                         continue;
1543
1544                 err = got_packidx_get_packfile_path(&pack_relpath,
1545                     dent->d_name);
1546                 if (err)
1547                         goto done;
1548
1549                 if (fstatat(packdir_fd, pack_relpath, &sb, 0) != -1) {
1550                         free(pack_relpath);
1551                         pack_relpath = NULL;
1552                         continue;
1553                 }
1554                 if (errno != ENOENT) {
1555                         err = got_error_from_errno_fmt("fstatat: %s/%s/%s",
1556                             got_repo_get_path_git_dir(repo),
1557                             GOT_OBJECTS_PACK_DIR,
1558                             pack_relpath);
1559                         goto done;
1560                 }
1561
1562                 if (!dry_run) {
1563                         if (unlinkat(packdir_fd, dent->d_name, 0) == -1) {
1564                                 err = got_error_from_errno("unlinkat");
1565                                 goto done;
1566                         }
1567                 }
1568                 if (progress_cb) {
1569                         char *path;
1570                         if (asprintf(&path, "%s/%s/%s",
1571                             got_repo_get_path_git_dir(repo),
1572                             GOT_OBJECTS_PACK_DIR,
1573                             dent->d_name) == -1) {
1574                                 err = got_error_from_errno("asprintf");
1575                                 goto done;
1576                         }
1577                         err = progress_cb(progress_arg, path);
1578                         free(path);
1579                         if (err)
1580                                 goto done;
1581                 }
1582                 free(pack_relpath);
1583                 pack_relpath = NULL;
1584         }
1585 done:
1586         if (packdir && closedir(packdir) != 0 && err == NULL)
1587                 err = got_error_from_errno("closedir");
1588         free(pack_relpath);
1589         return err;
1590 }