lib/pack_create_io.c

   1 /*
   2  * Copyright (c) 2020 Ori Bernstein
   3  * Copyright (c) 2021, 2022 Stefan Sperling <stsp@openbsd.org>
   4  *
   5  * Permission to use, copy, modify, and distribute this software for any
   6  * purpose with or without fee is hereby granted, provided that the above
   7  * copyright notice and this permission notice appear in all copies.
   8  *
   9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  16  */
  17
  18 #include "got_compat.h"
  19
  20 #include <sys/types.h>
  21 #include <sys/queue.h>
  22 #include <sys/uio.h>
  23
  24 #include <limits.h>
  25 #include <stdio.h>
  26 #include <stdint.h>
  27 #include <stdlib.h>
  28 #include <string.h>
  29 #include <time.h>
  30 #include <imsg.h>
  31 #include <inttypes.h>
  32 #include <unistd.h>
  33
  34 #include "got_error.h"
  35 #include "got_cancel.h"
  36 #include "got_object.h"
  37 #include "got_reference.h"
  38 #include "got_repository_admin.h"
  39 #include "got_path.h"
  40
  41 #include "got_lib_delta.h"
  42 #include "got_lib_hash.h"
  43 #include "got_lib_object.h"
  44 #include "got_lib_object_cache.h"
  45 #include "got_lib_object_idset.h"
  46 #include "got_lib_ratelimit.h"
  47 #include "got_lib_pack.h"
  48 #include "got_lib_pack_create.h"
  49 #include "got_lib_repository.h"
  50
  51 static const struct got_error *
  52 get_base_object_id(struct got_object_id *base_id, struct got_packidx *packidx,
  53     off_t base_offset)
  54 {
  55         const struct got_error *err;
  56         int idx;
  57
  58         err = got_packidx_get_offset_idx(&idx, packidx, base_offset);
  59         if (err)
  60                 return err;
  61         if (idx == -1)
  62                 return got_error(GOT_ERR_BAD_PACKIDX);
  63
  64         return got_packidx_get_object_id(base_id, packidx, idx);
  65 }
  66
  67 struct search_deltas_arg {
  68         struct got_pack_metavec *v;
  69         struct got_packidx *packidx;
  70         struct got_pack *pack;
  71         struct got_object_idset *idset;
  72         int ncolored, nfound, ntrees, ncommits;
  73         got_pack_progress_cb progress_cb;
  74         void *progress_arg;
  75         struct got_ratelimit *rl;
  76         got_cancel_cb cancel_cb;
  77         void *cancel_arg;
  78 };
  79
  80 static const struct got_error *
  81 search_delta_for_object(struct got_object_id *id, void *data, void *arg)
  82 {
  83         const struct got_error *err;
  84         struct search_deltas_arg *a = arg;
  85         int obj_idx;
  86         uint8_t *delta_buf = NULL;
  87         uint64_t base_size, result_size;
  88         size_t delta_size, delta_compressed_size;
  89         off_t delta_offset, delta_data_offset, base_offset;
  90         struct got_object_id base_id;
  91
  92         if (a->cancel_cb) {
  93                 err = a->cancel_cb(a->cancel_arg);
  94                 if (err)
  95                         return err;
  96         }
  97
  98         obj_idx = got_packidx_get_object_idx(a->packidx, id);
  99         if (obj_idx == -1)
 100                 return NULL; /* object not present in our pack file */
 101
 102         err = got_packfile_extract_raw_delta(&delta_buf, &delta_size,
 103             &delta_compressed_size, &delta_offset, &delta_data_offset,
 104             &base_offset, &base_id, &base_size, &result_size,
 105             a->pack, a->packidx, obj_idx);
 106         if (err) {
 107                 if (err->code == GOT_ERR_OBJ_TYPE)
 108                         return NULL; /* object not stored as a delta */
 109                 return err;
 110         }
 111
 112         /*
 113          * If this is an offset delta we must determine the base
 114          * object ID ourselves.
 115          */
 116         if (base_offset != 0) {
 117                 err = get_base_object_id(&base_id, a->packidx, base_offset);
 118                 if (err)
 119                         goto done;
 120         }
 121
 122         if (got_object_idset_contains(a->idset, &base_id)) {
 123                 struct got_pack_meta *m, *base;
 124
 125                 m = got_object_idset_get(a->idset, id);
 126                 if (m == NULL) {
 127                         err = got_error_msg(GOT_ERR_NO_OBJ,
 128                             "delta object not found");
 129                         goto done;
 130                 }
 131
 132                 base = got_object_idset_get(a->idset, &base_id);
 133                 if (m == NULL) {
 134                         err = got_error_msg(GOT_ERR_NO_OBJ,
 135                             "delta base object not found");
 136                         goto done;
 137                 }
 138
 139                 m->base_obj_id = got_object_id_dup(&base_id);
 140                 if (m->base_obj_id == NULL) {
 141                         err = got_error_from_errno("got_object_id_dup");
 142                         goto done;
 143                 }
 144
 145                 m->prev = base;
 146                 m->size = result_size;
 147                 m->delta_len = delta_size;
 148                 m->delta_compressed_len = delta_compressed_size;
 149                 m->reused_delta_offset = delta_data_offset;
 150                 m->delta_offset = 0;
 151
 152                 err = got_pack_add_meta(m, a->v);
 153                 if (err)
 154                         goto done;
 155
 156                 err = got_pack_report_progress(a->progress_cb, a->progress_arg,
 157                     a->rl, a->ncolored, a->nfound, a->ntrees, 0L, a->ncommits,
 158                     got_object_idset_num_elements(a->idset), a->v->nmeta, 0, 0);
 159                 if (err)
 160                         goto done;
 161         }
 162 done:
 163         free(delta_buf);
 164         return err;
 165 }
 166
 167 const struct got_error *
 168 got_pack_search_deltas(struct got_packidx **packidx, struct got_pack **pack,
 169     struct got_pack_metavec *v, struct got_object_idset *idset,
 170     int ncolored, int nfound, int ntrees, int ncommits,
 171     struct got_repository *repo,
 172     got_pack_progress_cb progress_cb, void *progress_arg,
 173     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 174 {
 175         const struct got_error *err = NULL;
 176         struct search_deltas_arg sda;
 177
 178         *packidx = NULL;
 179         *pack = NULL;
 180
 181         err = got_pack_find_pack_for_reuse(packidx, repo);
 182         if (err)
 183                 return err;
 184
 185         if (*packidx == NULL)
 186                 return NULL;
 187
 188         err = got_pack_cache_pack_for_packidx(pack, *packidx, repo);
 189         if (err)
 190                 return err;
 191
 192         memset(&sda, 0, sizeof(sda));
 193         sda.v = v;
 194         sda.idset = idset;
 195         sda.pack = *pack;
 196         sda.packidx = *packidx;
 197         sda.ncolored = ncolored;
 198         sda.nfound = nfound;
 199         sda.ntrees = ntrees;
 200         sda.ncommits = ncommits;
 201         sda.progress_cb = progress_cb;
 202         sda.progress_arg = progress_arg;
 203         sda.rl = rl;
 204         sda.cancel_cb = cancel_cb;
 205         sda.cancel_arg = cancel_arg;
 206         return got_object_idset_for_each(idset, search_delta_for_object, &sda);
 207 }
 208
 209 const struct got_error *
 210 got_pack_load_packed_object_ids(int *found_all_objects,
 211     struct got_object_id **ours, int nours,
 212     struct got_object_id **theirs, int ntheirs,
 213     int want_meta, uint32_t seed, struct got_object_idset *idset,
 214     struct got_object_idset *idset_exclude, int loose_obj_only,
 215     struct got_repository *repo, struct got_packidx *packidx,
 216     int *ncolored, int *nfound, int *ntrees,
 217     got_pack_progress_cb progress_cb, void *progress_arg,
 218     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 219 {
 220         /* We do not need this optimized traversal while using direct I/O. */
 221         *found_all_objects = 0;
 222         return NULL;
 223 }
 224
 225 const struct got_error *
 226 got_pack_paint_commits(int *ncolored, struct got_object_id_queue *ids, int nids,
 227     struct got_object_idset *keep, struct got_object_idset *drop,
 228     struct got_object_idset *skip, struct got_repository *repo,
 229     got_pack_progress_cb progress_cb, void *progress_arg,
 230     struct got_ratelimit *rl, got_cancel_cb cancel_cb, void *cancel_arg)
 231 {
 232         const struct got_error *err = NULL;
 233         struct got_commit_object *commit = NULL;
 234         struct got_packidx *packidx = NULL;
 235         struct got_pack *pack = NULL;
 236         const struct got_object_id_queue *parents;
 237         struct got_object_qid *qid = NULL;
 238         int nqueued = nids, nskip = 0;
 239
 240         while (!STAILQ_EMPTY(ids) && nskip != nqueued) {
 241                 intptr_t color;
 242
 243                 if (cancel_cb) {
 244                         err = cancel_cb(cancel_arg);
 245                         if (err)
 246                                 break;
 247                 }
 248
 249                 qid = STAILQ_FIRST(ids);
 250                 STAILQ_REMOVE_HEAD(ids, entry);
 251                 nqueued--;
 252                 color = (intptr_t)qid->data;
 253                 if (color == COLOR_SKIP)
 254                         nskip--;
 255
 256                 if (got_object_idset_contains(skip, &qid->id)) {
 257                         got_object_qid_free(qid);
 258                         qid = NULL;
 259                         continue;
 260                 }
 261                 if (color == COLOR_KEEP &&
 262                     got_object_idset_contains(keep, &qid->id)) {
 263                         got_object_qid_free(qid);
 264                         qid = NULL;
 265                         continue;
 266                 }
 267                 if (color == COLOR_DROP &&
 268                     got_object_idset_contains(drop, &qid->id)) {
 269                         got_object_qid_free(qid);
 270                         qid = NULL;
 271                         continue;
 272                 }
 273
 274                 switch (color) {
 275                 case COLOR_KEEP:
 276                         if (got_object_idset_contains(drop, &qid->id)) {
 277                                 err = got_pack_paint_commit(qid, COLOR_SKIP);
 278                                 if (err)
 279                                         goto done;
 280                                 err = got_object_idset_add(skip, &qid->id,
 281                                     NULL);
 282                                 if (err)
 283                                         goto done;
 284                                 err = got_pack_repaint_parent_commits(&qid->id,
 285                                     COLOR_SKIP, skip, skip, repo);
 286                                 if (err)
 287                                         goto done;
 288                         } else
 289                                 (*ncolored)++;
 290                         err = got_object_idset_add(keep, &qid->id, NULL);
 291                         if (err)
 292                                 goto done;
 293                         break;
 294                 case COLOR_DROP:
 295                         if (got_object_idset_contains(keep, &qid->id)) {
 296                                 err = got_pack_paint_commit(qid, COLOR_SKIP);
 297                                 if (err)
 298                                         goto done;
 299                                 err = got_object_idset_add(skip, &qid->id,
 300                                     NULL);
 301                                 if (err)
 302                                         goto done;
 303                                 err = got_pack_repaint_parent_commits(&qid->id,
 304                                     COLOR_SKIP, skip, skip, repo);
 305                                 if (err)
 306                                         goto done;
 307                         } else
 308                                 (*ncolored)++;
 309                         err = got_object_idset_add(drop, &qid->id, NULL);
 310                         if (err)
 311                                 goto done;
 312                         break;
 313                 case COLOR_SKIP:
 314                         if (!got_object_idset_contains(skip, &qid->id)) {
 315                                 err = got_object_idset_add(skip, &qid->id,
 316                                     NULL);
 317                                 if (err)
 318                                         goto done;
 319                         }
 320                         break;
 321                 default:
 322                         /* should not happen */
 323                         err = got_error_fmt(GOT_ERR_NOT_IMPL,
 324                             "%s invalid commit color %"PRIdPTR, __func__,
 325                             color);
 326                         goto done;
 327                 }
 328
 329                 err = got_pack_report_progress(progress_cb, progress_arg, rl,
 330                     *ncolored, 0, 0, 0L, 0, 0, 0, 0, 0);
 331                 if (err)
 332                         break;
 333
 334                 err = got_object_open_as_commit(&commit, repo, &qid->id);
 335                 if (err)
 336                         break;
 337
 338                 parents = got_object_commit_get_parent_ids(commit);
 339                 if (parents) {
 340                         struct got_object_qid *pid;
 341                         color = (intptr_t)qid->data;
 342                         STAILQ_FOREACH(pid, parents, entry) {
 343                                 err = got_pack_queue_commit_id(ids, &pid->id,
 344                                     color, repo);
 345                                 if (err)
 346                                         break;
 347                                 nqueued++;
 348                                 if (color == COLOR_SKIP)
 349                                         nskip++;
 350                         }
 351                 }
 352
 353                 if (pack == NULL && (commit->flags & GOT_COMMIT_FLAG_PACKED)) {
 354                         /*
 355                          * We now know that at least one pack file exists.
 356                          * Pin a suitable pack to ensure it remains cached
 357                          * while we are churning through commit history.
 358                          */
 359                         if (packidx == NULL) {
 360                                 err = got_pack_find_pack_for_commit_painting(
 361                                     &packidx, ids, nqueued, repo);
 362                                 if (err)
 363                                         goto done;
 364                         }
 365                         if (packidx != NULL) {
 366                                 err = got_pack_cache_pack_for_packidx(&pack,
 367                                     packidx, repo);
 368                                 if (err)
 369                                         goto done;
 370                                 err = got_repo_pin_pack(repo, packidx, pack);
 371                                 if (err)
 372                                         goto done;
 373                         }
 374                 }
 375
 376                 got_object_commit_close(commit);
 377                 commit = NULL;
 378
 379                 got_object_qid_free(qid);
 380                 qid = NULL;
 381         }
 382 done:
 383         if (commit)
 384                 got_object_commit_close(commit);
 385         got_object_qid_free(qid);
 386         got_repo_unpin_pack(repo);
 387         return err;
 388 }