lib/got_lib_pack.h

   1 /*
   2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 struct got_pack_privsep_child {
  18         int imsg_fd;
  19         pid_t pid;
  20         struct imsgbuf *ibuf;
  21 };
  22
  23 /* An open pack file. */
  24 struct got_pack {
  25         char *path_packfile;
  26         int fd;
  27         uint8_t *map;
  28         off_t filesize;
  29         struct got_pack_privsep_child *privsep_child;
  30         int basefd;
  31         int accumfd;
  32         int child_has_tempfiles;
  33         int child_has_delta_outfd;
  34         struct got_delta_cache *delta_cache;
  35 };
  36
  37 struct got_packidx;
  38
  39 const struct got_error *got_pack_start_privsep_child(struct got_pack *,
  40     struct got_packidx *);
  41 const struct got_error *got_pack_close(struct got_pack *);
  42
  43 const struct got_error *got_pack_parse_offset_delta(off_t *, size_t *,
  44     struct got_pack *, off_t, size_t);
  45 const struct got_error *got_pack_parse_ref_delta(struct got_object_id *,
  46     struct got_pack *, off_t, int);
  47 const struct got_error *got_pack_resolve_delta_chain(struct got_delta_chain *,
  48     struct got_packidx *, struct got_pack *, off_t, size_t, int, size_t,
  49     unsigned int);
  50 const struct got_error *got_pack_parse_object_type_and_size(uint8_t *,
  51     uint64_t *, size_t *, struct got_pack *, off_t);
  52
  53 #define GOT_PACK_PREFIX         "pack-"
  54 #define GOT_PACKFILE_SUFFIX     ".pack"
  55 #define GOT_PACKIDX_SUFFIX              ".idx"
  56 #define GOT_PACKFILE_NAMELEN    (strlen(GOT_PACK_PREFIX) + \
  57                                 SHA1_DIGEST_STRING_LENGTH - 1 + \
  58                                 strlen(GOT_PACKFILE_SUFFIX))
  59 #define GOT_PACKIDX_NAMELEN     (strlen(GOT_PACK_PREFIX) + \
  60                                 SHA1_DIGEST_STRING_LENGTH - 1 + \
  61                                 strlen(GOT_PACKIDX_SUFFIX))
  62
  63 /* See Documentation/technical/pack-format.txt in Git. */
  64
  65 struct got_packidx_trailer {
  66         u_int8_t        packfile_sha1[SHA1_DIGEST_LENGTH];
  67         u_int8_t        packidx_sha1[SHA1_DIGEST_LENGTH];
  68 } __attribute__((__packed__));
  69
  70 struct got_packidx_object_id {
  71         u_int8_t sha1[SHA1_DIGEST_LENGTH];
  72 } __attribute__((__packed__));
  73
  74 /* Ignore pack index version 1 which is no longer written by Git. */
  75 #define GOT_PACKIDX_VERSION 2
  76
  77 struct got_packidx_v2_hdr {
  78         uint32_t        *magic;         /* big endian */
  79 #define GOT_PACKIDX_V2_MAGIC 0xff744f63 /* "\377t0c" */
  80         uint32_t        *version;
  81
  82         /*
  83          * Each entry N in the fanout table contains the number of objects in
  84          * the packfile whose SHA1 begins with a byte less than or equal to N.
  85          * The last entry (index 255) contains the number of objects in the
  86          * pack file whose first SHA1 byte is <= 0xff, and thus records the
  87          * total number of objects in the pack file. All pointer variables
  88          * below point to tables with a corresponding number of entries.
  89          */
  90         uint32_t        *fanout_table;  /* values are big endian */
  91 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
  92
  93         /* Sorted SHA1 checksums for each object in the pack file. */
  94         struct got_packidx_object_id *sorted_ids;
  95
  96         /* CRC32 of the packed representation of each object. */
  97         uint32_t        *crc32;
  98
  99         /* Offset into the pack file for each object. */
 100         uint32_t        *offsets;               /* values are big endian */
 101 #define GOT_PACKIDX_OFFSET_VAL_MASK             0x7fffffff
 102 #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX     0x80000000
 103
 104         /* Large offsets table is empty for pack files < 2 GB. */
 105         uint64_t        *large_offsets;         /* values are big endian */
 106
 107         struct got_packidx_trailer *trailer;
 108 };
 109
 110 struct got_pack_offset_index {
 111         uint32_t offset;
 112         uint32_t idx;
 113 };
 114
 115 struct got_pack_large_offset_index {
 116         uint64_t offset;
 117         uint32_t idx;
 118 };
 119
 120 /* An open pack index file. */
 121 struct got_packidx {
 122         char *path_packidx; /* actual on-disk path */
 123         int fd;
 124         uint8_t *map;
 125         size_t len;
 126         size_t nlargeobj;
 127         struct got_packidx_v2_hdr hdr; /* convenient pointers into map */
 128         struct got_pack_offset_index *sorted_offsets;
 129         struct got_pack_large_offset_index *sorted_large_offsets;
 130 };
 131
 132 struct got_packfile_hdr {
 133         uint32_t        signature;
 134 #define GOT_PACKFILE_SIGNATURE  0x5041434b      /* 'P' 'A' 'C' 'K' */
 135         uint32_t        version;        /* big endian */
 136 #define GOT_PACKFILE_VERSION 2
 137         uint32_t        nobjects;       /* big endian */
 138 };
 139
 140 struct got_packfile_obj_hdr {
 141         /*
 142          * The object size field uses a variable length encoding:
 143          * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
 144          * least significant part and sizeN is the most significant part.
 145          * If the MSB of a size byte is set, an additional size byte follows.
 146          * Of the 7 remaining bits of size0, the first 3 bits indicate the
 147          * object's type, and the remaining 4 bits contribute to the size.
 148          */
 149         uint8_t *size;          /* variable length */
 150 #define GOT_PACK_OBJ_SIZE_MORE          0x80
 151 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK    0x70 /* See struct got_object->type */
 152 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT      4
 153 #define GOT_PACK_OBJ_SIZE0_VAL_MASK     0x0f
 154 #define GOT_PACK_OBJ_SIZE_VAL_MASK      0x7f
 155 };
 156
 157 /* If object is not a DELTA type. */
 158 struct got_packfile_object_data {
 159         uint8_t *data;  /* compressed */
 160 };
 161
 162 /* If object is of type GOT_OBJ_TYPE_REF_DELTA. */
 163 struct got_packfile_object_data_ref_delta {
 164         uint8_t sha1[SHA1_DIGEST_LENGTH];
 165         uint8_t *delta_data;            /* compressed */
 166 };
 167
 168 /* If object is of type GOT_OBJ_TYPE_OFFSET_DELTA. */
 169 struct got_packfile_object_data_offset_delta {
 170         /*
 171          * This offset is interpreted as a negative offset from
 172          * the got_packfile_obj_hdr corresponding to this object.
 173          * The size provided in the header specifies the amount
 174          * of compressed delta data that follows.
 175          *
 176          * This field uses a variable length encoding of N bytes,
 177          * where the MSB is always set except for the last byte.
 178          * The value is encoded as a series of N 7 bit integers,
 179          * which are concatenated, and if N > 1 the value 2^7 +
 180          * 2^14 + ... + 2^(7 * (n-1)) is added to the result.
 181          */
 182         uint8_t *offset;        /* variable length */
 183 #define GOT_PACK_OBJ_DELTA_OFF_MORE             0x80
 184 #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK         0x7f
 185         uint8_t *delta_data;            /* compressed */
 186 };
 187
 188 struct got_packfile_obj_data {
 189         union {
 190                 struct got_packfile_object_data data;
 191                 struct got_packfile_object_data_ref_delta ref_delta;
 192                 struct got_packfile_object_data_offset_delta offset_delta;
 193         } __attribute__((__packed__));
 194 } __attribute__((__packed__));
 195
 196 const struct got_error *got_packidx_init_hdr(struct got_packidx *, int, off_t);
 197 const struct got_error *got_packidx_open(struct got_packidx **,
 198     int, const char *, int);
 199 const struct got_error *got_packidx_close(struct got_packidx *);
 200 const struct got_error *got_packidx_get_packfile_path(char **, const char *);
 201 off_t got_packidx_get_object_offset(struct got_packidx *, int idx);
 202 int got_packidx_get_object_idx(struct got_packidx *, struct got_object_id *);
 203 const struct got_error *got_packidx_get_offset_idx(int *, struct got_packidx *,
 204     off_t);
 205 const struct got_error *got_packidx_get_object_id(struct got_object_id *,
 206     struct got_packidx *, int);
 207 const struct got_error *got_packidx_match_id_str_prefix(
 208     struct got_object_id_queue *, struct got_packidx *, const char *);
 209
 210 const struct got_error *got_packfile_open_object(struct got_object **,
 211     struct got_pack *, struct got_packidx *, int, struct got_object_id *);
 212 const struct got_error *got_pack_get_delta_chain_max_size(uint64_t *,
 213     struct got_delta_chain *, struct got_pack *);
 214 const struct got_error *got_pack_get_max_delta_object_size(uint64_t *,
 215     struct got_object *, struct got_pack *);
 216 const struct got_error *got_pack_dump_delta_chain_to_file(size_t *,
 217     struct got_delta_chain *, struct got_pack *, FILE *, FILE *, FILE *);
 218 const struct got_error *got_pack_dump_delta_chain_to_mem(uint8_t **, size_t *,
 219     struct got_delta_chain *, struct got_pack *);
 220 const struct got_error *got_packfile_extract_object(struct got_pack *,
 221     struct got_object *, FILE *, FILE *, FILE *);
 222 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
 223     struct got_object *, struct got_pack *);
 224 const struct got_error *got_packfile_extract_raw_delta(uint8_t **, size_t *,
 225     size_t *, off_t *, off_t *, off_t *, struct got_object_id *, uint64_t *,
 226     uint64_t *, struct got_pack *, struct got_packidx *, int);