lib/got_lib_pack.h

   1 /*
   2  * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
   3  *
   4  * Permission to use, copy, modify, and distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 struct got_pack_privsep_child {
  18         int imsg_fd;
  19         pid_t pid;
  20         struct imsgbuf *ibuf;
  21 };
  22
  23 /* An open pack file. */
  24 struct got_pack {
  25         char *path_packfile;
  26         int fd;
  27         enum got_hash_algorithm algo;
  28         uint8_t *map;
  29         off_t filesize;
  30         struct got_pack_privsep_child *privsep_child;
  31         int basefd;
  32         int accumfd;
  33         int child_has_tempfiles;
  34         int child_has_delta_outfd;
  35         struct got_delta_cache *delta_cache;
  36 };
  37
  38 struct got_packidx;
  39
  40 const struct got_error *got_pack_start_privsep_child(struct got_pack *,
  41     struct got_packidx *);
  42 const struct got_error *got_pack_close(struct got_pack *);
  43
  44 const struct got_error *got_pack_parse_offset_delta(off_t *, size_t *,
  45     struct got_pack *, off_t, size_t);
  46 const struct got_error *got_pack_parse_ref_delta(struct got_object_id *,
  47     struct got_pack *, off_t, int);
  48 const struct got_error *got_pack_resolve_delta_chain(struct got_delta_chain *,
  49     struct got_packidx *, struct got_pack *, off_t, size_t, int, size_t,
  50     unsigned int);
  51 const struct got_error *got_pack_parse_object_type_and_size(uint8_t *,
  52     uint64_t *, size_t *, struct got_pack *, off_t);
  53
  54 #define GOT_PACK_PREFIX         "pack-"
  55 #define GOT_PACKFILE_SUFFIX     ".pack"
  56 #define GOT_PACKIDX_SUFFIX      ".idx"
  57 #define GOT_PACKFILE_NAMELEN    (strlen(GOT_PACK_PREFIX) + \
  58                                 SHA1_DIGEST_STRING_LENGTH - 1 + \
  59                                 strlen(GOT_PACKFILE_SUFFIX))
  60 #define GOT_PACKIDX_NAMELEN(digest_len) \
  61                                 (strlen(GOT_PACK_PREFIX) + \
  62                                 digest_len - 1 + strlen(GOT_PACKIDX_SUFFIX))
  63
  64 /* See Documentation/technical/pack-format.txt in Git. */
  65
  66 struct got_packidx_trailer {
  67         u_int8_t        packfile_hash[GOT_HASH_DIGEST_MAXLEN];
  68         u_int8_t        packidx_hash[GOT_HASH_DIGEST_MAXLEN];
  69 } __attribute__((__packed__));
  70
  71 /* Ignore pack index version 1 which is no longer written by Git. */
  72 #define GOT_PACKIDX_VERSION 2
  73
  74 struct got_packidx_v2_hdr {
  75         uint32_t        *magic;         /* big endian */
  76 #define GOT_PACKIDX_V2_MAGIC 0xff744f63 /* "\377t0c" */
  77         uint32_t        *version;
  78
  79         /*
  80          * Each entry N in the fanout table contains the number of objects in
  81          * the packfile whose hash begins with a byte less than or equal to N.
  82          * The last entry (index 255) contains the number of objects in the
  83          * pack file whose first hash byte is <= 0xff, and thus records the
  84          * total number of objects in the pack file. All pointer variables
  85          * below point to tables with a corresponding number of entries.
  86          */
  87         uint32_t        *fanout_table;  /* values are big endian */
  88 #define GOT_PACKIDX_V2_FANOUT_TABLE_ITEMS (0xff + 1)
  89
  90         /* Sorted checksums for each object in the pack file. */
  91         uint8_t *sorted_ids;
  92
  93         /* CRC32 of the packed representation of each object. */
  94         uint32_t        *crc32;
  95
  96         /* Offset into the pack file for each object. */
  97         uint32_t        *offsets;               /* values are big endian */
  98 #define GOT_PACKIDX_OFFSET_VAL_MASK             0x7fffffff
  99 #define GOT_PACKIDX_OFFSET_VAL_IS_LARGE_IDX     0x80000000
 100
 101         /* Large offsets table is empty for pack files < 2 GB. */
 102         uint64_t        *large_offsets;         /* values are big endian */
 103
 104         struct got_packidx_trailer trailer;
 105 };
 106
 107 struct got_pack_offset_index {
 108         uint32_t offset;
 109         uint32_t idx;
 110 };
 111
 112 struct got_pack_large_offset_index {
 113         uint64_t offset;
 114         uint32_t idx;
 115 };
 116
 117 /* An open pack index file. */
 118 struct got_packidx {
 119         char *path_packidx; /* actual on-disk path */
 120         int fd;
 121         enum got_hash_algorithm algo;
 122         uint8_t *map;
 123         size_t len;
 124         size_t nlargeobj;
 125         struct got_packidx_v2_hdr hdr; /* convenient pointers into map */
 126         struct got_pack_offset_index *sorted_offsets;
 127         struct got_pack_large_offset_index *sorted_large_offsets;
 128 };
 129
 130 struct got_packfile_hdr {
 131         uint32_t        signature;
 132 #define GOT_PACKFILE_SIGNATURE  0x5041434b      /* 'P' 'A' 'C' 'K' */
 133         uint32_t        version;        /* big endian */
 134 #define GOT_PACKFILE_VERSION 2
 135         uint32_t        nobjects;       /* big endian */
 136 };
 137
 138 struct got_packfile_obj_hdr {
 139         /*
 140          * The object size field uses a variable length encoding:
 141          * size0...sizeN form a 4+7+7+...+7 bit integer, where size0 is the
 142          * least significant part and sizeN is the most significant part.
 143          * If the MSB of a size byte is set, an additional size byte follows.
 144          * Of the 7 remaining bits of size0, the first 3 bits indicate the
 145          * object's type, and the remaining 4 bits contribute to the size.
 146          */
 147         uint8_t *size;          /* variable length */
 148 #define GOT_PACK_OBJ_SIZE_MORE          0x80
 149 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK    0x70 /* See struct got_object->type */
 150 #define GOT_PACK_OBJ_SIZE0_TYPE_MASK_SHIFT      4
 151 #define GOT_PACK_OBJ_SIZE0_VAL_MASK     0x0f
 152 #define GOT_PACK_OBJ_SIZE_VAL_MASK      0x7f
 153 };
 154
 155 #define GOT_PACK_OBJ_DELTA_OFF_MORE             0x80
 156 #define GOT_PACK_OBJ_DELTA_OFF_VAL_MASK         0x7f
 157
 158 const struct got_error *got_packidx_init_hdr(struct got_packidx *, int, off_t);
 159 const struct got_error *got_packidx_open(struct got_packidx **,
 160     int, const char *, int, enum got_hash_algorithm);
 161 const struct got_error *got_packidx_close(struct got_packidx *);
 162 const struct got_error *got_packidx_get_packfile_path(char **, const char *);
 163 off_t got_packidx_get_object_offset(struct got_packidx *, int idx);
 164 int got_packidx_get_object_idx(struct got_packidx *, struct got_object_id *);
 165 const struct got_error *got_packidx_get_offset_idx(int *, struct got_packidx *,
 166     off_t);
 167 const struct got_error *got_packidx_get_object_id(struct got_object_id *,
 168     struct got_packidx *, int);
 169 const struct got_error *got_packidx_match_id_str_prefix(
 170     struct got_object_id_queue *, struct got_packidx *, const char *);
 171
 172 const struct got_error *got_packfile_open_object(struct got_object **,
 173     struct got_pack *, struct got_packidx *, int, struct got_object_id *);
 174 const struct got_error *got_pack_get_delta_chain_max_size(uint64_t *,
 175     struct got_delta_chain *, struct got_pack *);
 176 const struct got_error *got_pack_get_max_delta_object_size(uint64_t *,
 177     struct got_object *, struct got_pack *);
 178 const struct got_error *got_pack_dump_delta_chain_to_file(size_t *,
 179     struct got_delta_chain *, struct got_pack *, FILE *, FILE *, FILE *);
 180 const struct got_error *got_pack_dump_delta_chain_to_mem(uint8_t **, size_t *,
 181     struct got_delta_chain *, struct got_pack *);
 182 const struct got_error *got_packfile_extract_object(struct got_pack *,
 183     struct got_object *, FILE *, FILE *, FILE *);
 184 const struct got_error *got_packfile_extract_object_to_mem(uint8_t **, size_t *,
 185     struct got_object *, struct got_pack *);
 186 const struct got_error *got_packfile_extract_raw_delta(uint8_t **, size_t *,
 187     size_t *, off_t *, off_t *, off_t *, struct got_object_id *, uint64_t *,
 188     uint64_t *, struct got_pack *, struct got_packidx *, int);