pack-write.c

   1 #include "cache.h"
   2 #include "pack.h"
   3 #include "csum-file.h"
   4
   5 uint32_t pack_idx_default_version = 1;
   6 uint32_t pack_idx_off32_limit = 0x7fffffff;
   7
   8 static int sha1_compare(const void *_a, const void *_b)
   9 {
  10         struct pack_idx_entry *a = *(struct pack_idx_entry **)_a;
  11         struct pack_idx_entry *b = *(struct pack_idx_entry **)_b;
  12         return hashcmp(a->sha1, b->sha1);
  13 }
  14
  15 /*
  16  * On entry *sha1 contains the pack content SHA1 hash, on exit it is
  17  * the SHA1 hash of sorted object names. The objects array passed in
  18  * will be sorted by SHA1 on exit.
  19  */
  20 const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
  21 {
  22         struct sha1file *f;
  23         struct pack_idx_entry **sorted_by_sha, **list, **last;
  24         off_t last_obj_offset = 0;
  25         uint32_t array[256];
  26         int i, fd;
  27         SHA_CTX ctx;
  28         uint32_t index_version;
  29
  30         if (nr_objects) {
  31                 sorted_by_sha = objects;
  32                 list = sorted_by_sha;
  33                 last = sorted_by_sha + nr_objects;
  34                 for (i = 0; i < nr_objects; ++i) {
  35                         if (objects[i]->offset > last_obj_offset)
  36                                 last_obj_offset = objects[i]->offset;
  37                 }
  38                 qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
  39                       sha1_compare);
  40         }
  41         else
  42                 sorted_by_sha = list = last = NULL;
  43
  44         if (!index_name) {
  45                 static char tmpfile[PATH_MAX];
  46                 snprintf(tmpfile, sizeof(tmpfile),
  47                          "%s/tmp_idx_XXXXXX", get_object_directory());
  48                 fd = xmkstemp(tmpfile);
  49                 index_name = xstrdup(tmpfile);
  50         } else {
  51                 unlink(index_name);
  52                 fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
  53         }
  54         if (fd < 0)
  55                 die("unable to create %s: %s", index_name, strerror(errno));
  56         f = sha1fd(fd, index_name);
  57
  58         /* if last object's offset is >= 2^31 we should use index V3 */
  59         if (pack_idx_default_version != 1)
  60                 index_version = pack_idx_default_version;
  61         else
  62                 index_version =
  63                 (last_obj_offset >> 31) ? 3 : pack_idx_default_version;
  64
  65         /* index versions 2 and above need a header */
  66         if (index_version >= 2) {
  67                 struct pack_idx_header hdr;
  68                 hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
  69                 hdr.idx_version = htonl(index_version);
  70                 sha1write(f, &hdr, sizeof(hdr));
  71         }
  72
  73         /*
  74          * Write the first-level table (the list is sorted,
  75          * but we use a 256-entry lookup to be able to avoid
  76          * having to do eight extra binary search iterations).
  77          */
  78         for (i = 0; i < 256; i++) {
  79                 struct pack_idx_entry **next = list;
  80                 while (next < last) {
  81                         struct pack_idx_entry *obj = *next;
  82                         if (obj->sha1[0] != i)
  83                                 break;
  84                         next++;
  85                 }
  86                 array[i] = htonl(next - sorted_by_sha);
  87                 list = next;
  88         }
  89         sha1write(f, array, 256 * 4);
  90
  91         /* compute the SHA1 hash of sorted object names. */
  92         SHA1_Init(&ctx);
  93
  94         /*
  95          * Write the actual SHA1 entries..
  96          */
  97         list = sorted_by_sha;
  98         for (i = 0; i < nr_objects; i++) {
  99                 struct pack_idx_entry *obj = *list++;
 100                 if (index_version < 2) {
 101                         uint32_t offset = htonl(obj->offset);
 102                         sha1write(f, &offset, 4);
 103                 }
 104                 if (index_version < 3)
 105                         sha1write(f, obj->sha1, 20);
 106                 SHA1_Update(&ctx, obj->sha1, 20);
 107         }
 108
 109         if (index_version >= 2) {
 110                 unsigned int nr_large_offset = 0;
 111
 112                 /* write the crc32 table */
 113                 list = sorted_by_sha;
 114                 for (i = 0; i < nr_objects; i++) {
 115                         struct pack_idx_entry *obj = *list++;
 116                         uint32_t crc32_val = htonl(obj->crc32);
 117                         sha1write(f, &crc32_val, 4);
 118                 }
 119
 120                 /* write the 32-bit offset table */
 121                 list = sorted_by_sha;
 122                 for (i = 0; i < nr_objects; i++) {
 123                         struct pack_idx_entry *obj = *list++;
 124                         uint32_t offset = (obj->offset <= pack_idx_off32_limit) ?
 125                                 obj->offset : (0x80000000 | nr_large_offset++);
 126                         offset = htonl(offset);
 127                         sha1write(f, &offset, 4);
 128                 }
 129
 130                 /* write the large offset table */
 131                 list = sorted_by_sha;
 132                 while (nr_large_offset) {
 133                         struct pack_idx_entry *obj = *list++;
 134                         uint64_t offset = obj->offset;
 135                         if (offset > pack_idx_off32_limit) {
 136                                 uint32_t split[2];
 137                                 split[0] = htonl(offset >> 32);
 138                                 split[1] = htonl(offset & 0xffffffff);
 139                                 sha1write(f, split, 8);
 140                                 nr_large_offset--;
 141                         }
 142                 }
 143         }
 144
 145         sha1write(f, sha1, 20);
 146         sha1close(f, NULL, 1);
 147         SHA1_Final(sha1, &ctx);
 148         return index_name;
 149 }
 150
 151 void fixup_pack_header_footer(int pack_fd,
 152                          unsigned char *pack_file_sha1,
 153                          const char *pack_name,
 154                          uint32_t object_count)
 155 {
 156         static const int buf_sz = 128 * 1024;
 157         SHA_CTX c;
 158         struct pack_header hdr;
 159         char *buf;
 160
 161         if (lseek(pack_fd, 0, SEEK_SET) != 0)
 162                 die("Failed seeking to start: %s", strerror(errno));
 163         if (read_in_full(pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
 164                 die("Unable to reread header of %s: %s", pack_name, strerror(errno));
 165         if (lseek(pack_fd, 0, SEEK_SET) != 0)
 166                 die("Failed seeking to start: %s", strerror(errno));
 167         hdr.hdr_entries = htonl(object_count);
 168         write_or_die(pack_fd, &hdr, sizeof(hdr));
 169
 170         SHA1_Init(&c);
 171         SHA1_Update(&c, &hdr, sizeof(hdr));
 172
 173         buf = xmalloc(buf_sz);
 174         for (;;) {
 175                 ssize_t n = xread(pack_fd, buf, buf_sz);
 176                 if (!n)
 177                         break;
 178                 if (n < 0)
 179                         die("Failed to checksum %s: %s", pack_name, strerror(errno));
 180                 SHA1_Update(&c, buf, n);
 181         }
 182         free(buf);
 183
 184         SHA1_Final(pack_file_sha1, &c);
 185         write_or_die(pack_fd, pack_file_sha1, 20);
 186 }