refspec: store raw refspecs inside refspec_item
[git/gitster.git] / reftable / reader.c
blob6494ce2e32705e391ab1a1077dba07112cedf928
1 /*
2 Copyright 2020 Google LLC
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file or at
6 https://developers.google.com/open-source/licenses/bsd
7 */
9 #include "reader.h"
11 #include "system.h"
12 #include "block.h"
13 #include "constants.h"
14 #include "iter.h"
15 #include "record.h"
16 #include "reftable-error.h"
18 uint64_t block_source_size(struct reftable_block_source *source)
20 return source->ops->size(source->arg);
23 int block_source_read_block(struct reftable_block_source *source,
24 struct reftable_block *dest, uint64_t off,
25 uint32_t size)
27 int result = source->ops->read_block(source->arg, dest, off, size);
28 dest->source = *source;
29 return result;
32 void block_source_close(struct reftable_block_source *source)
34 if (!source->ops) {
35 return;
38 source->ops->close(source->arg);
39 source->ops = NULL;
42 static struct reftable_reader_offsets *
43 reader_offsets_for(struct reftable_reader *r, uint8_t typ)
45 switch (typ) {
46 case BLOCK_TYPE_REF:
47 return &r->ref_offsets;
48 case BLOCK_TYPE_LOG:
49 return &r->log_offsets;
50 case BLOCK_TYPE_OBJ:
51 return &r->obj_offsets;
53 abort();
56 static int reader_get_block(struct reftable_reader *r,
57 struct reftable_block *dest, uint64_t off,
58 uint32_t sz)
60 if (off >= r->size)
61 return 0;
63 if (off + sz > r->size) {
64 sz = r->size - off;
67 return block_source_read_block(&r->source, dest, off, sz);
70 uint32_t reftable_reader_hash_id(struct reftable_reader *r)
72 return r->hash_id;
75 const char *reader_name(struct reftable_reader *r)
77 return r->name;
80 static int parse_footer(struct reftable_reader *r, uint8_t *footer,
81 uint8_t *header)
83 uint8_t *f = footer;
84 uint8_t first_block_typ;
85 int err = 0;
86 uint32_t computed_crc;
87 uint32_t file_crc;
89 if (memcmp(f, "REFT", 4)) {
90 err = REFTABLE_FORMAT_ERROR;
91 goto done;
93 f += 4;
95 if (memcmp(footer, header, header_size(r->version))) {
96 err = REFTABLE_FORMAT_ERROR;
97 goto done;
100 f++;
101 r->block_size = get_be24(f);
103 f += 3;
104 r->min_update_index = get_be64(f);
105 f += 8;
106 r->max_update_index = get_be64(f);
107 f += 8;
109 if (r->version == 1) {
110 r->hash_id = GIT_SHA1_FORMAT_ID;
111 } else {
112 r->hash_id = get_be32(f);
113 switch (r->hash_id) {
114 case GIT_SHA1_FORMAT_ID:
115 break;
116 case GIT_SHA256_FORMAT_ID:
117 break;
118 default:
119 err = REFTABLE_FORMAT_ERROR;
120 goto done;
122 f += 4;
125 r->ref_offsets.index_offset = get_be64(f);
126 f += 8;
128 r->obj_offsets.offset = get_be64(f);
129 f += 8;
131 r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
132 r->obj_offsets.offset >>= 5;
134 r->obj_offsets.index_offset = get_be64(f);
135 f += 8;
136 r->log_offsets.offset = get_be64(f);
137 f += 8;
138 r->log_offsets.index_offset = get_be64(f);
139 f += 8;
141 computed_crc = crc32(0, footer, f - footer);
142 file_crc = get_be32(f);
143 f += 4;
144 if (computed_crc != file_crc) {
145 err = REFTABLE_FORMAT_ERROR;
146 goto done;
149 first_block_typ = header[header_size(r->version)];
150 r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF);
151 r->ref_offsets.offset = 0;
152 r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG ||
153 r->log_offsets.offset > 0);
154 r->obj_offsets.is_present = r->obj_offsets.offset > 0;
155 if (r->obj_offsets.is_present && !r->object_id_len) {
156 err = REFTABLE_FORMAT_ERROR;
157 goto done;
160 err = 0;
161 done:
162 return err;
165 struct table_iter {
166 struct reftable_reader *r;
167 uint8_t typ;
168 uint64_t block_off;
169 struct block_reader br;
170 struct block_iter bi;
171 int is_finished;
174 static int table_iter_init(struct table_iter *ti, struct reftable_reader *r)
176 struct block_iter bi = BLOCK_ITER_INIT;
177 memset(ti, 0, sizeof(*ti));
178 reftable_reader_incref(r);
179 ti->r = r;
180 ti->bi = bi;
181 return 0;
184 static int table_iter_next_in_block(struct table_iter *ti,
185 struct reftable_record *rec)
187 int res = block_iter_next(&ti->bi, rec);
188 if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
189 rec->u.ref.update_index += ti->r->min_update_index;
192 return res;
195 static void table_iter_block_done(struct table_iter *ti)
197 block_reader_release(&ti->br);
198 block_iter_reset(&ti->bi);
201 static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off,
202 int version)
204 int32_t result = 0;
206 if (off == 0) {
207 data += header_size(version);
210 *typ = data[0];
211 if (reftable_is_block_type(*typ)) {
212 result = get_be24(data + 1);
214 return result;
217 int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
218 uint64_t next_off, uint8_t want_typ)
220 int32_t guess_block_size = r->block_size ? r->block_size :
221 DEFAULT_BLOCK_SIZE;
222 struct reftable_block block = { NULL };
223 uint8_t block_typ = 0;
224 int err = 0;
225 uint32_t header_off = next_off ? 0 : header_size(r->version);
226 int32_t block_size = 0;
228 if (next_off >= r->size)
229 return 1;
231 err = reader_get_block(r, &block, next_off, guess_block_size);
232 if (err < 0)
233 goto done;
235 block_size = extract_block_size(block.data, &block_typ, next_off,
236 r->version);
237 if (block_size < 0) {
238 err = block_size;
239 goto done;
241 if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
242 err = 1;
243 goto done;
246 if (block_size > guess_block_size) {
247 reftable_block_done(&block);
248 err = reader_get_block(r, &block, next_off, block_size);
249 if (err < 0) {
250 goto done;
254 err = block_reader_init(br, &block, header_off, r->block_size,
255 hash_size(r->hash_id));
256 done:
257 reftable_block_done(&block);
259 return err;
262 static void table_iter_close(struct table_iter *ti)
264 table_iter_block_done(ti);
265 block_iter_close(&ti->bi);
266 reftable_reader_decref(ti->r);
269 static int table_iter_next_block(struct table_iter *ti)
271 uint64_t next_block_off = ti->block_off + ti->br.full_block_size;
272 int err;
274 err = reader_init_block_reader(ti->r, &ti->br, next_block_off, ti->typ);
275 if (err > 0)
276 ti->is_finished = 1;
277 if (err)
278 return err;
280 ti->block_off = next_block_off;
281 ti->is_finished = 0;
282 block_iter_seek_start(&ti->bi, &ti->br);
284 return 0;
287 static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
289 if (reftable_record_type(rec) != ti->typ)
290 return REFTABLE_API_ERROR;
292 while (1) {
293 int err;
295 if (ti->is_finished)
296 return 1;
299 * Check whether the current block still has more records. If
300 * so, return it. If the iterator returns positive then the
301 * current block has been exhausted.
303 err = table_iter_next_in_block(ti, rec);
304 if (err <= 0)
305 return err;
308 * Otherwise, we need to continue to the next block in the
309 * table and retry. If there are no more blocks then the
310 * iterator is drained.
312 err = table_iter_next_block(ti);
313 if (err) {
314 ti->is_finished = 1;
315 return err;
320 static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ)
322 int err;
324 err = reader_init_block_reader(ti->r, &ti->br, off, typ);
325 if (err != 0)
326 return err;
328 ti->typ = block_reader_type(&ti->br);
329 ti->block_off = off;
330 block_iter_seek_start(&ti->bi, &ti->br);
331 ti->is_finished = 0;
332 return 0;
335 static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index)
337 struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
338 uint64_t off = offs->offset;
339 if (index) {
340 off = offs->index_offset;
341 if (off == 0) {
342 return 1;
344 typ = BLOCK_TYPE_INDEX;
347 return table_iter_seek_to(ti, off, typ);
350 static int table_iter_seek_linear(struct table_iter *ti,
351 struct reftable_record *want)
353 struct strbuf want_key = STRBUF_INIT;
354 struct strbuf got_key = STRBUF_INIT;
355 struct reftable_record rec;
356 int err;
358 reftable_record_init(&rec, reftable_record_type(want));
359 reftable_record_key(want, &want_key);
362 * First we need to locate the block that must contain our record. To
363 * do so we scan through blocks linearly until we find the first block
364 * whose first key is bigger than our wanted key. Once we have found
365 * that block we know that the key must be contained in the preceding
366 * block.
368 * This algorithm is somewhat unfortunate because it means that we
369 * always have to seek one block too far and then back up. But as we
370 * can only decode the _first_ key of a block but not its _last_ key we
371 * have no other way to do this.
373 while (1) {
374 struct table_iter next = *ti;
377 * We must be careful to not modify underlying data of `ti`
378 * because we may find that `next` does not contain our desired
379 * block, but that `ti` does. In that case, we would discard
380 * `next` and continue with `ti`.
382 * This also means that we cannot reuse allocated memory for
383 * `next` here. While it would be great if we could, it should
384 * in practice not be too bad given that we should only ever
385 * end up doing linear seeks with at most three blocks. As soon
386 * as we have more than three blocks we would have an index, so
387 * we would not do a linear search there anymore.
389 memset(&next.br.block, 0, sizeof(next.br.block));
390 next.br.zstream = NULL;
391 next.br.uncompressed_data = NULL;
392 next.br.uncompressed_cap = 0;
394 err = table_iter_next_block(&next);
395 if (err < 0)
396 goto done;
397 if (err > 0)
398 break;
400 err = block_reader_first_key(&next.br, &got_key);
401 if (err < 0)
402 goto done;
404 if (strbuf_cmp(&got_key, &want_key) > 0) {
405 table_iter_block_done(&next);
406 break;
409 table_iter_block_done(ti);
410 *ti = next;
414 * We have located the block that must contain our record, so we seek
415 * the wanted key inside of it. If the block does not contain our key
416 * we know that the corresponding record does not exist.
418 err = block_iter_seek_key(&ti->bi, &ti->br, &want_key);
419 if (err < 0)
420 goto done;
421 err = 0;
423 done:
424 reftable_record_release(&rec);
425 strbuf_release(&want_key);
426 strbuf_release(&got_key);
427 return err;
430 static int table_iter_seek_indexed(struct table_iter *ti,
431 struct reftable_record *rec)
433 struct reftable_record want_index = {
434 .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = STRBUF_INIT }
436 struct reftable_record index_result = {
437 .type = BLOCK_TYPE_INDEX,
438 .u.idx = { .last_key = STRBUF_INIT },
440 int err;
442 reftable_record_key(rec, &want_index.u.idx.last_key);
445 * The index may consist of multiple levels, where each level may have
446 * multiple index blocks. We start by doing a linear search in the
447 * highest layer that identifies the relevant index block as well as
448 * the record inside that block that corresponds to our wanted key.
450 err = table_iter_seek_linear(ti, &want_index);
451 if (err < 0)
452 goto done;
455 * Traverse down the levels until we find a non-index entry.
457 while (1) {
459 * In case we seek a record that does not exist the index iter
460 * will tell us that the iterator is over. This works because
461 * the last index entry of the current level will contain the
462 * last key it knows about. So in case our seeked key is larger
463 * than the last indexed key we know that it won't exist.
465 * There is one subtlety in the layout of the index section
466 * that makes this work as expected: the highest-level index is
467 * at end of the section and will point backwards and thus we
468 * start reading from the end of the index section, not the
469 * beginning.
471 * If that wasn't the case and the order was reversed then the
472 * linear seek would seek into the lower levels and traverse
473 * all levels of the index only to find out that the key does
474 * not exist.
476 err = table_iter_next(ti, &index_result);
477 if (err != 0)
478 goto done;
480 err = table_iter_seek_to(ti, index_result.u.idx.offset, 0);
481 if (err != 0)
482 goto done;
484 err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key);
485 if (err < 0)
486 goto done;
488 if (ti->typ == reftable_record_type(rec)) {
489 err = 0;
490 break;
493 if (ti->typ != BLOCK_TYPE_INDEX) {
494 err = REFTABLE_FORMAT_ERROR;
495 goto done;
499 done:
500 reftable_record_release(&want_index);
501 reftable_record_release(&index_result);
502 return err;
505 static int table_iter_seek(struct table_iter *ti,
506 struct reftable_record *want)
508 uint8_t typ = reftable_record_type(want);
509 struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
510 int err;
512 err = table_iter_seek_start(ti, reftable_record_type(want),
513 !!offs->index_offset);
514 if (err < 0)
515 goto out;
517 if (offs->index_offset)
518 err = table_iter_seek_indexed(ti, want);
519 else
520 err = table_iter_seek_linear(ti, want);
521 if (err)
522 goto out;
524 out:
525 return err;
528 static int table_iter_seek_void(void *ti, struct reftable_record *want)
530 return table_iter_seek(ti, want);
533 static int table_iter_next_void(void *ti, struct reftable_record *rec)
535 return table_iter_next(ti, rec);
538 static void table_iter_close_void(void *ti)
540 table_iter_close(ti);
543 static struct reftable_iterator_vtable table_iter_vtable = {
544 .seek = &table_iter_seek_void,
545 .next = &table_iter_next_void,
546 .close = &table_iter_close_void,
549 static void iterator_from_table_iter(struct reftable_iterator *it,
550 struct table_iter *ti)
552 assert(!it->ops);
553 it->iter_arg = ti;
554 it->ops = &table_iter_vtable;
557 void reader_init_iter(struct reftable_reader *r,
558 struct reftable_iterator *it,
559 uint8_t typ)
561 struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
563 if (offs->is_present) {
564 struct table_iter *ti;
565 REFTABLE_ALLOC_ARRAY(ti, 1);
566 table_iter_init(ti, r);
567 iterator_from_table_iter(it, ti);
568 } else {
569 iterator_set_empty(it);
573 void reftable_reader_init_ref_iterator(struct reftable_reader *r,
574 struct reftable_iterator *it)
576 reader_init_iter(r, it, BLOCK_TYPE_REF);
579 void reftable_reader_init_log_iterator(struct reftable_reader *r,
580 struct reftable_iterator *it)
582 reader_init_iter(r, it, BLOCK_TYPE_LOG);
585 int reftable_reader_new(struct reftable_reader **out,
586 struct reftable_block_source *source, char const *name)
588 struct reftable_block footer = { 0 };
589 struct reftable_block header = { 0 };
590 struct reftable_reader *r;
591 uint64_t file_size = block_source_size(source);
592 uint32_t read_size;
593 int err;
595 REFTABLE_CALLOC_ARRAY(r, 1);
598 * We need one extra byte to read the type of first block. We also
599 * pretend to always be reading v2 of the format because it is larger.
601 read_size = header_size(2) + 1;
602 if (read_size > file_size) {
603 err = REFTABLE_FORMAT_ERROR;
604 goto done;
607 err = block_source_read_block(source, &header, 0, read_size);
608 if (err != read_size) {
609 err = REFTABLE_IO_ERROR;
610 goto done;
613 if (memcmp(header.data, "REFT", 4)) {
614 err = REFTABLE_FORMAT_ERROR;
615 goto done;
617 r->version = header.data[4];
618 if (r->version != 1 && r->version != 2) {
619 err = REFTABLE_FORMAT_ERROR;
620 goto done;
623 r->size = file_size - footer_size(r->version);
624 r->source = *source;
625 r->name = xstrdup(name);
626 r->hash_id = 0;
627 r->refcount = 1;
629 err = block_source_read_block(source, &footer, r->size,
630 footer_size(r->version));
631 if (err != footer_size(r->version)) {
632 err = REFTABLE_IO_ERROR;
633 goto done;
636 err = parse_footer(r, footer.data, header.data);
637 if (err)
638 goto done;
640 *out = r;
642 done:
643 reftable_block_done(&footer);
644 reftable_block_done(&header);
645 if (err) {
646 reftable_free(r);
647 block_source_close(source);
649 return err;
652 void reftable_reader_incref(struct reftable_reader *r)
654 if (!r->refcount)
655 BUG("cannot increment ref counter of dead reader");
656 r->refcount++;
659 void reftable_reader_decref(struct reftable_reader *r)
661 if (!r)
662 return;
663 if (!r->refcount)
664 BUG("cannot decrement ref counter of dead reader");
665 if (--r->refcount)
666 return;
667 block_source_close(&r->source);
668 FREE_AND_NULL(r->name);
669 reftable_free(r);
672 static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
673 struct reftable_iterator *it,
674 uint8_t *oid)
676 struct reftable_record want = {
677 .type = BLOCK_TYPE_OBJ,
678 .u.obj = {
679 .hash_prefix = oid,
680 .hash_prefix_len = r->object_id_len,
683 struct reftable_iterator oit = { NULL };
684 struct reftable_record got = {
685 .type = BLOCK_TYPE_OBJ,
686 .u.obj = { 0 },
688 int err = 0;
689 struct indexed_table_ref_iter *itr = NULL;
691 /* Look through the reverse index. */
692 reader_init_iter(r, &oit, BLOCK_TYPE_OBJ);
693 err = iterator_seek(&oit, &want);
694 if (err != 0)
695 goto done;
697 /* read out the reftable_obj_record */
698 err = iterator_next(&oit, &got);
699 if (err < 0)
700 goto done;
702 if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix,
703 r->object_id_len)) {
704 /* didn't find it; return empty iterator */
705 iterator_set_empty(it);
706 err = 0;
707 goto done;
710 err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id),
711 got.u.obj.offsets,
712 got.u.obj.offset_len);
713 if (err < 0)
714 goto done;
715 got.u.obj.offsets = NULL;
716 iterator_from_indexed_table_ref_iter(it, itr);
718 done:
719 reftable_iterator_destroy(&oit);
720 reftable_record_release(&got);
721 return err;
724 static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
725 struct reftable_iterator *it,
726 uint8_t *oid)
728 struct table_iter *ti;
729 struct filtering_ref_iterator *filter = NULL;
730 struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
731 int oid_len = hash_size(r->hash_id);
732 int err;
734 REFTABLE_ALLOC_ARRAY(ti, 1);
735 table_iter_init(ti, r);
736 err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0);
737 if (err < 0) {
738 reftable_free(ti);
739 return err;
742 filter = reftable_malloc(sizeof(struct filtering_ref_iterator));
743 *filter = empty;
745 strbuf_add(&filter->oid, oid, oid_len);
746 iterator_from_table_iter(&filter->it, ti);
748 iterator_from_filtering_ref_iterator(it, filter);
749 return 0;
752 int reftable_reader_refs_for(struct reftable_reader *r,
753 struct reftable_iterator *it, uint8_t *oid)
755 if (r->obj_offsets.is_present)
756 return reftable_reader_refs_for_indexed(r, it, oid);
757 return reftable_reader_refs_for_unindexed(r, it, oid);
760 uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
762 return r->max_update_index;
765 uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
767 return r->min_update_index;
770 int reftable_reader_print_blocks(const char *tablename)
772 struct {
773 const char *name;
774 int type;
775 } sections[] = {
777 .name = "ref",
778 .type = BLOCK_TYPE_REF,
781 .name = "obj",
782 .type = BLOCK_TYPE_OBJ,
785 .name = "log",
786 .type = BLOCK_TYPE_LOG,
789 struct reftable_block_source src = { 0 };
790 struct reftable_reader *r = NULL;
791 struct table_iter ti = { 0 };
792 size_t i;
793 int err;
795 err = reftable_block_source_from_file(&src, tablename);
796 if (err < 0)
797 goto done;
799 err = reftable_reader_new(&r, &src, tablename);
800 if (err < 0)
801 goto done;
803 table_iter_init(&ti, r);
805 printf("header:\n");
806 printf(" block_size: %d\n", r->block_size);
808 for (i = 0; i < ARRAY_SIZE(sections); i++) {
809 err = table_iter_seek_start(&ti, sections[i].type, 0);
810 if (err < 0)
811 goto done;
812 if (err > 0)
813 continue;
815 printf("%s:\n", sections[i].name);
817 while (1) {
818 printf(" - length: %u\n", ti.br.block_len);
819 printf(" restarts: %u\n", ti.br.restart_count);
821 err = table_iter_next_block(&ti);
822 if (err < 0)
823 goto done;
824 if (err > 0)
825 break;
829 done:
830 reftable_reader_decref(r);
831 table_iter_close(&ti);
832 return err;