reftable/reader: rename `reftable_new_reader()`
[git/gitster.git] / reftable / reader.c
blobea4fdea90b675de0d84b6404bf88dd304cfdfc03
1 /*
2 Copyright 2020 Google LLC
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file or at
6 https://developers.google.com/open-source/licenses/bsd
7 */
9 #include "reader.h"
11 #include "system.h"
12 #include "block.h"
13 #include "constants.h"
14 #include "iter.h"
15 #include "record.h"
16 #include "reftable-error.h"
18 uint64_t block_source_size(struct reftable_block_source *source)
20 return source->ops->size(source->arg);
23 int block_source_read_block(struct reftable_block_source *source,
24 struct reftable_block *dest, uint64_t off,
25 uint32_t size)
27 int result = source->ops->read_block(source->arg, dest, off, size);
28 dest->source = *source;
29 return result;
32 void block_source_close(struct reftable_block_source *source)
34 if (!source->ops) {
35 return;
38 source->ops->close(source->arg);
39 source->ops = NULL;
42 static struct reftable_reader_offsets *
43 reader_offsets_for(struct reftable_reader *r, uint8_t typ)
45 switch (typ) {
46 case BLOCK_TYPE_REF:
47 return &r->ref_offsets;
48 case BLOCK_TYPE_LOG:
49 return &r->log_offsets;
50 case BLOCK_TYPE_OBJ:
51 return &r->obj_offsets;
53 abort();
56 static int reader_get_block(struct reftable_reader *r,
57 struct reftable_block *dest, uint64_t off,
58 uint32_t sz)
60 if (off >= r->size)
61 return 0;
63 if (off + sz > r->size) {
64 sz = r->size - off;
67 return block_source_read_block(&r->source, dest, off, sz);
70 uint32_t reftable_reader_hash_id(struct reftable_reader *r)
72 return r->hash_id;
75 const char *reader_name(struct reftable_reader *r)
77 return r->name;
80 static int parse_footer(struct reftable_reader *r, uint8_t *footer,
81 uint8_t *header)
83 uint8_t *f = footer;
84 uint8_t first_block_typ;
85 int err = 0;
86 uint32_t computed_crc;
87 uint32_t file_crc;
89 if (memcmp(f, "REFT", 4)) {
90 err = REFTABLE_FORMAT_ERROR;
91 goto done;
93 f += 4;
95 if (memcmp(footer, header, header_size(r->version))) {
96 err = REFTABLE_FORMAT_ERROR;
97 goto done;
100 f++;
101 r->block_size = get_be24(f);
103 f += 3;
104 r->min_update_index = get_be64(f);
105 f += 8;
106 r->max_update_index = get_be64(f);
107 f += 8;
109 if (r->version == 1) {
110 r->hash_id = GIT_SHA1_FORMAT_ID;
111 } else {
112 r->hash_id = get_be32(f);
113 switch (r->hash_id) {
114 case GIT_SHA1_FORMAT_ID:
115 break;
116 case GIT_SHA256_FORMAT_ID:
117 break;
118 default:
119 err = REFTABLE_FORMAT_ERROR;
120 goto done;
122 f += 4;
125 r->ref_offsets.index_offset = get_be64(f);
126 f += 8;
128 r->obj_offsets.offset = get_be64(f);
129 f += 8;
131 r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
132 r->obj_offsets.offset >>= 5;
134 r->obj_offsets.index_offset = get_be64(f);
135 f += 8;
136 r->log_offsets.offset = get_be64(f);
137 f += 8;
138 r->log_offsets.index_offset = get_be64(f);
139 f += 8;
141 computed_crc = crc32(0, footer, f - footer);
142 file_crc = get_be32(f);
143 f += 4;
144 if (computed_crc != file_crc) {
145 err = REFTABLE_FORMAT_ERROR;
146 goto done;
149 first_block_typ = header[header_size(r->version)];
150 r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF);
151 r->ref_offsets.offset = 0;
152 r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG ||
153 r->log_offsets.offset > 0);
154 r->obj_offsets.is_present = r->obj_offsets.offset > 0;
155 if (r->obj_offsets.is_present && !r->object_id_len) {
156 err = REFTABLE_FORMAT_ERROR;
157 goto done;
160 err = 0;
161 done:
162 return err;
165 int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
166 const char *name)
168 struct reftable_block footer = { NULL };
169 struct reftable_block header = { NULL };
170 int err = 0;
171 uint64_t file_size = block_source_size(source);
173 /* Need +1 to read type of first block. */
174 uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */
175 memset(r, 0, sizeof(struct reftable_reader));
177 if (read_size > file_size) {
178 err = REFTABLE_FORMAT_ERROR;
179 goto done;
182 err = block_source_read_block(source, &header, 0, read_size);
183 if (err != read_size) {
184 err = REFTABLE_IO_ERROR;
185 goto done;
188 if (memcmp(header.data, "REFT", 4)) {
189 err = REFTABLE_FORMAT_ERROR;
190 goto done;
192 r->version = header.data[4];
193 if (r->version != 1 && r->version != 2) {
194 err = REFTABLE_FORMAT_ERROR;
195 goto done;
198 r->size = file_size - footer_size(r->version);
199 r->source = *source;
200 r->name = xstrdup(name);
201 r->hash_id = 0;
203 err = block_source_read_block(source, &footer, r->size,
204 footer_size(r->version));
205 if (err != footer_size(r->version)) {
206 err = REFTABLE_IO_ERROR;
207 goto done;
210 err = parse_footer(r, footer.data, header.data);
211 done:
212 reftable_block_done(&footer);
213 reftable_block_done(&header);
214 return err;
217 struct table_iter {
218 struct reftable_reader *r;
219 uint8_t typ;
220 uint64_t block_off;
221 struct block_reader br;
222 struct block_iter bi;
223 int is_finished;
226 static int table_iter_init(struct table_iter *ti, struct reftable_reader *r)
228 struct block_iter bi = BLOCK_ITER_INIT;
229 memset(ti, 0, sizeof(*ti));
230 ti->r = r;
231 ti->bi = bi;
232 return 0;
235 static int table_iter_next_in_block(struct table_iter *ti,
236 struct reftable_record *rec)
238 int res = block_iter_next(&ti->bi, rec);
239 if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
240 rec->u.ref.update_index += ti->r->min_update_index;
243 return res;
246 static void table_iter_block_done(struct table_iter *ti)
248 block_reader_release(&ti->br);
249 block_iter_reset(&ti->bi);
252 static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off,
253 int version)
255 int32_t result = 0;
257 if (off == 0) {
258 data += header_size(version);
261 *typ = data[0];
262 if (reftable_is_block_type(*typ)) {
263 result = get_be24(data + 1);
265 return result;
268 int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
269 uint64_t next_off, uint8_t want_typ)
271 int32_t guess_block_size = r->block_size ? r->block_size :
272 DEFAULT_BLOCK_SIZE;
273 struct reftable_block block = { NULL };
274 uint8_t block_typ = 0;
275 int err = 0;
276 uint32_t header_off = next_off ? 0 : header_size(r->version);
277 int32_t block_size = 0;
279 if (next_off >= r->size)
280 return 1;
282 err = reader_get_block(r, &block, next_off, guess_block_size);
283 if (err < 0)
284 goto done;
286 block_size = extract_block_size(block.data, &block_typ, next_off,
287 r->version);
288 if (block_size < 0) {
289 err = block_size;
290 goto done;
292 if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
293 err = 1;
294 goto done;
297 if (block_size > guess_block_size) {
298 reftable_block_done(&block);
299 err = reader_get_block(r, &block, next_off, block_size);
300 if (err < 0) {
301 goto done;
305 err = block_reader_init(br, &block, header_off, r->block_size,
306 hash_size(r->hash_id));
307 done:
308 reftable_block_done(&block);
310 return err;
313 static void table_iter_close(struct table_iter *ti)
315 table_iter_block_done(ti);
316 block_iter_close(&ti->bi);
319 static int table_iter_next_block(struct table_iter *ti)
321 uint64_t next_block_off = ti->block_off + ti->br.full_block_size;
322 int err;
324 err = reader_init_block_reader(ti->r, &ti->br, next_block_off, ti->typ);
325 if (err > 0)
326 ti->is_finished = 1;
327 if (err)
328 return err;
330 ti->block_off = next_block_off;
331 ti->is_finished = 0;
332 block_iter_seek_start(&ti->bi, &ti->br);
334 return 0;
337 static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
339 if (reftable_record_type(rec) != ti->typ)
340 return REFTABLE_API_ERROR;
342 while (1) {
343 int err;
345 if (ti->is_finished)
346 return 1;
349 * Check whether the current block still has more records. If
350 * so, return it. If the iterator returns positive then the
351 * current block has been exhausted.
353 err = table_iter_next_in_block(ti, rec);
354 if (err <= 0)
355 return err;
358 * Otherwise, we need to continue to the next block in the
359 * table and retry. If there are no more blocks then the
360 * iterator is drained.
362 err = table_iter_next_block(ti);
363 if (err) {
364 ti->is_finished = 1;
365 return err;
370 static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ)
372 int err;
374 err = reader_init_block_reader(ti->r, &ti->br, off, typ);
375 if (err != 0)
376 return err;
378 ti->typ = block_reader_type(&ti->br);
379 ti->block_off = off;
380 block_iter_seek_start(&ti->bi, &ti->br);
381 return 0;
384 static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index)
386 struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
387 uint64_t off = offs->offset;
388 if (index) {
389 off = offs->index_offset;
390 if (off == 0) {
391 return 1;
393 typ = BLOCK_TYPE_INDEX;
396 return table_iter_seek_to(ti, off, typ);
399 static int table_iter_seek_linear(struct table_iter *ti,
400 struct reftable_record *want)
402 struct strbuf want_key = STRBUF_INIT;
403 struct strbuf got_key = STRBUF_INIT;
404 struct reftable_record rec;
405 int err;
407 reftable_record_init(&rec, reftable_record_type(want));
408 reftable_record_key(want, &want_key);
411 * First we need to locate the block that must contain our record. To
412 * do so we scan through blocks linearly until we find the first block
413 * whose first key is bigger than our wanted key. Once we have found
414 * that block we know that the key must be contained in the preceding
415 * block.
417 * This algorithm is somewhat unfortunate because it means that we
418 * always have to seek one block too far and then back up. But as we
419 * can only decode the _first_ key of a block but not its _last_ key we
420 * have no other way to do this.
422 while (1) {
423 struct table_iter next = *ti;
426 * We must be careful to not modify underlying data of `ti`
427 * because we may find that `next` does not contain our desired
428 * block, but that `ti` does. In that case, we would discard
429 * `next` and continue with `ti`.
431 * This also means that we cannot reuse allocated memory for
432 * `next` here. While it would be great if we could, it should
433 * in practice not be too bad given that we should only ever
434 * end up doing linear seeks with at most three blocks. As soon
435 * as we have more than three blocks we would have an index, so
436 * we would not do a linear search there anymore.
438 memset(&next.br.block, 0, sizeof(next.br.block));
439 next.br.zstream = NULL;
440 next.br.uncompressed_data = NULL;
441 next.br.uncompressed_cap = 0;
443 err = table_iter_next_block(&next);
444 if (err < 0)
445 goto done;
446 if (err > 0)
447 break;
449 err = block_reader_first_key(&next.br, &got_key);
450 if (err < 0)
451 goto done;
453 if (strbuf_cmp(&got_key, &want_key) > 0) {
454 table_iter_block_done(&next);
455 break;
458 table_iter_block_done(ti);
459 *ti = next;
463 * We have located the block that must contain our record, so we seek
464 * the wanted key inside of it. If the block does not contain our key
465 * we know that the corresponding record does not exist.
467 err = block_iter_seek_key(&ti->bi, &ti->br, &want_key);
468 if (err < 0)
469 goto done;
470 err = 0;
472 done:
473 reftable_record_release(&rec);
474 strbuf_release(&want_key);
475 strbuf_release(&got_key);
476 return err;
479 static int table_iter_seek_indexed(struct table_iter *ti,
480 struct reftable_record *rec)
482 struct reftable_record want_index = {
483 .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = STRBUF_INIT }
485 struct reftable_record index_result = {
486 .type = BLOCK_TYPE_INDEX,
487 .u.idx = { .last_key = STRBUF_INIT },
489 int err;
491 reftable_record_key(rec, &want_index.u.idx.last_key);
494 * The index may consist of multiple levels, where each level may have
495 * multiple index blocks. We start by doing a linear search in the
496 * highest layer that identifies the relevant index block as well as
497 * the record inside that block that corresponds to our wanted key.
499 err = table_iter_seek_linear(ti, &want_index);
500 if (err < 0)
501 goto done;
504 * Traverse down the levels until we find a non-index entry.
506 while (1) {
508 * In case we seek a record that does not exist the index iter
509 * will tell us that the iterator is over. This works because
510 * the last index entry of the current level will contain the
511 * last key it knows about. So in case our seeked key is larger
512 * than the last indexed key we know that it won't exist.
514 * There is one subtlety in the layout of the index section
515 * that makes this work as expected: the highest-level index is
516 * at end of the section and will point backwards and thus we
517 * start reading from the end of the index section, not the
518 * beginning.
520 * If that wasn't the case and the order was reversed then the
521 * linear seek would seek into the lower levels and traverse
522 * all levels of the index only to find out that the key does
523 * not exist.
525 err = table_iter_next(ti, &index_result);
526 if (err != 0)
527 goto done;
529 err = table_iter_seek_to(ti, index_result.u.idx.offset, 0);
530 if (err != 0)
531 goto done;
533 err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key);
534 if (err < 0)
535 goto done;
537 if (ti->typ == reftable_record_type(rec)) {
538 err = 0;
539 break;
542 if (ti->typ != BLOCK_TYPE_INDEX) {
543 err = REFTABLE_FORMAT_ERROR;
544 goto done;
548 done:
549 reftable_record_release(&want_index);
550 reftable_record_release(&index_result);
551 return err;
554 static int table_iter_seek(struct table_iter *ti,
555 struct reftable_record *want)
557 uint8_t typ = reftable_record_type(want);
558 struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ);
559 int err;
561 err = table_iter_seek_start(ti, reftable_record_type(want),
562 !!offs->index_offset);
563 if (err < 0)
564 goto out;
566 if (offs->index_offset)
567 err = table_iter_seek_indexed(ti, want);
568 else
569 err = table_iter_seek_linear(ti, want);
570 if (err)
571 goto out;
573 out:
574 return err;
577 static int table_iter_seek_void(void *ti, struct reftable_record *want)
579 return table_iter_seek(ti, want);
582 static int table_iter_next_void(void *ti, struct reftable_record *rec)
584 return table_iter_next(ti, rec);
587 static void table_iter_close_void(void *ti)
589 table_iter_close(ti);
592 static struct reftable_iterator_vtable table_iter_vtable = {
593 .seek = &table_iter_seek_void,
594 .next = &table_iter_next_void,
595 .close = &table_iter_close_void,
598 static void iterator_from_table_iter(struct reftable_iterator *it,
599 struct table_iter *ti)
601 assert(!it->ops);
602 it->iter_arg = ti;
603 it->ops = &table_iter_vtable;
606 void reader_init_iter(struct reftable_reader *r,
607 struct reftable_iterator *it,
608 uint8_t typ)
610 struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
612 if (offs->is_present) {
613 struct table_iter *ti;
614 REFTABLE_ALLOC_ARRAY(ti, 1);
615 table_iter_init(ti, r);
616 iterator_from_table_iter(it, ti);
617 } else {
618 iterator_set_empty(it);
622 void reftable_reader_init_ref_iterator(struct reftable_reader *r,
623 struct reftable_iterator *it)
625 reader_init_iter(r, it, BLOCK_TYPE_REF);
628 void reftable_reader_init_log_iterator(struct reftable_reader *r,
629 struct reftable_iterator *it)
631 reader_init_iter(r, it, BLOCK_TYPE_LOG);
634 void reader_close(struct reftable_reader *r)
636 block_source_close(&r->source);
637 FREE_AND_NULL(r->name);
640 int reftable_reader_new(struct reftable_reader **p,
641 struct reftable_block_source *src, char const *name)
643 struct reftable_reader *rd = reftable_calloc(1, sizeof(*rd));
644 int err = init_reader(rd, src, name);
645 if (err == 0) {
646 *p = rd;
647 } else {
648 block_source_close(src);
649 reftable_free(rd);
651 return err;
654 void reftable_reader_free(struct reftable_reader *r)
656 if (!r)
657 return;
658 reader_close(r);
659 reftable_free(r);
662 static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
663 struct reftable_iterator *it,
664 uint8_t *oid)
666 struct reftable_record want = {
667 .type = BLOCK_TYPE_OBJ,
668 .u.obj = {
669 .hash_prefix = oid,
670 .hash_prefix_len = r->object_id_len,
673 struct reftable_iterator oit = { NULL };
674 struct reftable_record got = {
675 .type = BLOCK_TYPE_OBJ,
676 .u.obj = { 0 },
678 int err = 0;
679 struct indexed_table_ref_iter *itr = NULL;
681 /* Look through the reverse index. */
682 reader_init_iter(r, &oit, BLOCK_TYPE_OBJ);
683 err = iterator_seek(&oit, &want);
684 if (err != 0)
685 goto done;
687 /* read out the reftable_obj_record */
688 err = iterator_next(&oit, &got);
689 if (err < 0)
690 goto done;
692 if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix,
693 r->object_id_len)) {
694 /* didn't find it; return empty iterator */
695 iterator_set_empty(it);
696 err = 0;
697 goto done;
700 err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id),
701 got.u.obj.offsets,
702 got.u.obj.offset_len);
703 if (err < 0)
704 goto done;
705 got.u.obj.offsets = NULL;
706 iterator_from_indexed_table_ref_iter(it, itr);
708 done:
709 reftable_iterator_destroy(&oit);
710 reftable_record_release(&got);
711 return err;
714 static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
715 struct reftable_iterator *it,
716 uint8_t *oid)
718 struct table_iter *ti;
719 struct filtering_ref_iterator *filter = NULL;
720 struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
721 int oid_len = hash_size(r->hash_id);
722 int err;
724 REFTABLE_ALLOC_ARRAY(ti, 1);
725 table_iter_init(ti, r);
726 err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0);
727 if (err < 0) {
728 reftable_free(ti);
729 return err;
732 filter = reftable_malloc(sizeof(struct filtering_ref_iterator));
733 *filter = empty;
735 strbuf_add(&filter->oid, oid, oid_len);
736 iterator_from_table_iter(&filter->it, ti);
738 iterator_from_filtering_ref_iterator(it, filter);
739 return 0;
742 int reftable_reader_refs_for(struct reftable_reader *r,
743 struct reftable_iterator *it, uint8_t *oid)
745 if (r->obj_offsets.is_present)
746 return reftable_reader_refs_for_indexed(r, it, oid);
747 return reftable_reader_refs_for_unindexed(r, it, oid);
750 uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
752 return r->max_update_index;
755 uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
757 return r->min_update_index;
760 int reftable_reader_print_blocks(const char *tablename)
762 struct {
763 const char *name;
764 int type;
765 } sections[] = {
767 .name = "ref",
768 .type = BLOCK_TYPE_REF,
771 .name = "obj",
772 .type = BLOCK_TYPE_OBJ,
775 .name = "log",
776 .type = BLOCK_TYPE_LOG,
779 struct reftable_block_source src = { 0 };
780 struct reftable_reader *r = NULL;
781 struct table_iter ti = { 0 };
782 size_t i;
783 int err;
785 err = reftable_block_source_from_file(&src, tablename);
786 if (err < 0)
787 goto done;
789 err = reftable_reader_new(&r, &src, tablename);
790 if (err < 0)
791 goto done;
793 table_iter_init(&ti, r);
795 printf("header:\n");
796 printf(" block_size: %d\n", r->block_size);
798 for (i = 0; i < ARRAY_SIZE(sections); i++) {
799 err = table_iter_seek_start(&ti, sections[i].type, 0);
800 if (err < 0)
801 goto done;
802 if (err > 0)
803 continue;
805 printf("%s:\n", sections[i].name);
807 while (1) {
808 printf(" - length: %u\n", ti.br.block_len);
809 printf(" restarts: %u\n", ti.br.restart_count);
811 err = table_iter_next_block(&ti);
812 if (err < 0)
813 goto done;
814 if (err > 0)
815 break;
819 done:
820 reftable_reader_free(r);
821 table_iter_close(&ti);
822 return err;