2 Copyright 2020 Google LLC
4 Use of this source code is governed by a BSD-style
5 license that can be found in the LICENSE file or at
6 https://developers.google.com/open-source/licenses/bsd
13 #include "constants.h"
16 #include "reftable-error.h"
18 uint64_t block_source_size(struct reftable_block_source
*source
)
20 return source
->ops
->size(source
->arg
);
23 int block_source_read_block(struct reftable_block_source
*source
,
24 struct reftable_block
*dest
, uint64_t off
,
27 int result
= source
->ops
->read_block(source
->arg
, dest
, off
, size
);
28 dest
->source
= *source
;
32 void block_source_close(struct reftable_block_source
*source
)
38 source
->ops
->close(source
->arg
);
42 static struct reftable_reader_offsets
*
43 reader_offsets_for(struct reftable_reader
*r
, uint8_t typ
)
47 return &r
->ref_offsets
;
49 return &r
->log_offsets
;
51 return &r
->obj_offsets
;
56 static int reader_get_block(struct reftable_reader
*r
,
57 struct reftable_block
*dest
, uint64_t off
,
63 if (off
+ sz
> r
->size
) {
67 return block_source_read_block(&r
->source
, dest
, off
, sz
);
70 uint32_t reftable_reader_hash_id(struct reftable_reader
*r
)
75 const char *reader_name(struct reftable_reader
*r
)
80 static int parse_footer(struct reftable_reader
*r
, uint8_t *footer
,
84 uint8_t first_block_typ
;
86 uint32_t computed_crc
;
89 if (memcmp(f
, "REFT", 4)) {
90 err
= REFTABLE_FORMAT_ERROR
;
95 if (memcmp(footer
, header
, header_size(r
->version
))) {
96 err
= REFTABLE_FORMAT_ERROR
;
101 r
->block_size
= get_be24(f
);
104 r
->min_update_index
= get_be64(f
);
106 r
->max_update_index
= get_be64(f
);
109 if (r
->version
== 1) {
110 r
->hash_id
= GIT_SHA1_FORMAT_ID
;
112 r
->hash_id
= get_be32(f
);
113 switch (r
->hash_id
) {
114 case GIT_SHA1_FORMAT_ID
:
116 case GIT_SHA256_FORMAT_ID
:
119 err
= REFTABLE_FORMAT_ERROR
;
125 r
->ref_offsets
.index_offset
= get_be64(f
);
128 r
->obj_offsets
.offset
= get_be64(f
);
131 r
->object_id_len
= r
->obj_offsets
.offset
& ((1 << 5) - 1);
132 r
->obj_offsets
.offset
>>= 5;
134 r
->obj_offsets
.index_offset
= get_be64(f
);
136 r
->log_offsets
.offset
= get_be64(f
);
138 r
->log_offsets
.index_offset
= get_be64(f
);
141 computed_crc
= crc32(0, footer
, f
- footer
);
142 file_crc
= get_be32(f
);
144 if (computed_crc
!= file_crc
) {
145 err
= REFTABLE_FORMAT_ERROR
;
149 first_block_typ
= header
[header_size(r
->version
)];
150 r
->ref_offsets
.is_present
= (first_block_typ
== BLOCK_TYPE_REF
);
151 r
->ref_offsets
.offset
= 0;
152 r
->log_offsets
.is_present
= (first_block_typ
== BLOCK_TYPE_LOG
||
153 r
->log_offsets
.offset
> 0);
154 r
->obj_offsets
.is_present
= r
->obj_offsets
.offset
> 0;
155 if (r
->obj_offsets
.is_present
&& !r
->object_id_len
) {
156 err
= REFTABLE_FORMAT_ERROR
;
166 struct reftable_reader
*r
;
169 struct block_reader br
;
170 struct block_iter bi
;
174 static int table_iter_init(struct table_iter
*ti
, struct reftable_reader
*r
)
176 struct block_iter bi
= BLOCK_ITER_INIT
;
177 memset(ti
, 0, sizeof(*ti
));
178 reftable_reader_incref(r
);
184 static int table_iter_next_in_block(struct table_iter
*ti
,
185 struct reftable_record
*rec
)
187 int res
= block_iter_next(&ti
->bi
, rec
);
188 if (res
== 0 && reftable_record_type(rec
) == BLOCK_TYPE_REF
) {
189 rec
->u
.ref
.update_index
+= ti
->r
->min_update_index
;
195 static void table_iter_block_done(struct table_iter
*ti
)
197 block_reader_release(&ti
->br
);
198 block_iter_reset(&ti
->bi
);
201 static int32_t extract_block_size(uint8_t *data
, uint8_t *typ
, uint64_t off
,
207 data
+= header_size(version
);
211 if (reftable_is_block_type(*typ
)) {
212 result
= get_be24(data
+ 1);
217 int reader_init_block_reader(struct reftable_reader
*r
, struct block_reader
*br
,
218 uint64_t next_off
, uint8_t want_typ
)
220 int32_t guess_block_size
= r
->block_size
? r
->block_size
:
222 struct reftable_block block
= { NULL
};
223 uint8_t block_typ
= 0;
225 uint32_t header_off
= next_off
? 0 : header_size(r
->version
);
226 int32_t block_size
= 0;
228 if (next_off
>= r
->size
)
231 err
= reader_get_block(r
, &block
, next_off
, guess_block_size
);
235 block_size
= extract_block_size(block
.data
, &block_typ
, next_off
,
237 if (block_size
< 0) {
241 if (want_typ
!= BLOCK_TYPE_ANY
&& block_typ
!= want_typ
) {
246 if (block_size
> guess_block_size
) {
247 reftable_block_done(&block
);
248 err
= reader_get_block(r
, &block
, next_off
, block_size
);
254 err
= block_reader_init(br
, &block
, header_off
, r
->block_size
,
255 hash_size(r
->hash_id
));
257 reftable_block_done(&block
);
262 static void table_iter_close(struct table_iter
*ti
)
264 table_iter_block_done(ti
);
265 block_iter_close(&ti
->bi
);
266 reftable_reader_decref(ti
->r
);
269 static int table_iter_next_block(struct table_iter
*ti
)
271 uint64_t next_block_off
= ti
->block_off
+ ti
->br
.full_block_size
;
274 err
= reader_init_block_reader(ti
->r
, &ti
->br
, next_block_off
, ti
->typ
);
280 ti
->block_off
= next_block_off
;
282 block_iter_seek_start(&ti
->bi
, &ti
->br
);
287 static int table_iter_next(struct table_iter
*ti
, struct reftable_record
*rec
)
289 if (reftable_record_type(rec
) != ti
->typ
)
290 return REFTABLE_API_ERROR
;
299 * Check whether the current block still has more records. If
300 * so, return it. If the iterator returns positive then the
301 * current block has been exhausted.
303 err
= table_iter_next_in_block(ti
, rec
);
308 * Otherwise, we need to continue to the next block in the
309 * table and retry. If there are no more blocks then the
310 * iterator is drained.
312 err
= table_iter_next_block(ti
);
320 static int table_iter_seek_to(struct table_iter
*ti
, uint64_t off
, uint8_t typ
)
324 err
= reader_init_block_reader(ti
->r
, &ti
->br
, off
, typ
);
328 ti
->typ
= block_reader_type(&ti
->br
);
330 block_iter_seek_start(&ti
->bi
, &ti
->br
);
335 static int table_iter_seek_start(struct table_iter
*ti
, uint8_t typ
, int index
)
337 struct reftable_reader_offsets
*offs
= reader_offsets_for(ti
->r
, typ
);
338 uint64_t off
= offs
->offset
;
340 off
= offs
->index_offset
;
344 typ
= BLOCK_TYPE_INDEX
;
347 return table_iter_seek_to(ti
, off
, typ
);
350 static int table_iter_seek_linear(struct table_iter
*ti
,
351 struct reftable_record
*want
)
353 struct strbuf want_key
= STRBUF_INIT
;
354 struct strbuf got_key
= STRBUF_INIT
;
355 struct reftable_record rec
;
358 reftable_record_init(&rec
, reftable_record_type(want
));
359 reftable_record_key(want
, &want_key
);
362 * First we need to locate the block that must contain our record. To
363 * do so we scan through blocks linearly until we find the first block
364 * whose first key is bigger than our wanted key. Once we have found
365 * that block we know that the key must be contained in the preceding
368 * This algorithm is somewhat unfortunate because it means that we
369 * always have to seek one block too far and then back up. But as we
370 * can only decode the _first_ key of a block but not its _last_ key we
371 * have no other way to do this.
374 struct table_iter next
= *ti
;
377 * We must be careful to not modify underlying data of `ti`
378 * because we may find that `next` does not contain our desired
379 * block, but that `ti` does. In that case, we would discard
380 * `next` and continue with `ti`.
382 * This also means that we cannot reuse allocated memory for
383 * `next` here. While it would be great if we could, it should
384 * in practice not be too bad given that we should only ever
385 * end up doing linear seeks with at most three blocks. As soon
386 * as we have more than three blocks we would have an index, so
387 * we would not do a linear search there anymore.
389 memset(&next
.br
.block
, 0, sizeof(next
.br
.block
));
390 next
.br
.zstream
= NULL
;
391 next
.br
.uncompressed_data
= NULL
;
392 next
.br
.uncompressed_cap
= 0;
394 err
= table_iter_next_block(&next
);
400 err
= block_reader_first_key(&next
.br
, &got_key
);
404 if (strbuf_cmp(&got_key
, &want_key
) > 0) {
405 table_iter_block_done(&next
);
409 table_iter_block_done(ti
);
414 * We have located the block that must contain our record, so we seek
415 * the wanted key inside of it. If the block does not contain our key
416 * we know that the corresponding record does not exist.
418 err
= block_iter_seek_key(&ti
->bi
, &ti
->br
, &want_key
);
424 reftable_record_release(&rec
);
425 strbuf_release(&want_key
);
426 strbuf_release(&got_key
);
430 static int table_iter_seek_indexed(struct table_iter
*ti
,
431 struct reftable_record
*rec
)
433 struct reftable_record want_index
= {
434 .type
= BLOCK_TYPE_INDEX
, .u
.idx
= { .last_key
= STRBUF_INIT
}
436 struct reftable_record index_result
= {
437 .type
= BLOCK_TYPE_INDEX
,
438 .u
.idx
= { .last_key
= STRBUF_INIT
},
442 reftable_record_key(rec
, &want_index
.u
.idx
.last_key
);
445 * The index may consist of multiple levels, where each level may have
446 * multiple index blocks. We start by doing a linear search in the
447 * highest layer that identifies the relevant index block as well as
448 * the record inside that block that corresponds to our wanted key.
450 err
= table_iter_seek_linear(ti
, &want_index
);
455 * Traverse down the levels until we find a non-index entry.
459 * In case we seek a record that does not exist the index iter
460 * will tell us that the iterator is over. This works because
461 * the last index entry of the current level will contain the
462 * last key it knows about. So in case our seeked key is larger
463 * than the last indexed key we know that it won't exist.
465 * There is one subtlety in the layout of the index section
466 * that makes this work as expected: the highest-level index is
467 * at end of the section and will point backwards and thus we
468 * start reading from the end of the index section, not the
471 * If that wasn't the case and the order was reversed then the
472 * linear seek would seek into the lower levels and traverse
473 * all levels of the index only to find out that the key does
476 err
= table_iter_next(ti
, &index_result
);
480 err
= table_iter_seek_to(ti
, index_result
.u
.idx
.offset
, 0);
484 err
= block_iter_seek_key(&ti
->bi
, &ti
->br
, &want_index
.u
.idx
.last_key
);
488 if (ti
->typ
== reftable_record_type(rec
)) {
493 if (ti
->typ
!= BLOCK_TYPE_INDEX
) {
494 err
= REFTABLE_FORMAT_ERROR
;
500 reftable_record_release(&want_index
);
501 reftable_record_release(&index_result
);
505 static int table_iter_seek(struct table_iter
*ti
,
506 struct reftable_record
*want
)
508 uint8_t typ
= reftable_record_type(want
);
509 struct reftable_reader_offsets
*offs
= reader_offsets_for(ti
->r
, typ
);
512 err
= table_iter_seek_start(ti
, reftable_record_type(want
),
513 !!offs
->index_offset
);
517 if (offs
->index_offset
)
518 err
= table_iter_seek_indexed(ti
, want
);
520 err
= table_iter_seek_linear(ti
, want
);
528 static int table_iter_seek_void(void *ti
, struct reftable_record
*want
)
530 return table_iter_seek(ti
, want
);
533 static int table_iter_next_void(void *ti
, struct reftable_record
*rec
)
535 return table_iter_next(ti
, rec
);
538 static void table_iter_close_void(void *ti
)
540 table_iter_close(ti
);
543 static struct reftable_iterator_vtable table_iter_vtable
= {
544 .seek
= &table_iter_seek_void
,
545 .next
= &table_iter_next_void
,
546 .close
= &table_iter_close_void
,
549 static void iterator_from_table_iter(struct reftable_iterator
*it
,
550 struct table_iter
*ti
)
554 it
->ops
= &table_iter_vtable
;
557 void reader_init_iter(struct reftable_reader
*r
,
558 struct reftable_iterator
*it
,
561 struct reftable_reader_offsets
*offs
= reader_offsets_for(r
, typ
);
563 if (offs
->is_present
) {
564 struct table_iter
*ti
;
565 REFTABLE_ALLOC_ARRAY(ti
, 1);
566 table_iter_init(ti
, r
);
567 iterator_from_table_iter(it
, ti
);
569 iterator_set_empty(it
);
573 void reftable_reader_init_ref_iterator(struct reftable_reader
*r
,
574 struct reftable_iterator
*it
)
576 reader_init_iter(r
, it
, BLOCK_TYPE_REF
);
579 void reftable_reader_init_log_iterator(struct reftable_reader
*r
,
580 struct reftable_iterator
*it
)
582 reader_init_iter(r
, it
, BLOCK_TYPE_LOG
);
585 int reftable_reader_new(struct reftable_reader
**out
,
586 struct reftable_block_source
*source
, char const *name
)
588 struct reftable_block footer
= { 0 };
589 struct reftable_block header
= { 0 };
590 struct reftable_reader
*r
;
591 uint64_t file_size
= block_source_size(source
);
595 REFTABLE_CALLOC_ARRAY(r
, 1);
598 * We need one extra byte to read the type of first block. We also
599 * pretend to always be reading v2 of the format because it is larger.
601 read_size
= header_size(2) + 1;
602 if (read_size
> file_size
) {
603 err
= REFTABLE_FORMAT_ERROR
;
607 err
= block_source_read_block(source
, &header
, 0, read_size
);
608 if (err
!= read_size
) {
609 err
= REFTABLE_IO_ERROR
;
613 if (memcmp(header
.data
, "REFT", 4)) {
614 err
= REFTABLE_FORMAT_ERROR
;
617 r
->version
= header
.data
[4];
618 if (r
->version
!= 1 && r
->version
!= 2) {
619 err
= REFTABLE_FORMAT_ERROR
;
623 r
->size
= file_size
- footer_size(r
->version
);
625 r
->name
= xstrdup(name
);
629 err
= block_source_read_block(source
, &footer
, r
->size
,
630 footer_size(r
->version
));
631 if (err
!= footer_size(r
->version
)) {
632 err
= REFTABLE_IO_ERROR
;
636 err
= parse_footer(r
, footer
.data
, header
.data
);
643 reftable_block_done(&footer
);
644 reftable_block_done(&header
);
647 block_source_close(source
);
652 void reftable_reader_incref(struct reftable_reader
*r
)
655 BUG("cannot increment ref counter of dead reader");
659 void reftable_reader_decref(struct reftable_reader
*r
)
664 BUG("cannot decrement ref counter of dead reader");
667 block_source_close(&r
->source
);
668 FREE_AND_NULL(r
->name
);
672 static int reftable_reader_refs_for_indexed(struct reftable_reader
*r
,
673 struct reftable_iterator
*it
,
676 struct reftable_record want
= {
677 .type
= BLOCK_TYPE_OBJ
,
680 .hash_prefix_len
= r
->object_id_len
,
683 struct reftable_iterator oit
= { NULL
};
684 struct reftable_record got
= {
685 .type
= BLOCK_TYPE_OBJ
,
689 struct indexed_table_ref_iter
*itr
= NULL
;
691 /* Look through the reverse index. */
692 reader_init_iter(r
, &oit
, BLOCK_TYPE_OBJ
);
693 err
= iterator_seek(&oit
, &want
);
697 /* read out the reftable_obj_record */
698 err
= iterator_next(&oit
, &got
);
702 if (err
> 0 || memcmp(want
.u
.obj
.hash_prefix
, got
.u
.obj
.hash_prefix
,
704 /* didn't find it; return empty iterator */
705 iterator_set_empty(it
);
710 err
= new_indexed_table_ref_iter(&itr
, r
, oid
, hash_size(r
->hash_id
),
712 got
.u
.obj
.offset_len
);
715 got
.u
.obj
.offsets
= NULL
;
716 iterator_from_indexed_table_ref_iter(it
, itr
);
719 reftable_iterator_destroy(&oit
);
720 reftable_record_release(&got
);
724 static int reftable_reader_refs_for_unindexed(struct reftable_reader
*r
,
725 struct reftable_iterator
*it
,
728 struct table_iter
*ti
;
729 struct filtering_ref_iterator
*filter
= NULL
;
730 struct filtering_ref_iterator empty
= FILTERING_REF_ITERATOR_INIT
;
731 int oid_len
= hash_size(r
->hash_id
);
734 REFTABLE_ALLOC_ARRAY(ti
, 1);
735 table_iter_init(ti
, r
);
736 err
= table_iter_seek_start(ti
, BLOCK_TYPE_REF
, 0);
742 filter
= reftable_malloc(sizeof(struct filtering_ref_iterator
));
745 strbuf_add(&filter
->oid
, oid
, oid_len
);
746 iterator_from_table_iter(&filter
->it
, ti
);
748 iterator_from_filtering_ref_iterator(it
, filter
);
752 int reftable_reader_refs_for(struct reftable_reader
*r
,
753 struct reftable_iterator
*it
, uint8_t *oid
)
755 if (r
->obj_offsets
.is_present
)
756 return reftable_reader_refs_for_indexed(r
, it
, oid
);
757 return reftable_reader_refs_for_unindexed(r
, it
, oid
);
760 uint64_t reftable_reader_max_update_index(struct reftable_reader
*r
)
762 return r
->max_update_index
;
765 uint64_t reftable_reader_min_update_index(struct reftable_reader
*r
)
767 return r
->min_update_index
;
770 int reftable_reader_print_blocks(const char *tablename
)
778 .type
= BLOCK_TYPE_REF
,
782 .type
= BLOCK_TYPE_OBJ
,
786 .type
= BLOCK_TYPE_LOG
,
789 struct reftable_block_source src
= { 0 };
790 struct reftable_reader
*r
= NULL
;
791 struct table_iter ti
= { 0 };
795 err
= reftable_block_source_from_file(&src
, tablename
);
799 err
= reftable_reader_new(&r
, &src
, tablename
);
803 table_iter_init(&ti
, r
);
806 printf(" block_size: %d\n", r
->block_size
);
808 for (i
= 0; i
< ARRAY_SIZE(sections
); i
++) {
809 err
= table_iter_seek_start(&ti
, sections
[i
].type
, 0);
815 printf("%s:\n", sections
[i
].name
);
818 printf(" - length: %u\n", ti
.br
.block_len
);
819 printf(" restarts: %u\n", ti
.br
.restart_count
);
821 err
= table_iter_next_block(&ti
);
830 reftable_reader_decref(r
);
831 table_iter_close(&ti
);