reftable/reftable-writer.h

   1 /*
   2 Copyright 2020 Google LLC
   3
   4 Use of this source code is governed by a BSD-style
   5 license that can be found in the LICENSE file or at
   6 https://developers.google.com/open-source/licenses/bsd
   7 */
   8
   9 #ifndef REFTABLE_WRITER_H
  10 #define REFTABLE_WRITER_H
  11
  12 #include "reftable-record.h"
  13
  14 #include <stdint.h>
  15 #include <unistd.h> /* ssize_t */
  16
  17 /* Writing single reftables */
  18
  19 /* reftable_write_options sets options for writing a single reftable. */
  20 struct reftable_write_options {
  21         /* boolean: do not pad out blocks to block size. */
  22         unsigned unpadded : 1;
  23
  24         /* the blocksize. Should be less than 2^24. */
  25         uint32_t block_size;
  26
  27         /* boolean: do not generate a SHA1 => ref index. */
  28         unsigned skip_index_objects : 1;
  29
  30         /* how often to write complete keys in each block. */
  31         uint16_t restart_interval;
  32
  33         /* 4-byte identifier ("sha1", "s256") of the hash.
  34          * Defaults to SHA1 if unset
  35          */
  36         uint32_t hash_id;
  37
  38         /* Default mode for creating files. If unset, use 0666 (+umask) */
  39         unsigned int default_permissions;
  40
  41         /* boolean: copy log messages exactly. If unset, check that the message
  42          *   is a single line, and add '\n' if missing.
  43          */
  44         unsigned exact_log_message : 1;
  45
  46         /* boolean: Prevent auto-compaction of tables. */
  47         unsigned disable_auto_compact : 1;
  48
  49         /*
  50          * Geometric sequence factor used by auto-compaction to decide which
  51          * tables to compact. Defaults to 2 if unset.
  52          */
  53         uint8_t auto_compaction_factor;
  54
  55         /*
  56          * The number of milliseconds to wait when trying to lock "tables.list".
  57          * Note that this does not apply to locking individual tables, as these
  58          * should only ever be locked when already holding the "tables.list"
  59          * lock.
  60          *
  61          * Passing 0 will fail immediately when the file is locked, passing a
  62          * negative value will cause us to block indefinitely.
  63          */
  64         long lock_timeout_ms;
  65 };
  66
  67 /* reftable_block_stats holds statistics for a single block type */
  68 struct reftable_block_stats {
  69         /* total number of entries written */
  70         int entries;
  71         /* total number of key restarts */
  72         int restarts;
  73         /* total number of blocks */
  74         int blocks;
  75         /* total number of index blocks */
  76         int index_blocks;
  77         /* depth of the index */
  78         int max_index_level;
  79
  80         /* offset of the first block for this type */
  81         uint64_t offset;
  82         /* offset of the top level index block for this type, or 0 if not
  83          * present */
  84         uint64_t index_offset;
  85 };
  86
  87 /* stats holds overall statistics for a single reftable */
  88 struct reftable_stats {
  89         /* total number of blocks written. */
  90         int blocks;
  91         /* stats for ref data */
  92         struct reftable_block_stats ref_stats;
  93         /* stats for the SHA1 to ref map. */
  94         struct reftable_block_stats obj_stats;
  95         /* stats for index blocks */
  96         struct reftable_block_stats idx_stats;
  97         /* stats for log blocks */
  98         struct reftable_block_stats log_stats;
  99
 100         /* disambiguation length of shortened object IDs. */
 101         int object_id_len;
 102 };
 103
 104 struct reftable_writer;
 105
 106 /* Create a new writer. */
 107 int reftable_writer_new(struct reftable_writer **out,
 108                         ssize_t (*writer_func)(void *, const void *, size_t),
 109                         int (*flush_func)(void *),
 110                         void *writer_arg, const struct reftable_write_options *opts);
 111
 112 /* Set the range of update indices for the records we will add. When writing a
 113    table into a stack, the min should be at least
 114    reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned.
 115
 116    For transactional updates to a stack, typically min==max, and the
 117    update_index can be obtained by inspeciting the stack. When converting an
 118    existing ref database into a single reftable, this would be a range of
 119    update-index timestamps.
 120  */
 121 void reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
 122                                 uint64_t max);
 123
 124 /*
 125   Add a reftable_ref_record. The record should have names that come after
 126   already added records.
 127
 128   The update_index must be within the limits set by
 129   reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned. It is an
 130   REFTABLE_API_ERROR error to write a ref record after a log record.
 131 */
 132 int reftable_writer_add_ref(struct reftable_writer *w,
 133                             struct reftable_ref_record *ref);
 134
 135 /*
 136   Convenience function to add multiple reftable_ref_records; the function sorts
 137   the records before adding them, reordering the records array passed in.
 138 */
 139 int reftable_writer_add_refs(struct reftable_writer *w,
 140                              struct reftable_ref_record *refs, int n);
 141
 142 /*
 143   adds reftable_log_records. Log records are keyed by (refname, decreasing
 144   update_index). The key for the record added must come after the already added
 145   log records.
 146 */
 147 int reftable_writer_add_log(struct reftable_writer *w,
 148                             struct reftable_log_record *log);
 149
 150 /*
 151   Convenience function to add multiple reftable_log_records; the function sorts
 152   the records before adding them, reordering records array passed in.
 153 */
 154 int reftable_writer_add_logs(struct reftable_writer *w,
 155                              struct reftable_log_record *logs, int n);
 156
 157 /* reftable_writer_close finalizes the reftable. The writer is retained so
 158  * statistics can be inspected. */
 159 int reftable_writer_close(struct reftable_writer *w);
 160
 161 /* writer_stats returns the statistics on the reftable being written.
 162
 163    This struct becomes invalid when the writer is freed.
 164  */
 165 const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w);
 166
 167 /* reftable_writer_free deallocates memory for the writer */
 168 void reftable_writer_free(struct reftable_writer *w);
 169
 170 #endif