src/include/access/xlogrecord.h

   1 /*
   2  * xlogrecord.h
   3  *
   4  * Definitions for the WAL record format.
   5  *
   6  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
   7  * Portions Copyright (c) 1994, Regents of the University of California
   8  *
   9  * src/include/access/xlogrecord.h
  10  */
  11 #ifndef XLOGRECORD_H
  12 #define XLOGRECORD_H
  13
  14 #include "access/rmgr.h"
  15 #include "access/xlogdefs.h"
  16 #include "port/pg_crc32c.h"
  17 #include "storage/block.h"
  18 #include "storage/relfilelocator.h"
  19
  20 /*
  21  * The overall layout of an XLOG record is:
  22  *              Fixed-size header (XLogRecord struct)
  23  *              XLogRecordBlockHeader struct
  24  *              XLogRecordBlockHeader struct
  25  *              ...
  26  *              XLogRecordDataHeader[Short|Long] struct
  27  *              block data
  28  *              block data
  29  *              ...
  30  *              main data
  31  *
  32  * There can be zero or more XLogRecordBlockHeaders, and 0 or more bytes of
  33  * rmgr-specific data not associated with a block.  XLogRecord structs
  34  * always start on MAXALIGN boundaries in the WAL files, but the rest of
  35  * the fields are not aligned.
  36  *
  37  * The XLogRecordBlockHeader, XLogRecordDataHeaderShort and
  38  * XLogRecordDataHeaderLong structs all begin with a single 'id' byte. It's
  39  * used to distinguish between block references, and the main data structs.
  40  */
  41 typedef struct XLogRecord
  42 {
  43         uint32          xl_tot_len;             /* total len of entire record */
  44         TransactionId xl_xid;           /* xact id */
  45         XLogRecPtr      xl_prev;                /* ptr to previous record in log */
  46         uint8           xl_info;                /* flag bits, see below */
  47         RmgrId          xl_rmid;                /* resource manager for this record */
  48         /* 2 bytes of padding here, initialize to zero */
  49         pg_crc32c       xl_crc;                 /* CRC for this record */
  50
  51         /* XLogRecordBlockHeaders and XLogRecordDataHeader follow, no padding */
  52
  53 } XLogRecord;
  54
  55 #define SizeOfXLogRecord        (offsetof(XLogRecord, xl_crc) + sizeof(pg_crc32c))
  56
  57 /*
  58  * The high 4 bits in xl_info may be used freely by rmgr. The
  59  * XLR_SPECIAL_REL_UPDATE and XLR_CHECK_CONSISTENCY bits can be passed by
  60  * XLogInsert caller. The rest are set internally by XLogInsert.
  61  */
  62 #define XLR_INFO_MASK                   0x0F
  63 #define XLR_RMGR_INFO_MASK              0xF0
  64
  65 /*
  66  * XLogReader needs to allocate all the data of a WAL record in a single
  67  * chunk.  This means that a single XLogRecord cannot exceed MaxAllocSize
  68  * in length if we ignore any allocation overhead of the XLogReader.
  69  *
  70  * To accommodate some overhead, this value allows for 4M of allocation
  71  * overhead, that should be plenty enough for what the XLogReader
  72  * infrastructure expects as extra.
  73  */
  74 #define XLogRecordMaxSize       (1020 * 1024 * 1024)
  75
  76 /*
  77  * If a WAL record modifies any relation files, in ways not covered by the
  78  * usual block references, this flag is set. This is not used for anything
  79  * by PostgreSQL itself, but it allows external tools that read WAL and keep
  80  * track of modified blocks to recognize such special record types.
  81  */
  82 #define XLR_SPECIAL_REL_UPDATE  0x01
  83
  84 /*
  85  * Enforces consistency checks of replayed WAL at recovery. If enabled,
  86  * each record will log a full-page write for each block modified by the
  87  * record and will reuse it afterwards for consistency checks. The caller
  88  * of XLogInsert can use this value if necessary, but if
  89  * wal_consistency_checking is enabled for a rmgr this is set unconditionally.
  90  */
  91 #define XLR_CHECK_CONSISTENCY   0x02
  92
  93 /*
  94  * Header info for block data appended to an XLOG record.
  95  *
  96  * 'data_length' is the length of the rmgr-specific payload data associated
  97  * with this block. It does not include the possible full page image, nor
  98  * XLogRecordBlockHeader struct itself.
  99  *
 100  * Note that we don't attempt to align the XLogRecordBlockHeader struct!
 101  * So, the struct must be copied to aligned local storage before use.
 102  */
 103 typedef struct XLogRecordBlockHeader
 104 {
 105         uint8           id;                             /* block reference ID */
 106         uint8           fork_flags;             /* fork within the relation, and flags */
 107         uint16          data_length;    /* number of payload bytes (not including page
 108                                                                  * image) */
 109
 110         /* If BKPBLOCK_HAS_IMAGE, an XLogRecordBlockImageHeader struct follows */
 111         /* If BKPBLOCK_SAME_REL is not set, a RelFileLocator follows */
 112         /* BlockNumber follows */
 113 } XLogRecordBlockHeader;
 114
 115 #define SizeOfXLogRecordBlockHeader (offsetof(XLogRecordBlockHeader, data_length) + sizeof(uint16))
 116
 117 /*
 118  * Additional header information when a full-page image is included
 119  * (i.e. when BKPBLOCK_HAS_IMAGE is set).
 120  *
 121  * The XLOG code is aware that PG data pages usually contain an unused "hole"
 122  * in the middle, which contains only zero bytes.  Since we know that the
 123  * "hole" is all zeros, we remove it from the stored data (and it's not counted
 124  * in the XLOG record's CRC, either).  Hence, the amount of block data actually
 125  * present is (BLCKSZ - <length of "hole" bytes>).
 126  *
 127  * Additionally, when wal_compression is enabled, we will try to compress full
 128  * page images using one of the supported algorithms, after removing the
 129  * "hole". This can reduce the WAL volume, but at some extra cost of CPU spent
 130  * on the compression during WAL logging. In this case, since the "hole"
 131  * length cannot be calculated by subtracting the number of page image bytes
 132  * from BLCKSZ, basically it needs to be stored as an extra information.
 133  * But when no "hole" exists, we can assume that the "hole" length is zero
 134  * and no such an extra information needs to be stored. Note that
 135  * the original version of page image is stored in WAL instead of the
 136  * compressed one if the number of bytes saved by compression is less than
 137  * the length of extra information. Hence, when a page image is successfully
 138  * compressed, the amount of block data actually present is less than
 139  * BLCKSZ - the length of "hole" bytes - the length of extra information.
 140  */
 141 typedef struct XLogRecordBlockImageHeader
 142 {
 143         uint16          length;                 /* number of page image bytes */
 144         uint16          hole_offset;    /* number of bytes before "hole" */
 145         uint8           bimg_info;              /* flag bits, see below */
 146
 147         /*
 148          * If BKPIMAGE_HAS_HOLE and BKPIMAGE_COMPRESSED(), an
 149          * XLogRecordBlockCompressHeader struct follows.
 150          */
 151 } XLogRecordBlockImageHeader;
 152
 153 #define SizeOfXLogRecordBlockImageHeader        \
 154         (offsetof(XLogRecordBlockImageHeader, bimg_info) + sizeof(uint8))
 155
 156 /* Information stored in bimg_info */
 157 #define BKPIMAGE_HAS_HOLE               0x01    /* page image has "hole" */
 158 #define BKPIMAGE_APPLY                  0x02    /* page image should be restored
 159                                                                                  * during replay */
 160 /* compression methods supported */
 161 #define BKPIMAGE_COMPRESS_PGLZ  0x04
 162 #define BKPIMAGE_COMPRESS_LZ4   0x08
 163 #define BKPIMAGE_COMPRESS_ZSTD  0x10
 164
 165 #define BKPIMAGE_COMPRESSED(info) \
 166         ((info & (BKPIMAGE_COMPRESS_PGLZ | BKPIMAGE_COMPRESS_LZ4 | \
 167                           BKPIMAGE_COMPRESS_ZSTD)) != 0)
 168
 169 /*
 170  * Extra header information used when page image has "hole" and
 171  * is compressed.
 172  */
 173 typedef struct XLogRecordBlockCompressHeader
 174 {
 175         uint16          hole_length;    /* number of bytes in "hole" */
 176 } XLogRecordBlockCompressHeader;
 177
 178 #define SizeOfXLogRecordBlockCompressHeader \
 179         sizeof(XLogRecordBlockCompressHeader)
 180
 181 /*
 182  * Maximum size of the header for a block reference. This is used to size a
 183  * temporary buffer for constructing the header.
 184  */
 185 #define MaxSizeOfXLogRecordBlockHeader \
 186         (SizeOfXLogRecordBlockHeader + \
 187          SizeOfXLogRecordBlockImageHeader + \
 188          SizeOfXLogRecordBlockCompressHeader + \
 189          sizeof(RelFileLocator) + \
 190          sizeof(BlockNumber))
 191
 192 /*
 193  * The fork number fits in the lower 4 bits in the fork_flags field. The upper
 194  * bits are used for flags.
 195  */
 196 #define BKPBLOCK_FORK_MASK      0x0F
 197 #define BKPBLOCK_FLAG_MASK      0xF0
 198 #define BKPBLOCK_HAS_IMAGE      0x10    /* block data is an XLogRecordBlockImage */
 199 #define BKPBLOCK_HAS_DATA       0x20
 200 #define BKPBLOCK_WILL_INIT      0x40    /* redo will re-init the page */
 201 #define BKPBLOCK_SAME_REL       0x80    /* RelFileLocator omitted, same as
 202                                                                          * previous */
 203
 204 /*
 205  * XLogRecordDataHeaderShort/Long are used for the "main data" portion of
 206  * the record. If the length of the data is less than 256 bytes, the short
 207  * form is used, with a single byte to hold the length. Otherwise the long
 208  * form is used.
 209  *
 210  * (These structs are currently not used in the code, they are here just for
 211  * documentation purposes).
 212  */
 213 typedef struct XLogRecordDataHeaderShort
 214 {
 215         uint8           id;                             /* XLR_BLOCK_ID_DATA_SHORT */
 216         uint8           data_length;    /* number of payload bytes */
 217 }                       XLogRecordDataHeaderShort;
 218
 219 #define SizeOfXLogRecordDataHeaderShort (sizeof(uint8) * 2)
 220
 221 typedef struct XLogRecordDataHeaderLong
 222 {
 223         uint8           id;                             /* XLR_BLOCK_ID_DATA_LONG */
 224         /* followed by uint32 data_length, unaligned */
 225 }                       XLogRecordDataHeaderLong;
 226
 227 #define SizeOfXLogRecordDataHeaderLong (sizeof(uint8) + sizeof(uint32))
 228
 229 /*
 230  * Block IDs used to distinguish different kinds of record fragments. Block
 231  * references are numbered from 0 to XLR_MAX_BLOCK_ID. A rmgr is free to use
 232  * any ID number in that range (although you should stick to small numbers,
 233  * because the WAL machinery is optimized for that case). A few ID
 234  * numbers are reserved to denote the "main" data portion of the record,
 235  * as well as replication-supporting transaction metadata.
 236  *
 237  * The maximum is currently set at 32, quite arbitrarily. Most records only
 238  * need a handful of block references, but there are a few exceptions that
 239  * need more.
 240  */
 241 #define XLR_MAX_BLOCK_ID                        32
 242
 243 #define XLR_BLOCK_ID_DATA_SHORT         255
 244 #define XLR_BLOCK_ID_DATA_LONG          254
 245 #define XLR_BLOCK_ID_ORIGIN                     253
 246 #define XLR_BLOCK_ID_TOPLEVEL_XID       252
 247
 248 #endif                                                  /* XLOGRECORD_H */