src/btreeInt.h

   1 /*
   2 ** 2004 April 6
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 *************************************************************************
  12 ** This file implements an external (disk-based) database using BTrees.
  13 ** For a detailed discussion of BTrees, refer to
  14 **
  15 **     Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
  16 **     "Sorting And Searching", pages 473-480. Addison-Wesley
  17 **     Publishing Company, Reading, Massachusetts.
  18 **
  19 ** The basic idea is that each page of the file contains N database
  20 ** entries and N+1 pointers to subpages.
  21 **
  22 **   ----------------------------------------------------------------
  23 **   |  Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) |
  24 **   ----------------------------------------------------------------
  25 **
  26 ** All of the keys on the page that Ptr(0) points to have values less
  27 ** than Key(0).  All of the keys on page Ptr(1) and its subpages have
  28 ** values greater than Key(0) and less than Key(1).  All of the keys
  29 ** on Ptr(N) and its subpages have values greater than Key(N-1).  And
  30 ** so forth.
  31 **
  32 ** Finding a particular key requires reading O(log(M)) pages from the
  33 ** disk where M is the number of entries in the tree.
  34 **
  35 ** In this implementation, a single file can hold one or more separate
  36 ** BTrees.  Each BTree is identified by the index of its root page.  The
  37 ** key and data for any entry are combined to form the "payload".  A
  38 ** fixed amount of payload can be carried directly on the database
  39 ** page.  If the payload is larger than the preset amount then surplus
  40 ** bytes are stored on overflow pages.  The payload for an entry
  41 ** and the preceding pointer are combined to form a "Cell".  Each
  42 ** page has a small header which contains the Ptr(N) pointer and other
  43 ** information such as the size of key and data.
  44 **
  45 ** FORMAT DETAILS
  46 **
  47 ** The file is divided into pages.  The first page is called page 1,
  48 ** the second is page 2, and so forth.  A page number of zero indicates
  49 ** "no such page".  The page size can be any power of 2 between 512 and 65536.
  50 ** Each page can be either a btree page, a freelist page, an overflow
  51 ** page, or a pointer-map page.
  52 **
  53 ** The first page is always a btree page.  The first 100 bytes of the first
  54 ** page contain a special header (the "file header") that describes the file.
  55 ** The format of the file header is as follows:
  56 **
  57 **   OFFSET   SIZE    DESCRIPTION
  58 **      0      16     Header string: "SQLite format 3\000"
  59 **     16       2     Page size in bytes.  (1 means 65536)
  60 **     18       1     File format write version
  61 **     19       1     File format read version
  62 **     20       1     Bytes of unused space at the end of each page
  63 **     21       1     Max embedded payload fraction (must be 64)
  64 **     22       1     Min embedded payload fraction (must be 32)
  65 **     23       1     Min leaf payload fraction (must be 32)
  66 **     24       4     File change counter
  67 **     28       4     The size of the database in pages
  68 **     32       4     First freelist page
  69 **     36       4     Number of freelist pages in the file
  70 **     40      60     15 4-byte meta values passed to higher layers
  71 **
  72 **     40       4     Schema cookie
  73 **     44       4     File format of schema layer
  74 **     48       4     Size of page cache
  75 **     52       4     Largest root-page (auto/incr_vacuum)
  76 **     56       4     1=UTF-8 2=UTF16le 3=UTF16be
  77 **     60       4     User version
  78 **     64       4     Incremental vacuum mode
  79 **     68       4     Application-ID
  80 **     72      20     unused
  81 **     92       4     The version-valid-for number
  82 **     96       4     SQLITE_VERSION_NUMBER
  83 **
  84 ** All of the integer values are big-endian (most significant byte first).
  85 **
  86 ** The file change counter is incremented when the database is changed
  87 ** This counter allows other processes to know when the file has changed
  88 ** and thus when they need to flush their cache.
  89 **
  90 ** The max embedded payload fraction is the amount of the total usable
  91 ** space in a page that can be consumed by a single cell for standard
  92 ** B-tree (non-LEAFDATA) tables.  A value of 255 means 100%.  The default
  93 ** is to limit the maximum cell size so that at least 4 cells will fit
  94 ** on one page.  Thus the default max embedded payload fraction is 64.
  95 **
  96 ** If the payload for a cell is larger than the max payload, then extra
  97 ** payload is spilled to overflow pages.  Once an overflow page is allocated,
  98 ** as many bytes as possible are moved into the overflow pages without letting
  99 ** the cell size drop below the min embedded payload fraction.
 100 **
 101 ** The min leaf payload fraction is like the min embedded payload fraction
 102 ** except that it applies to leaf nodes in a LEAFDATA tree.  The maximum
 103 ** payload fraction for a LEAFDATA tree is always 100% (or 255) and it
 104 ** not specified in the header.
 105 **
 106 ** Each btree pages is divided into three sections:  The header, the
 107 ** cell pointer array, and the cell content area.  Page 1 also has a 100-byte
 108 ** file header that occurs before the page header.
 109 **
 110 **      |----------------|
 111 **      | file header    |   100 bytes.  Page 1 only.
 112 **      |----------------|
 113 **      | page header    |   8 bytes for leaves.  12 bytes for interior nodes
 114 **      |----------------|
 115 **      | cell pointer   |   |  2 bytes per cell.  Sorted order.
 116 **      | array          |   |  Grows downward
 117 **      |                |   v
 118 **      |----------------|
 119 **      | unallocated    |
 120 **      | space          |
 121 **      |----------------|   ^  Grows upwards
 122 **      | cell content   |   |  Arbitrary order interspersed with freeblocks.
 123 **      | area           |   |  and free space fragments.
 124 **      |----------------|
 125 **
 126 ** The page headers looks like this:
 127 **
 128 **   OFFSET   SIZE     DESCRIPTION
 129 **      0       1      Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf
 130 **      1       2      byte offset to the first freeblock
 131 **      3       2      number of cells on this page
 132 **      5       2      first byte of the cell content area
 133 **      7       1      number of fragmented free bytes
 134 **      8       4      Right child (the Ptr(N) value).  Omitted on leaves.
 135 **
 136 ** The flags define the format of this btree page.  The leaf flag means that
 137 ** this page has no children.  The zerodata flag means that this page carries
 138 ** only keys and no data.  The intkey flag means that the key is an integer
 139 ** which is stored in the key size entry of the cell header rather than in
 140 ** the payload area.
 141 **
 142 ** The cell pointer array begins on the first byte after the page header.
 143 ** The cell pointer array contains zero or more 2-byte numbers which are
 144 ** offsets from the beginning of the page to the cell content in the cell
 145 ** content area.  The cell pointers occur in sorted order.  The system strives
 146 ** to keep free space after the last cell pointer so that new cells can
 147 ** be easily added without having to defragment the page.
 148 **
 149 ** Cell content is stored at the very end of the page and grows toward the
 150 ** beginning of the page.
 151 **
 152 ** Unused space within the cell content area is collected into a linked list of
 153 ** freeblocks.  Each freeblock is at least 4 bytes in size.  The byte offset
 154 ** to the first freeblock is given in the header.  Freeblocks occur in
 155 ** increasing order.  Because a freeblock must be at least 4 bytes in size,
 156 ** any group of 3 or fewer unused bytes in the cell content area cannot
 157 ** exist on the freeblock chain.  A group of 3 or fewer free bytes is called
 158 ** a fragment.  The total number of bytes in all fragments is recorded.
 159 ** in the page header at offset 7.
 160 **
 161 **    SIZE    DESCRIPTION
 162 **      2     Byte offset of the next freeblock
 163 **      2     Bytes in this freeblock
 164 **
 165 ** Cells are of variable length.  Cells are stored in the cell content area at
 166 ** the end of the page.  Pointers to the cells are in the cell pointer array
 167 ** that immediately follows the page header.  Cells is not necessarily
 168 ** contiguous or in order, but cell pointers are contiguous and in order.
 169 **
 170 ** Cell content makes use of variable length integers.  A variable
 171 ** length integer is 1 to 9 bytes where the lower 7 bits of each
 172 ** byte are used.  The integer consists of all bytes that have bit 8 set and
 173 ** the first byte with bit 8 clear.  The most significant byte of the integer
 174 ** appears first.  A variable-length integer may not be more than 9 bytes long.
 175 ** As a special case, all 8 bits of the 9th byte are used as data.  This
 176 ** allows a 64-bit integer to be encoded in 9 bytes.
 177 **
 178 **    0x00                      becomes  0x00000000
 179 **    0x7f                      becomes  0x0000007f
 180 **    0x81 0x00                 becomes  0x00000080
 181 **    0x82 0x00                 becomes  0x00000100
 182 **    0x80 0x7f                 becomes  0x0000007f
 183 **    0x81 0x91 0xd1 0xac 0x78  becomes  0x12345678
 184 **    0x81 0x81 0x81 0x81 0x01  becomes  0x10204081
 185 **
 186 ** Variable length integers are used for rowids and to hold the number of
 187 ** bytes of key and data in a btree cell.
 188 **
 189 ** The content of a cell looks like this:
 190 **
 191 **    SIZE    DESCRIPTION
 192 **      4     Page number of the left child. Omitted if leaf flag is set.
 193 **     var    Number of bytes of data. Omitted if the zerodata flag is set.
 194 **     var    Number of bytes of key. Or the key itself if intkey flag is set.
 195 **      *     Payload
 196 **      4     First page of the overflow chain.  Omitted if no overflow
 197 **
 198 ** Overflow pages form a linked list.  Each page except the last is completely
 199 ** filled with data (pagesize - 4 bytes).  The last page can have as little
 200 ** as 1 byte of data.
 201 **
 202 **    SIZE    DESCRIPTION
 203 **      4     Page number of next overflow page
 204 **      *     Data
 205 **
 206 ** Freelist pages come in two subtypes: trunk pages and leaf pages.  The
 207 ** file header points to the first in a linked list of trunk page.  Each trunk
 208 ** page points to multiple leaf pages.  The content of a leaf page is
 209 ** unspecified.  A trunk page looks like this:
 210 **
 211 **    SIZE    DESCRIPTION
 212 **      4     Page number of next trunk page
 213 **      4     Number of leaf pointers on this page
 214 **      *     zero or more pages numbers of leaves
 215 */
 216 #include "sqliteInt.h"
 217
 218
 219 /* The following value is the maximum cell size assuming a maximum page
 220 ** size give above.
 221 */
 222 #define MX_CELL_SIZE(pBt)  ((int)(pBt->pageSize-8))
 223
 224 /* The maximum number of cells on a single page of the database.  This
 225 ** assumes a minimum cell size of 6 bytes  (4 bytes for the cell itself
 226 ** plus 2 bytes for the index to the cell in the page header).  Such
 227 ** small cells will be rare, but they are possible.
 228 */
 229 #define MX_CELL(pBt) ((pBt->pageSize-8)/6)
 230
 231 /* Forward declarations */
 232 typedef struct MemPage MemPage;
 233 typedef struct BtLock BtLock;
 234 typedef struct CellInfo CellInfo;
 235
 236 /*
 237 ** This is a magic string that appears at the beginning of every
 238 ** SQLite database in order to identify the file as a real database.
 239 **
 240 ** You can change this value at compile-time by specifying a
 241 ** -DSQLITE_FILE_HEADER="..." on the compiler command-line.  The
 242 ** header must be exactly 16 bytes including the zero-terminator so
 243 ** the string itself should be 15 characters long.  If you change
 244 ** the header, then your custom library will not be able to read
 245 ** databases generated by the standard tools and the standard tools
 246 ** will not be able to read databases created by your custom library.
 247 */
 248 #ifndef SQLITE_FILE_HEADER /* 123456789 123456 */
 249 #  define SQLITE_FILE_HEADER "SQLite format 3"
 250 #endif
 251
 252 /*
 253 ** Page type flags.  An ORed combination of these flags appear as the
 254 ** first byte of on-disk image of every BTree page.
 255 */
 256 #define PTF_INTKEY    0x01
 257 #define PTF_ZERODATA  0x02
 258 #define PTF_LEAFDATA  0x04
 259 #define PTF_LEAF      0x08
 260
 261 /*
 262 ** An instance of this object stores information about each a single database
 263 ** page that has been loaded into memory.  The information in this object
 264 ** is derived from the raw on-disk page content.
 265 **
 266 ** As each database page is loaded into memory, the pager allocates an
 267 ** instance of this object and zeros the first 8 bytes.  (This is the
 268 ** "extra" information associated with each page of the pager.)
 269 **
 270 ** Access to all fields of this structure is controlled by the mutex
 271 ** stored in MemPage.pBt->mutex.
 272 */
 273 struct MemPage {
 274   u8 isInit;           /* True if previously initialized. MUST BE FIRST! */
 275   u8 intKey;           /* True if table b-trees.  False for index b-trees */
 276   u8 intKeyLeaf;       /* True if the leaf of an intKey table */
 277   Pgno pgno;           /* Page number for this page */
 278   /* Only the first 8 bytes (above) are zeroed by pager.c when a new page
 279   ** is allocated. All fields that follow must be initialized before use */
 280   u8 leaf;             /* True if a leaf page */
 281   u8 hdrOffset;        /* 100 for page 1.  0 otherwise */
 282   u8 childPtrSize;     /* 0 if leaf==1.  4 if leaf==0 */
 283   u8 max1bytePayload;  /* min(maxLocal,127) */
 284   u8 nOverflow;        /* Number of overflow cell bodies in aCell[] */
 285   u16 maxLocal;        /* Copy of BtShared.maxLocal or BtShared.maxLeaf */
 286   u16 minLocal;        /* Copy of BtShared.minLocal or BtShared.minLeaf */
 287   u16 cellOffset;      /* Index in aData of first cell pointer */
 288   int nFree;           /* Number of free bytes on the page. -1 for unknown */
 289   u16 nCell;           /* Number of cells on this page, local and ovfl */
 290   u16 maskPage;        /* Mask for page offset */
 291   u16 aiOvfl[4];       /* Insert the i-th overflow cell before the aiOvfl-th
 292                        ** non-overflow cell */
 293   u8 *apOvfl[4];       /* Pointers to the body of overflow cells */
 294   BtShared *pBt;       /* Pointer to BtShared that this page is part of */
 295   u8 *aData;           /* Pointer to disk image of the page data */
 296   u8 *aDataEnd;        /* One byte past the end of the entire page - not just
 297                        ** the usable space, the entire page.  Used to prevent
 298                        ** corruption-induced buffer overflow. */
 299   u8 *aCellIdx;        /* The cell index area */
 300   u8 *aDataOfst;       /* Same as aData for leaves.  aData+4 for interior */
 301   DbPage *pDbPage;     /* Pager page handle */
 302   u16 (*xCellSize)(MemPage*,u8*);             /* cellSizePtr method */
 303   void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */
 304 };
 305
 306 /*
 307 ** A linked list of the following structures is stored at BtShared.pLock.
 308 ** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor
 309 ** is opened on the table with root page BtShared.iTable. Locks are removed
 310 ** from this list when a transaction is committed or rolled back, or when
 311 ** a btree handle is closed.
 312 */
 313 struct BtLock {
 314   Btree *pBtree;        /* Btree handle holding this lock */
 315   Pgno iTable;          /* Root page of table */
 316   u8 eLock;             /* READ_LOCK or WRITE_LOCK */
 317   BtLock *pNext;        /* Next in BtShared.pLock list */
 318 };
 319
 320 /* Candidate values for BtLock.eLock */
 321 #define READ_LOCK     1
 322 #define WRITE_LOCK    2
 323
 324 /* A Btree handle
 325 **
 326 ** A database connection contains a pointer to an instance of
 327 ** this object for every database file that it has open.  This structure
 328 ** is opaque to the database connection.  The database connection cannot
 329 ** see the internals of this structure and only deals with pointers to
 330 ** this structure.
 331 **
 332 ** For some database files, the same underlying database cache might be
 333 ** shared between multiple connections.  In that case, each connection
 334 ** has it own instance of this object.  But each instance of this object
 335 ** points to the same BtShared object.  The database cache and the
 336 ** schema associated with the database file are all contained within
 337 ** the BtShared object.
 338 **
 339 ** All fields in this structure are accessed under sqlite3.mutex.
 340 ** The pBt pointer itself may not be changed while there exists cursors
 341 ** in the referenced BtShared that point back to this Btree since those
 342 ** cursors have to go through this Btree to find their BtShared and
 343 ** they often do so without holding sqlite3.mutex.
 344 */
 345 struct Btree {
 346   sqlite3 *db;       /* The database connection holding this btree */
 347   BtShared *pBt;     /* Sharable content of this btree */
 348   u8 inTrans;        /* TRANS_NONE, TRANS_READ or TRANS_WRITE */
 349   u8 sharable;       /* True if we can share pBt with another db */
 350   u8 locked;         /* True if db currently has pBt locked */
 351   u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */
 352   int wantToLock;    /* Number of nested calls to sqlite3BtreeEnter() */
 353   int nBackup;       /* Number of backup operations reading this btree */
 354   u32 iBDataVersion; /* Combines with pBt->pPager->iDataVersion */
 355   Btree *pNext;      /* List of other sharable Btrees from the same db */
 356   Btree *pPrev;      /* Back pointer of the same list */
 357 #ifdef SQLITE_DEBUG
 358   u64 nSeek;         /* Calls to sqlite3BtreeMovetoUnpacked() */
 359 #endif
 360 #ifndef SQLITE_OMIT_SHARED_CACHE
 361   BtLock lock;       /* Object used to lock page 1 */
 362 #endif
 363 };
 364
 365 /*
 366 ** Btree.inTrans may take one of the following values.
 367 **
 368 ** If the shared-data extension is enabled, there may be multiple users
 369 ** of the Btree structure. At most one of these may open a write transaction,
 370 ** but any number may have active read transactions.
 371 **
 372 ** These values must match SQLITE_TXN_NONE, SQLITE_TXN_READ, and
 373 ** SQLITE_TXN_WRITE
 374 */
 375 #define TRANS_NONE  0
 376 #define TRANS_READ  1
 377 #define TRANS_WRITE 2
 378
 379 #if TRANS_NONE!=SQLITE_TXN_NONE
 380 # error wrong numeric code for no-transaction
 381 #endif
 382 #if TRANS_READ!=SQLITE_TXN_READ
 383 # error wrong numeric code for read-transaction
 384 #endif
 385 #if TRANS_WRITE!=SQLITE_TXN_WRITE
 386 # error wrong numeric code for write-transaction
 387 #endif
 388
 389
 390 /*
 391 ** An instance of this object represents a single database file.
 392 **
 393 ** A single database file can be in use at the same time by two
 394 ** or more database connections.  When two or more connections are
 395 ** sharing the same database file, each connection has it own
 396 ** private Btree object for the file and each of those Btrees points
 397 ** to this one BtShared object.  BtShared.nRef is the number of
 398 ** connections currently sharing this database file.
 399 **
 400 ** Fields in this structure are accessed under the BtShared.mutex
 401 ** mutex, except for nRef and pNext which are accessed under the
 402 ** global SQLITE_MUTEX_STATIC_MAIN mutex.  The pPager field
 403 ** may not be modified once it is initially set as long as nRef>0.
 404 ** The pSchema field may be set once under BtShared.mutex and
 405 ** thereafter is unchanged as long as nRef>0.
 406 **
 407 ** isPending:
 408 **
 409 **   If a BtShared client fails to obtain a write-lock on a database
 410 **   table (because there exists one or more read-locks on the table),
 411 **   the shared-cache enters 'pending-lock' state and isPending is
 412 **   set to true.
 413 **
 414 **   The shared-cache leaves the 'pending lock' state when either of
 415 **   the following occur:
 416 **
 417 **     1) The current writer (BtShared.pWriter) concludes its transaction, OR
 418 **     2) The number of locks held by other connections drops to zero.
 419 **
 420 **   while in the 'pending-lock' state, no connection may start a new
 421 **   transaction.
 422 **
 423 **   This feature is included to help prevent writer-starvation.
 424 */
 425 struct BtShared {
 426   Pager *pPager;        /* The page cache */
 427   sqlite3 *db;          /* Database connection currently using this Btree */
 428   BtCursor *pCursor;    /* A list of all open cursors */
 429   MemPage *pPage1;      /* First page of the database */
 430   u8 openFlags;         /* Flags to sqlite3BtreeOpen() */
 431 #ifndef SQLITE_OMIT_AUTOVACUUM
 432   u8 autoVacuum;        /* True if auto-vacuum is enabled */
 433   u8 incrVacuum;        /* True if incr-vacuum is enabled */
 434   u8 bDoTruncate;       /* True to truncate db on commit */
 435 #endif
 436   u8 inTransaction;     /* Transaction state */
 437   u8 max1bytePayload;   /* Maximum first byte of cell for a 1-byte payload */
 438   u8 nReserveWanted;    /* Desired number of extra bytes per page */
 439   u16 btsFlags;         /* Boolean parameters.  See BTS_* macros below */
 440   u16 maxLocal;         /* Maximum local payload in non-LEAFDATA tables */
 441   u16 minLocal;         /* Minimum local payload in non-LEAFDATA tables */
 442   u16 maxLeaf;          /* Maximum local payload in a LEAFDATA table */
 443   u16 minLeaf;          /* Minimum local payload in a LEAFDATA table */
 444   u32 pageSize;         /* Total number of bytes on a page */
 445   u32 usableSize;       /* Number of usable bytes on each page */
 446   int nTransaction;     /* Number of open transactions (read + write) */
 447   u32 nPage;            /* Number of pages in the database */
 448   void *pSchema;        /* Pointer to space allocated by sqlite3BtreeSchema() */
 449   void (*xFreeSchema)(void*);  /* Destructor for BtShared.pSchema */
 450   sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */
 451   Bitvec *pHasContent;  /* Set of pages moved to free-list this transaction */
 452 #ifndef SQLITE_OMIT_SHARED_CACHE
 453   int nRef;             /* Number of references to this structure */
 454   BtShared *pNext;      /* Next on a list of sharable BtShared structs */
 455   BtLock *pLock;        /* List of locks held on this shared-btree struct */
 456   Btree *pWriter;       /* Btree with currently open write transaction */
 457 #endif
 458   u8 *pTmpSpace;        /* Temp space sufficient to hold a single cell */
 459   int nPreformatSize;   /* Size of last cell written by TransferRow() */
 460 };
 461
 462 /*
 463 ** Allowed values for BtShared.btsFlags
 464 */
 465 #define BTS_READ_ONLY        0x0001   /* Underlying file is readonly */
 466 #define BTS_PAGESIZE_FIXED   0x0002   /* Page size can no longer be changed */
 467 #define BTS_SECURE_DELETE    0x0004   /* PRAGMA secure_delete is enabled */
 468 #define BTS_OVERWRITE        0x0008   /* Overwrite deleted content with zeros */
 469 #define BTS_FAST_SECURE      0x000c   /* Combination of the previous two */
 470 #define BTS_INITIALLY_EMPTY  0x0010   /* Database was empty at trans start */
 471 #define BTS_NO_WAL           0x0020   /* Do not open write-ahead-log files */
 472 #define BTS_EXCLUSIVE        0x0040   /* pWriter has an exclusive lock */
 473 #define BTS_PENDING          0x0080   /* Waiting for read-locks to clear */
 474
 475 /*
 476 ** An instance of the following structure is used to hold information
 477 ** about a cell.  The parseCellPtr() function fills in this structure
 478 ** based on information extract from the raw disk page.
 479 */
 480 struct CellInfo {
 481   i64 nKey;      /* The key for INTKEY tables, or nPayload otherwise */
 482   u8 *pPayload;  /* Pointer to the start of payload */
 483   u32 nPayload;  /* Bytes of payload */
 484   u16 nLocal;    /* Amount of payload held locally, not on overflow */
 485   u16 nSize;     /* Size of the cell content on the main b-tree page */
 486 };
 487
 488 /*
 489 ** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than
 490 ** this will be declared corrupt. This value is calculated based on a
 491 ** maximum database size of 2^31 pages a minimum fanout of 2 for a
 492 ** root-node and 3 for all other internal nodes.
 493 **
 494 ** If a tree that appears to be taller than this is encountered, it is
 495 ** assumed that the database is corrupt.
 496 */
 497 #define BTCURSOR_MAX_DEPTH 20
 498
 499 /*
 500 ** A cursor is a pointer to a particular entry within a particular
 501 ** b-tree within a database file.
 502 **
 503 ** The entry is identified by its MemPage and the index in
 504 ** MemPage.aCell[] of the entry.
 505 **
 506 ** A single database file can be shared by two more database connections,
 507 ** but cursors cannot be shared.  Each cursor is associated with a
 508 ** particular database connection identified BtCursor.pBtree.db.
 509 **
 510 ** Fields in this structure are accessed under the BtShared.mutex
 511 ** found at self->pBt->mutex.
 512 **
 513 ** skipNext meaning:
 514 ** The meaning of skipNext depends on the value of eState:
 515 **
 516 **   eState            Meaning of skipNext
 517 **   VALID             skipNext is meaningless and is ignored
 518 **   INVALID           skipNext is meaningless and is ignored
 519 **   SKIPNEXT          sqlite3BtreeNext() is a no-op if skipNext>0 and
 520 **                     sqlite3BtreePrevious() is no-op if skipNext<0.
 521 **   REQUIRESEEK       restoreCursorPosition() restores the cursor to
 522 **                     eState=SKIPNEXT if skipNext!=0
 523 **   FAULT             skipNext holds the cursor fault error code.
 524 */
 525 struct BtCursor {
 526   u8 eState;                /* One of the CURSOR_XXX constants (see below) */
 527   u8 curFlags;              /* zero or more BTCF_* flags defined below */
 528   u8 curPagerFlags;         /* Flags to send to sqlite3PagerGet() */
 529   u8 hints;                 /* As configured by CursorSetHints() */
 530   int skipNext;    /* Prev() is noop if negative. Next() is noop if positive.
 531                    ** Error code if eState==CURSOR_FAULT */
 532   Btree *pBtree;            /* The Btree to which this cursor belongs */
 533   Pgno *aOverflow;          /* Cache of overflow page locations */
 534   void *pKey;               /* Saved key that was cursor last known position */
 535   /* All fields above are zeroed when the cursor is allocated.  See
 536   ** sqlite3BtreeCursorZero().  Fields that follow must be manually
 537   ** initialized. */
 538 #define BTCURSOR_FIRST_UNINIT pBt   /* Name of first uninitialized field */
 539   BtShared *pBt;            /* The BtShared this cursor points to */
 540   BtCursor *pNext;          /* Forms a linked list of all cursors */
 541   CellInfo info;            /* A parse of the cell we are pointing at */
 542   i64 nKey;                 /* Size of pKey, or last integer key */
 543   Pgno pgnoRoot;            /* The root page of this tree */
 544   i8 iPage;                 /* Index of current page in apPage */
 545   u8 curIntKey;             /* Value of apPage[0]->intKey */
 546   u16 ix;                   /* Current index for apPage[iPage] */
 547   u16 aiIdx[BTCURSOR_MAX_DEPTH-1];     /* Current index in apPage[i] */
 548   struct KeyInfo *pKeyInfo;            /* Arg passed to comparison function */
 549   MemPage *pPage;                        /* Current page */
 550   MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */
 551 };
 552
 553 /*
 554 ** Legal values for BtCursor.curFlags
 555 */
 556 #define BTCF_WriteFlag    0x01   /* True if a write cursor */
 557 #define BTCF_ValidNKey    0x02   /* True if info.nKey is valid */
 558 #define BTCF_ValidOvfl    0x04   /* True if aOverflow is valid */
 559 #define BTCF_AtLast       0x08   /* Cursor is pointing to the last entry */
 560 #define BTCF_Incrblob     0x10   /* True if an incremental I/O handle */
 561 #define BTCF_Multiple     0x20   /* Maybe another cursor on the same btree */
 562 #define BTCF_Pinned       0x40   /* Cursor is busy and cannot be moved */
 563
 564 /*
 565 ** Potential values for BtCursor.eState.
 566 **
 567 ** CURSOR_INVALID:
 568 **   Cursor does not point to a valid entry. This can happen (for example)
 569 **   because the table is empty or because BtreeCursorFirst() has not been
 570 **   called.
 571 **
 572 ** CURSOR_VALID:
 573 **   Cursor points to a valid entry. getPayload() etc. may be called.
 574 **
 575 ** CURSOR_SKIPNEXT:
 576 **   Cursor is valid except that the Cursor.skipNext field is non-zero
 577 **   indicating that the next sqlite3BtreeNext() or sqlite3BtreePrevious()
 578 **   operation should be a no-op.
 579 **
 580 ** CURSOR_REQUIRESEEK:
 581 **   The table that this cursor was opened on still exists, but has been
 582 **   modified since the cursor was last used. The cursor position is saved
 583 **   in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in
 584 **   this state, restoreCursorPosition() can be called to attempt to
 585 **   seek the cursor to the saved position.
 586 **
 587 ** CURSOR_FAULT:
 588 **   An unrecoverable error (an I/O error or a malloc failure) has occurred
 589 **   on a different connection that shares the BtShared cache with this
 590 **   cursor.  The error has left the cache in an inconsistent state.
 591 **   Do nothing else with this cursor.  Any attempt to use the cursor
 592 **   should return the error code stored in BtCursor.skipNext
 593 */
 594 #define CURSOR_VALID             0
 595 #define CURSOR_INVALID           1
 596 #define CURSOR_SKIPNEXT          2
 597 #define CURSOR_REQUIRESEEK       3
 598 #define CURSOR_FAULT             4
 599
 600 /*
 601 ** The database page the PENDING_BYTE occupies. This page is never used.
 602 */
 603 #define PENDING_BYTE_PAGE(pBt)  ((Pgno)((PENDING_BYTE/((pBt)->pageSize))+1))
 604
 605 /*
 606 ** These macros define the location of the pointer-map entry for a
 607 ** database page. The first argument to each is the number of usable
 608 ** bytes on each page of the database (often 1024). The second is the
 609 ** page number to look up in the pointer map.
 610 **
 611 ** PTRMAP_PAGENO returns the database page number of the pointer-map
 612 ** page that stores the required pointer. PTRMAP_PTROFFSET returns
 613 ** the offset of the requested map entry.
 614 **
 615 ** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page,
 616 ** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be
 617 ** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements
 618 ** this test.
 619 */
 620 #define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno)
 621 #define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1))
 622 #define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno))
 623
 624 /*
 625 ** The pointer map is a lookup table that identifies the parent page for
 626 ** each child page in the database file.  The parent page is the page that
 627 ** contains a pointer to the child.  Every page in the database contains
 628 ** 0 or 1 parent pages.  (In this context 'database page' refers
 629 ** to any page that is not part of the pointer map itself.)  Each pointer map
 630 ** entry consists of a single byte 'type' and a 4 byte parent page number.
 631 ** The PTRMAP_XXX identifiers below are the valid types.
 632 **
 633 ** The purpose of the pointer map is to facility moving pages from one
 634 ** position in the file to another as part of autovacuum.  When a page
 635 ** is moved, the pointer in its parent must be updated to point to the
 636 ** new location.  The pointer map is used to locate the parent page quickly.
 637 **
 638 ** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not
 639 **                  used in this case.
 640 **
 641 ** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number
 642 **                  is not used in this case.
 643 **
 644 ** PTRMAP_OVERFLOW1: The database page is the first page in a list of
 645 **                   overflow pages. The page number identifies the page that
 646 **                   contains the cell with a pointer to this overflow page.
 647 **
 648 ** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of
 649 **                   overflow pages. The page-number identifies the previous
 650 **                   page in the overflow page list.
 651 **
 652 ** PTRMAP_BTREE: The database page is a non-root btree page. The page number
 653 **               identifies the parent page in the btree.
 654 */
 655 #define PTRMAP_ROOTPAGE 1
 656 #define PTRMAP_FREEPAGE 2
 657 #define PTRMAP_OVERFLOW1 3
 658 #define PTRMAP_OVERFLOW2 4
 659 #define PTRMAP_BTREE 5
 660
 661 /* A bunch of assert() statements to check the transaction state variables
 662 ** of handle p (type Btree*) are internally consistent.
 663 */
 664 #define btreeIntegrity(p) \
 665   assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \
 666   assert( p->pBt->inTransaction>=p->inTrans );
 667
 668
 669 /*
 670 ** The ISAUTOVACUUM macro is used within balance_nonroot() to determine
 671 ** if the database supports auto-vacuum or not. Because it is used
 672 ** within an expression that is an argument to another macro
 673 ** (sqliteMallocRaw), it is not possible to use conditional compilation.
 674 ** So, this macro is defined instead.
 675 */
 676 #ifndef SQLITE_OMIT_AUTOVACUUM
 677 #define ISAUTOVACUUM(pBt) (pBt->autoVacuum)
 678 #else
 679 #define ISAUTOVACUUM(pBt) 0
 680 #endif
 681
 682
 683 /*
 684 ** This structure is passed around through all the PRAGMA integrity_check
 685 ** checking routines in order to keep track of some global state information.
 686 **
 687 ** The aRef[] array is allocated so that there is 1 bit for each page in
 688 ** the database. As the integrity-check proceeds, for each page used in
 689 ** the database the corresponding bit is set. This allows integrity-check to
 690 ** detect pages that are used twice and orphaned pages (both of which
 691 ** indicate corruption).
 692 */
 693 typedef struct IntegrityCk IntegrityCk;
 694 struct IntegrityCk {
 695   BtShared *pBt;    /* The tree being checked out */
 696   Pager *pPager;    /* The associated pager.  Also accessible by pBt->pPager */
 697   u8 *aPgRef;       /* 1 bit per page in the db (see above) */
 698   Pgno nCkPage;     /* Pages in the database.  0 for partial check */
 699   int mxErr;        /* Stop accumulating errors when this reaches zero */
 700   int nErr;         /* Number of messages written to zErrMsg so far */
 701   int rc;           /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */
 702   u32 nStep;        /* Number of steps into the integrity_check process */
 703   const char *zPfx; /* Error message prefix */
 704   Pgno v0;          /* Value for first %u substitution in zPfx (root page) */
 705   Pgno v1;          /* Value for second %u substitution in zPfx (current pg) */
 706   int v2;           /* Value for third %d substitution in zPfx */
 707   StrAccum errMsg;  /* Accumulate the error message text here */
 708   u32 *heap;        /* Min-heap used for analyzing cell coverage */
 709   sqlite3 *db;      /* Database connection running the check */
 710 };
 711
 712 /*
 713 ** Routines to read or write a two- and four-byte big-endian integer values.
 714 */
 715 #define get2byte(x)   ((x)[0]<<8 | (x)[1])
 716 #define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v))
 717 #define get4byte sqlite3Get4byte
 718 #define put4byte sqlite3Put4byte
 719
 720 /*
 721 ** get2byteAligned(), unlike get2byte(), requires that its argument point to a
 722 ** two-byte aligned address.  get2byteAligned() is only used for accessing the
 723 ** cell addresses in a btree header.
 724 */
 725 #if SQLITE_BYTEORDER==4321
 726 # define get2byteAligned(x)  (*(u16*)(x))
 727 #elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000
 728 # define get2byteAligned(x)  __builtin_bswap16(*(u16*)(x))
 729 #elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300
 730 # define get2byteAligned(x)  _byteswap_ushort(*(u16*)(x))
 731 #else
 732 # define get2byteAligned(x)  ((x)[0]<<8 | (x)[1])
 733 #endif