ext/recover/dbdata.c

   1 /*
   2 ** 2019-04-17
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This file contains an implementation of two eponymous virtual tables,
  14 ** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the
  15 ** "sqlite_dbpage" eponymous virtual table be available.
  16 **
  17 ** SQLITE_DBDATA:
  18 **   sqlite_dbdata is used to extract data directly from a database b-tree
  19 **   page and its associated overflow pages, bypassing the b-tree layer.
  20 **   The table schema is equivalent to:
  21 **
  22 **     CREATE TABLE sqlite_dbdata(
  23 **       pgno INTEGER,
  24 **       cell INTEGER,
  25 **       field INTEGER,
  26 **       value ANY,
  27 **       schema TEXT HIDDEN
  28 **     );
  29 **
  30 **   IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE
  31 **   FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND
  32 **   "schema".
  33 **
  34 **   Each page of the database is inspected. If it cannot be interpreted as
  35 **   a b-tree page, or if it is a b-tree page containing 0 entries, the
  36 **   sqlite_dbdata table contains no rows for that page.  Otherwise, the
  37 **   table contains one row for each field in the record associated with
  38 **   each cell on the page. For intkey b-trees, the key value is stored in
  39 **   field -1.
  40 **
  41 **   For example, for the database:
  42 **
  43 **     CREATE TABLE t1(a, b);     -- root page is page 2
  44 **     INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five');
  45 **     INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten');
  46 **
  47 **   the sqlite_dbdata table contains, as well as from entries related to
  48 **   page 1, content equivalent to:
  49 **
  50 **     INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES
  51 **         (2, 0, -1, 5     ),
  52 **         (2, 0,  0, 'v'   ),
  53 **         (2, 0,  1, 'five'),
  54 **         (2, 1, -1, 10    ),
  55 **         (2, 1,  0, 'x'   ),
  56 **         (2, 1,  1, 'ten' );
  57 **
  58 **   If database corruption is encountered, this module does not report an
  59 **   error. Instead, it attempts to extract as much data as possible and
  60 **   ignores the corruption.
  61 **
  62 ** SQLITE_DBPTR:
  63 **   The sqlite_dbptr table has the following schema:
  64 **
  65 **     CREATE TABLE sqlite_dbptr(
  66 **       pgno INTEGER,
  67 **       child INTEGER,
  68 **       schema TEXT HIDDEN
  69 **     );
  70 **
  71 **   It contains one entry for each b-tree pointer between a parent and
  72 **   child page in the database.
  73 */
  74
  75 #if !defined(SQLITEINT_H)
  76 #include "sqlite3ext.h"
  77
  78 typedef unsigned char u8;
  79 typedef unsigned int u32;
  80
  81 #endif
  82 SQLITE_EXTENSION_INIT1
  83 #include <string.h>
  84 #include <assert.h>
  85
  86 #ifndef SQLITE_OMIT_VIRTUALTABLE
  87
  88 #define DBDATA_PADDING_BYTES 100
  89
  90 typedef struct DbdataTable DbdataTable;
  91 typedef struct DbdataCursor DbdataCursor;
  92
  93 /* Cursor object */
  94 struct DbdataCursor {
  95   sqlite3_vtab_cursor base;       /* Base class.  Must be first */
  96   sqlite3_stmt *pStmt;            /* For fetching database pages */
  97
  98   int iPgno;                      /* Current page number */
  99   u8 *aPage;                      /* Buffer containing page */
 100   int nPage;                      /* Size of aPage[] in bytes */
 101   int nCell;                      /* Number of cells on aPage[] */
 102   int iCell;                      /* Current cell number */
 103   int bOnePage;                   /* True to stop after one page */
 104   int szDb;
 105   sqlite3_int64 iRowid;
 106
 107   /* Only for the sqlite_dbdata table */
 108   u8 *pRec;                       /* Buffer containing current record */
 109   sqlite3_int64 nRec;             /* Size of pRec[] in bytes */
 110   sqlite3_int64 nHdr;             /* Size of header in bytes */
 111   int iField;                     /* Current field number */
 112   u8 *pHdrPtr;
 113   u8 *pPtr;
 114   u32 enc;                        /* Text encoding */
 115
 116   sqlite3_int64 iIntkey;          /* Integer key value */
 117 };
 118
 119 /* Table object */
 120 struct DbdataTable {
 121   sqlite3_vtab base;              /* Base class.  Must be first */
 122   sqlite3 *db;                    /* The database connection */
 123   sqlite3_stmt *pStmt;            /* For fetching database pages */
 124   int bPtr;                       /* True for sqlite3_dbptr table */
 125 };
 126
 127 /* Column and schema definitions for sqlite_dbdata */
 128 #define DBDATA_COLUMN_PGNO        0
 129 #define DBDATA_COLUMN_CELL        1
 130 #define DBDATA_COLUMN_FIELD       2
 131 #define DBDATA_COLUMN_VALUE       3
 132 #define DBDATA_COLUMN_SCHEMA      4
 133 #define DBDATA_SCHEMA             \
 134       "CREATE TABLE x("           \
 135       "  pgno INTEGER,"           \
 136       "  cell INTEGER,"           \
 137       "  field INTEGER,"          \
 138       "  value ANY,"              \
 139       "  schema TEXT HIDDEN"      \
 140       ")"
 141
 142 /* Column and schema definitions for sqlite_dbptr */
 143 #define DBPTR_COLUMN_PGNO         0
 144 #define DBPTR_COLUMN_CHILD        1
 145 #define DBPTR_COLUMN_SCHEMA       2
 146 #define DBPTR_SCHEMA              \
 147       "CREATE TABLE x("           \
 148       "  pgno INTEGER,"           \
 149       "  child INTEGER,"          \
 150       "  schema TEXT HIDDEN"      \
 151       ")"
 152
 153 /*
 154 ** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual
 155 ** table.
 156 */
 157 static int dbdataConnect(
 158   sqlite3 *db,
 159   void *pAux,
 160   int argc, const char *const*argv,
 161   sqlite3_vtab **ppVtab,
 162   char **pzErr
 163 ){
 164   DbdataTable *pTab = 0;
 165   int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA);
 166
 167   (void)argc;
 168   (void)argv;
 169   (void)pzErr;
 170   sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
 171   if( rc==SQLITE_OK ){
 172     pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable));
 173     if( pTab==0 ){
 174       rc = SQLITE_NOMEM;
 175     }else{
 176       memset(pTab, 0, sizeof(DbdataTable));
 177       pTab->db = db;
 178       pTab->bPtr = (pAux!=0);
 179     }
 180   }
 181
 182   *ppVtab = (sqlite3_vtab*)pTab;
 183   return rc;
 184 }
 185
 186 /*
 187 ** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table.
 188 */
 189 static int dbdataDisconnect(sqlite3_vtab *pVtab){
 190   DbdataTable *pTab = (DbdataTable*)pVtab;
 191   if( pTab ){
 192     sqlite3_finalize(pTab->pStmt);
 193     sqlite3_free(pVtab);
 194   }
 195   return SQLITE_OK;
 196 }
 197
 198 /*
 199 ** This function interprets two types of constraints:
 200 **
 201 **       schema=?
 202 **       pgno=?
 203 **
 204 ** If neither are present, idxNum is set to 0. If schema=? is present,
 205 ** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit
 206 ** in idxNum is set.
 207 **
 208 ** If both parameters are present, schema is in position 0 and pgno in
 209 ** position 1.
 210 */
 211 static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){
 212   DbdataTable *pTab = (DbdataTable*)tab;
 213   int i;
 214   int iSchema = -1;
 215   int iPgno = -1;
 216   int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA);
 217
 218   for(i=0; i<pIdx->nConstraint; i++){
 219     struct sqlite3_index_constraint *p = &pIdx->aConstraint[i];
 220     if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
 221       if( p->iColumn==colSchema ){
 222         if( p->usable==0 ) return SQLITE_CONSTRAINT;
 223         iSchema = i;
 224       }
 225       if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){
 226         iPgno = i;
 227       }
 228     }
 229   }
 230
 231   if( iSchema>=0 ){
 232     pIdx->aConstraintUsage[iSchema].argvIndex = 1;
 233     pIdx->aConstraintUsage[iSchema].omit = 1;
 234   }
 235   if( iPgno>=0 ){
 236     pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0);
 237     pIdx->aConstraintUsage[iPgno].omit = 1;
 238     pIdx->estimatedCost = 100;
 239     pIdx->estimatedRows =  50;
 240
 241     if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){
 242       int iCol = pIdx->aOrderBy[0].iColumn;
 243       if( pIdx->nOrderBy==1 ){
 244         pIdx->orderByConsumed = (iCol==0 || iCol==1);
 245       }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){
 246         pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1);
 247       }
 248     }
 249
 250   }else{
 251     pIdx->estimatedCost = 100000000;
 252     pIdx->estimatedRows = 1000000000;
 253   }
 254   pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00);
 255   return SQLITE_OK;
 256 }
 257
 258 /*
 259 ** Open a new sqlite_dbdata or sqlite_dbptr cursor.
 260 */
 261 static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
 262   DbdataCursor *pCsr;
 263
 264   pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor));
 265   if( pCsr==0 ){
 266     return SQLITE_NOMEM;
 267   }else{
 268     memset(pCsr, 0, sizeof(DbdataCursor));
 269     pCsr->base.pVtab = pVTab;
 270   }
 271
 272   *ppCursor = (sqlite3_vtab_cursor *)pCsr;
 273   return SQLITE_OK;
 274 }
 275
 276 /*
 277 ** Restore a cursor object to the state it was in when first allocated
 278 ** by dbdataOpen().
 279 */
 280 static void dbdataResetCursor(DbdataCursor *pCsr){
 281   DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab);
 282   if( pTab->pStmt==0 ){
 283     pTab->pStmt = pCsr->pStmt;
 284   }else{
 285     sqlite3_finalize(pCsr->pStmt);
 286   }
 287   pCsr->pStmt = 0;
 288   pCsr->iPgno = 1;
 289   pCsr->iCell = 0;
 290   pCsr->iField = 0;
 291   pCsr->bOnePage = 0;
 292   sqlite3_free(pCsr->aPage);
 293   sqlite3_free(pCsr->pRec);
 294   pCsr->pRec = 0;
 295   pCsr->aPage = 0;
 296 }
 297
 298 /*
 299 ** Close an sqlite_dbdata or sqlite_dbptr cursor.
 300 */
 301 static int dbdataClose(sqlite3_vtab_cursor *pCursor){
 302   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 303   dbdataResetCursor(pCsr);
 304   sqlite3_free(pCsr);
 305   return SQLITE_OK;
 306 }
 307
 308 /*
 309 ** Utility methods to decode 16 and 32-bit big-endian unsigned integers.
 310 */
 311 static u32 get_uint16(unsigned char *a){
 312   return (a[0]<<8)|a[1];
 313 }
 314 static u32 get_uint32(unsigned char *a){
 315   return ((u32)a[0]<<24)
 316        | ((u32)a[1]<<16)
 317        | ((u32)a[2]<<8)
 318        | ((u32)a[3]);
 319 }
 320
 321 /*
 322 ** Load page pgno from the database via the sqlite_dbpage virtual table.
 323 ** If successful, set (*ppPage) to point to a buffer containing the page
 324 ** data, (*pnPage) to the size of that buffer in bytes and return
 325 ** SQLITE_OK. In this case it is the responsibility of the caller to
 326 ** eventually free the buffer using sqlite3_free().
 327 **
 328 ** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and
 329 ** return an SQLite error code.
 330 */
 331 static int dbdataLoadPage(
 332   DbdataCursor *pCsr,             /* Cursor object */
 333   u32 pgno,                       /* Page number of page to load */
 334   u8 **ppPage,                    /* OUT: pointer to page buffer */
 335   int *pnPage                     /* OUT: Size of (*ppPage) in bytes */
 336 ){
 337   int rc2;
 338   int rc = SQLITE_OK;
 339   sqlite3_stmt *pStmt = pCsr->pStmt;
 340
 341   *ppPage = 0;
 342   *pnPage = 0;
 343   if( pgno>0 ){
 344     sqlite3_bind_int64(pStmt, 2, pgno);
 345     if( SQLITE_ROW==sqlite3_step(pStmt) ){
 346       int nCopy = sqlite3_column_bytes(pStmt, 0);
 347       if( nCopy>0 ){
 348         u8 *pPage;
 349         pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
 350         if( pPage==0 ){
 351           rc = SQLITE_NOMEM;
 352         }else{
 353           const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
 354           memcpy(pPage, pCopy, nCopy);
 355           memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
 356         }
 357         *ppPage = pPage;
 358         *pnPage = nCopy;
 359       }
 360     }
 361     rc2 = sqlite3_reset(pStmt);
 362     if( rc==SQLITE_OK ) rc = rc2;
 363   }
 364
 365   return rc;
 366 }
 367
 368 /*
 369 ** Read a varint.  Put the value in *pVal and return the number of bytes.
 370 */
 371 static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
 372   sqlite3_uint64 u = 0;
 373   int i;
 374   for(i=0; i<8; i++){
 375     u = (u<<7) + (z[i]&0x7f);
 376     if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
 377   }
 378   u = (u<<8) + (z[i]&0xff);
 379   *pVal = (sqlite3_int64)u;
 380   return 9;
 381 }
 382
 383 /*
 384 ** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
 385 ** or greater than 0xFFFFFFFF. This can be used for all varints in an
 386 ** SQLite database except for key values in intkey tables.
 387 */
 388 static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
 389   sqlite3_int64 val;
 390   int nRet = dbdataGetVarint(z, &val);
 391   if( val<0 || val>0xFFFFFFFF ) val = 0;
 392   *pVal = val;
 393   return nRet;
 394 }
 395
 396 /*
 397 ** Return the number of bytes of space used by an SQLite value of type
 398 ** eType.
 399 */
 400 static int dbdataValueBytes(int eType){
 401   switch( eType ){
 402     case 0: case 8: case 9:
 403     case 10: case 11:
 404       return 0;
 405     case 1:
 406       return 1;
 407     case 2:
 408       return 2;
 409     case 3:
 410       return 3;
 411     case 4:
 412       return 4;
 413     case 5:
 414       return 6;
 415     case 6:
 416     case 7:
 417       return 8;
 418     default:
 419       if( eType>0 ){
 420         return ((eType-12) / 2);
 421       }
 422       return 0;
 423   }
 424 }
 425
 426 /*
 427 ** Load a value of type eType from buffer pData and use it to set the
 428 ** result of context object pCtx.
 429 */
 430 static void dbdataValue(
 431   sqlite3_context *pCtx,
 432   u32 enc,
 433   int eType,
 434   u8 *pData,
 435   sqlite3_int64 nData
 436 ){
 437   if( eType>=0 && dbdataValueBytes(eType)<=nData ){
 438     switch( eType ){
 439       case 0:
 440       case 10:
 441       case 11:
 442         sqlite3_result_null(pCtx);
 443         break;
 444
 445       case 8:
 446         sqlite3_result_int(pCtx, 0);
 447         break;
 448       case 9:
 449         sqlite3_result_int(pCtx, 1);
 450         break;
 451
 452       case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
 453         sqlite3_uint64 v = (signed char)pData[0];
 454         pData++;
 455         switch( eType ){
 456           case 7:
 457           case 6:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
 458           case 5:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
 459           case 4:  v = (v<<8) + pData[0];  pData++;
 460           case 3:  v = (v<<8) + pData[0];  pData++;
 461           case 2:  v = (v<<8) + pData[0];  pData++;
 462         }
 463
 464         if( eType==7 ){
 465           double r;
 466           memcpy(&r, &v, sizeof(r));
 467           sqlite3_result_double(pCtx, r);
 468         }else{
 469           sqlite3_result_int64(pCtx, (sqlite3_int64)v);
 470         }
 471         break;
 472       }
 473
 474       default: {
 475         int n = ((eType-12) / 2);
 476         if( eType % 2 ){
 477           switch( enc ){
 478 #ifndef SQLITE_OMIT_UTF16
 479             case SQLITE_UTF16BE:
 480               sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
 481               break;
 482             case SQLITE_UTF16LE:
 483               sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
 484               break;
 485 #endif
 486             default:
 487               sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
 488               break;
 489           }
 490         }else{
 491           sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
 492         }
 493       }
 494     }
 495   }
 496 }
 497
 498 /*
 499 ** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry.
 500 */
 501 static int dbdataNext(sqlite3_vtab_cursor *pCursor){
 502   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 503   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 504
 505   pCsr->iRowid++;
 506   while( 1 ){
 507     int rc;
 508     int iOff = (pCsr->iPgno==1 ? 100 : 0);
 509     int bNextPage = 0;
 510
 511     if( pCsr->aPage==0 ){
 512       while( 1 ){
 513         if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK;
 514         rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
 515         if( rc!=SQLITE_OK ) return rc;
 516         if( pCsr->aPage && pCsr->nPage>=256 ) break;
 517         sqlite3_free(pCsr->aPage);
 518         pCsr->aPage = 0;
 519         if( pCsr->bOnePage ) return SQLITE_OK;
 520         pCsr->iPgno++;
 521       }
 522
 523       assert( iOff+3+2<=pCsr->nPage );
 524       pCsr->iCell = pTab->bPtr ? -2 : 0;
 525       pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]);
 526     }
 527
 528     if( pTab->bPtr ){
 529       if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){
 530         pCsr->iCell = pCsr->nCell;
 531       }
 532       pCsr->iCell++;
 533       if( pCsr->iCell>=pCsr->nCell ){
 534         sqlite3_free(pCsr->aPage);
 535         pCsr->aPage = 0;
 536         if( pCsr->bOnePage ) return SQLITE_OK;
 537         pCsr->iPgno++;
 538       }else{
 539         return SQLITE_OK;
 540       }
 541     }else{
 542       /* If there is no record loaded, load it now. */
 543       if( pCsr->pRec==0 ){
 544         int bHasRowid = 0;
 545         int nPointer = 0;
 546         sqlite3_int64 nPayload = 0;
 547         sqlite3_int64 nHdr = 0;
 548         int iHdr;
 549         int U, X;
 550         int nLocal;
 551
 552         switch( pCsr->aPage[iOff] ){
 553           case 0x02:
 554             nPointer = 4;
 555             break;
 556           case 0x0a:
 557             break;
 558           case 0x0d:
 559             bHasRowid = 1;
 560             break;
 561           default:
 562             /* This is not a b-tree page with records on it. Continue. */
 563             pCsr->iCell = pCsr->nCell;
 564             break;
 565         }
 566
 567         if( pCsr->iCell>=pCsr->nCell ){
 568           bNextPage = 1;
 569         }else{
 570
 571           iOff += 8 + nPointer + pCsr->iCell*2;
 572           if( iOff>pCsr->nPage ){
 573             bNextPage = 1;
 574           }else{
 575             iOff = get_uint16(&pCsr->aPage[iOff]);
 576           }
 577
 578           /* For an interior node cell, skip past the child-page number */
 579           iOff += nPointer;
 580
 581           /* Load the "byte of payload including overflow" field */
 582           if( bNextPage || iOff>pCsr->nPage ){
 583             bNextPage = 1;
 584           }else{
 585             iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
 586           }
 587
 588           /* If this is a leaf intkey cell, load the rowid */
 589           if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){
 590             iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey);
 591           }
 592
 593           /* Figure out how much data to read from the local page */
 594           U = pCsr->nPage;
 595           if( bHasRowid ){
 596             X = U-35;
 597           }else{
 598             X = ((U-12)*64/255)-23;
 599           }
 600           if( nPayload<=X ){
 601             nLocal = nPayload;
 602           }else{
 603             int M, K;
 604             M = ((U-12)*32/255)-23;
 605             K = M+((nPayload-M)%(U-4));
 606             if( K<=X ){
 607               nLocal = K;
 608             }else{
 609               nLocal = M;
 610             }
 611           }
 612
 613           if( bNextPage || nLocal+iOff>pCsr->nPage ){
 614             bNextPage = 1;
 615           }else{
 616
 617             /* Allocate space for payload. And a bit more to catch small buffer
 618             ** overruns caused by attempting to read a varint or similar from
 619             ** near the end of a corrupt record.  */
 620             pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES);
 621             if( pCsr->pRec==0 ) return SQLITE_NOMEM;
 622             memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES);
 623             pCsr->nRec = nPayload;
 624
 625             /* Load the nLocal bytes of payload */
 626             memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal);
 627             iOff += nLocal;
 628
 629             /* Load content from overflow pages */
 630             if( nPayload>nLocal ){
 631               sqlite3_int64 nRem = nPayload - nLocal;
 632               u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
 633               while( nRem>0 ){
 634                 u8 *aOvfl = 0;
 635                 int nOvfl = 0;
 636                 int nCopy;
 637                 rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl);
 638                 assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage );
 639                 if( rc!=SQLITE_OK ) return rc;
 640                 if( aOvfl==0 ) break;
 641
 642                 nCopy = U-4;
 643                 if( nCopy>nRem ) nCopy = nRem;
 644                 memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
 645                 nRem -= nCopy;
 646
 647                 pgnoOvfl = get_uint32(aOvfl);
 648                 sqlite3_free(aOvfl);
 649               }
 650             }
 651
 652             iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
 653             if( nHdr>nPayload ) nHdr = 0;
 654             pCsr->nHdr = nHdr;
 655             pCsr->pHdrPtr = &pCsr->pRec[iHdr];
 656             pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
 657             pCsr->iField = (bHasRowid ? -1 : 0);
 658           }
 659         }
 660       }else{
 661         pCsr->iField++;
 662         if( pCsr->iField>0 ){
 663           sqlite3_int64 iType;
 664           if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
 665             bNextPage = 1;
 666           }else{
 667             pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
 668             pCsr->pPtr += dbdataValueBytes(iType);
 669           }
 670         }
 671       }
 672
 673       if( bNextPage ){
 674         sqlite3_free(pCsr->aPage);
 675         sqlite3_free(pCsr->pRec);
 676         pCsr->aPage = 0;
 677         pCsr->pRec = 0;
 678         if( pCsr->bOnePage ) return SQLITE_OK;
 679         pCsr->iPgno++;
 680       }else{
 681         if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){
 682           return SQLITE_OK;
 683         }
 684
 685         /* Advance to the next cell. The next iteration of the loop will load
 686         ** the record and so on. */
 687         sqlite3_free(pCsr->pRec);
 688         pCsr->pRec = 0;
 689         pCsr->iCell++;
 690       }
 691     }
 692   }
 693
 694   assert( !"can't get here" );
 695   return SQLITE_OK;
 696 }
 697
 698 /*
 699 ** Return true if the cursor is at EOF.
 700 */
 701 static int dbdataEof(sqlite3_vtab_cursor *pCursor){
 702   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 703   return pCsr->aPage==0;
 704 }
 705
 706 /*
 707 ** Return true if nul-terminated string zSchema ends in "()". Or false
 708 ** otherwise.
 709 */
 710 static int dbdataIsFunction(const char *zSchema){
 711   size_t n = strlen(zSchema);
 712   if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
 713     return (int)n-2;
 714   }
 715   return 0;
 716 }
 717
 718 /*
 719 ** Determine the size in pages of database zSchema (where zSchema is
 720 ** "main", "temp" or the name of an attached database) and set
 721 ** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise,
 722 ** an SQLite error code.
 723 */
 724 static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
 725   DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
 726   char *zSql = 0;
 727   int rc, rc2;
 728   int nFunc = 0;
 729   sqlite3_stmt *pStmt = 0;
 730
 731   if( (nFunc = dbdataIsFunction(zSchema))>0 ){
 732     zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
 733   }else{
 734     zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
 735   }
 736   if( zSql==0 ) return SQLITE_NOMEM;
 737
 738   rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
 739   sqlite3_free(zSql);
 740   if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
 741     pCsr->szDb = sqlite3_column_int(pStmt, 0);
 742   }
 743   rc2 = sqlite3_finalize(pStmt);
 744   if( rc==SQLITE_OK ) rc = rc2;
 745   return rc;
 746 }
 747
 748 /*
 749 ** Attempt to figure out the encoding of the database by retrieving page 1
 750 ** and inspecting the header field. If successful, set the pCsr->enc variable
 751 ** and return SQLITE_OK. Otherwise, return an SQLite error code.
 752 */
 753 static int dbdataGetEncoding(DbdataCursor *pCsr){
 754   int rc = SQLITE_OK;
 755   int nPg1 = 0;
 756   u8 *aPg1 = 0;
 757   rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
 758   if( rc==SQLITE_OK && nPg1>=(56+4) ){
 759     pCsr->enc = get_uint32(&aPg1[56]);
 760   }
 761   sqlite3_free(aPg1);
 762   return rc;
 763 }
 764
 765
 766 /*
 767 ** xFilter method for sqlite_dbdata and sqlite_dbptr.
 768 */
 769 static int dbdataFilter(
 770   sqlite3_vtab_cursor *pCursor,
 771   int idxNum, const char *idxStr,
 772   int argc, sqlite3_value **argv
 773 ){
 774   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 775   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 776   int rc = SQLITE_OK;
 777   const char *zSchema = "main";
 778   (void)idxStr;
 779   (void)argc;
 780
 781   dbdataResetCursor(pCsr);
 782   assert( pCsr->iPgno==1 );
 783   if( idxNum & 0x01 ){
 784     zSchema = (const char*)sqlite3_value_text(argv[0]);
 785     if( zSchema==0 ) zSchema = "";
 786   }
 787   if( idxNum & 0x02 ){
 788     pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
 789     pCsr->bOnePage = 1;
 790   }else{
 791     rc = dbdataDbsize(pCsr, zSchema);
 792   }
 793
 794   if( rc==SQLITE_OK ){
 795     int nFunc = 0;
 796     if( pTab->pStmt ){
 797       pCsr->pStmt = pTab->pStmt;
 798       pTab->pStmt = 0;
 799     }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
 800       char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
 801       if( zSql==0 ){
 802         rc = SQLITE_NOMEM;
 803       }else{
 804         rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
 805         sqlite3_free(zSql);
 806       }
 807     }else{
 808       rc = sqlite3_prepare_v2(pTab->db,
 809           "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
 810           &pCsr->pStmt, 0
 811       );
 812     }
 813   }
 814   if( rc==SQLITE_OK ){
 815     rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
 816   }
 817
 818   /* Try to determine the encoding of the db by inspecting the header
 819   ** field on page 1. */
 820   if( rc==SQLITE_OK ){
 821     rc = dbdataGetEncoding(pCsr);
 822   }
 823
 824   if( rc!=SQLITE_OK ){
 825     pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
 826   }
 827
 828   if( rc==SQLITE_OK ){
 829     rc = dbdataNext(pCursor);
 830   }
 831   return rc;
 832 }
 833
 834 /*
 835 ** Return a column for the sqlite_dbdata or sqlite_dbptr table.
 836 */
 837 static int dbdataColumn(
 838   sqlite3_vtab_cursor *pCursor,
 839   sqlite3_context *ctx,
 840   int i
 841 ){
 842   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 843   DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
 844   if( pTab->bPtr ){
 845     switch( i ){
 846       case DBPTR_COLUMN_PGNO:
 847         sqlite3_result_int64(ctx, pCsr->iPgno);
 848         break;
 849       case DBPTR_COLUMN_CHILD: {
 850         int iOff = pCsr->iPgno==1 ? 100 : 0;
 851         if( pCsr->iCell<0 ){
 852           iOff += 8;
 853         }else{
 854           iOff += 12 + pCsr->iCell*2;
 855           if( iOff>pCsr->nPage ) return SQLITE_OK;
 856           iOff = get_uint16(&pCsr->aPage[iOff]);
 857         }
 858         if( iOff<=pCsr->nPage ){
 859           sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff]));
 860         }
 861         break;
 862       }
 863     }
 864   }else{
 865     switch( i ){
 866       case DBDATA_COLUMN_PGNO:
 867         sqlite3_result_int64(ctx, pCsr->iPgno);
 868         break;
 869       case DBDATA_COLUMN_CELL:
 870         sqlite3_result_int(ctx, pCsr->iCell);
 871         break;
 872       case DBDATA_COLUMN_FIELD:
 873         sqlite3_result_int(ctx, pCsr->iField);
 874         break;
 875       case DBDATA_COLUMN_VALUE: {
 876         if( pCsr->iField<0 ){
 877           sqlite3_result_int64(ctx, pCsr->iIntkey);
 878         }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
 879           sqlite3_int64 iType;
 880           dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
 881           dbdataValue(
 882               ctx, pCsr->enc, iType, pCsr->pPtr,
 883               &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
 884           );
 885         }
 886         break;
 887       }
 888     }
 889   }
 890   return SQLITE_OK;
 891 }
 892
 893 /*
 894 ** Return the rowid for an sqlite_dbdata or sqlite_dptr table.
 895 */
 896 static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
 897   DbdataCursor *pCsr = (DbdataCursor*)pCursor;
 898   *pRowid = pCsr->iRowid;
 899   return SQLITE_OK;
 900 }
 901
 902
 903 /*
 904 ** Invoke this routine to register the "sqlite_dbdata" virtual table module
 905 */
 906 static int sqlite3DbdataRegister(sqlite3 *db){
 907   static sqlite3_module dbdata_module = {
 908     0,                            /* iVersion */
 909     0,                            /* xCreate */
 910     dbdataConnect,                /* xConnect */
 911     dbdataBestIndex,              /* xBestIndex */
 912     dbdataDisconnect,             /* xDisconnect */
 913     0,                            /* xDestroy */
 914     dbdataOpen,                   /* xOpen - open a cursor */
 915     dbdataClose,                  /* xClose - close a cursor */
 916     dbdataFilter,                 /* xFilter - configure scan constraints */
 917     dbdataNext,                   /* xNext - advance a cursor */
 918     dbdataEof,                    /* xEof - check for end of scan */
 919     dbdataColumn,                 /* xColumn - read data */
 920     dbdataRowid,                  /* xRowid - read data */
 921     0,                            /* xUpdate */
 922     0,                            /* xBegin */
 923     0,                            /* xSync */
 924     0,                            /* xCommit */
 925     0,                            /* xRollback */
 926     0,                            /* xFindMethod */
 927     0,                            /* xRename */
 928     0,                            /* xSavepoint */
 929     0,                            /* xRelease */
 930     0,                            /* xRollbackTo */
 931     0                             /* xShadowName */
 932   };
 933
 934   int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0);
 935   if( rc==SQLITE_OK ){
 936     rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1);
 937   }
 938   return rc;
 939 }
 940
 941 #ifdef _WIN32
 942 __declspec(dllexport)
 943 #endif
 944 int sqlite3_dbdata_init(
 945   sqlite3 *db,
 946   char **pzErrMsg,
 947   const sqlite3_api_routines *pApi
 948 ){
 949   SQLITE_EXTENSION_INIT2(pApi);
 950   (void)pzErrMsg;
 951   return sqlite3DbdataRegister(db);
 952 }
 953
 954 #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */