ext/fts5/fts5_storage.c

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14
  15
  16
  17 #include "fts5Int.h"
  18
  19 struct Fts5Storage {
  20   Fts5Config *pConfig;
  21   Fts5Index *pIndex;
  22   int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  23   i64 nTotalRow;                  /* Total number of rows in FTS table */
  24   i64 *aTotalSize;                /* Total sizes of each column */
  25   sqlite3_stmt *aStmt[11];
  26 };
  27
  28
  29 #if FTS5_STMT_SCAN_ASC!=0
  30 # error "FTS5_STMT_SCAN_ASC mismatch"
  31 #endif
  32 #if FTS5_STMT_SCAN_DESC!=1
  33 # error "FTS5_STMT_SCAN_DESC mismatch"
  34 #endif
  35 #if FTS5_STMT_LOOKUP!=2
  36 # error "FTS5_STMT_LOOKUP mismatch"
  37 #endif
  38
  39 #define FTS5_STMT_INSERT_CONTENT  3
  40 #define FTS5_STMT_REPLACE_CONTENT 4
  41 #define FTS5_STMT_DELETE_CONTENT  5
  42 #define FTS5_STMT_REPLACE_DOCSIZE  6
  43 #define FTS5_STMT_DELETE_DOCSIZE  7
  44 #define FTS5_STMT_LOOKUP_DOCSIZE  8
  45 #define FTS5_STMT_REPLACE_CONFIG 9
  46 #define FTS5_STMT_SCAN 10
  47
  48 /*
  49 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
  50 ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
  51 ** Return SQLITE_OK if successful, or an SQLite error code if an error
  52 ** occurs.
  53 */
  54 static int fts5StorageGetStmt(
  55   Fts5Storage *p,                 /* Storage handle */
  56   int eStmt,                      /* FTS5_STMT_XXX constant */
  57   sqlite3_stmt **ppStmt,          /* OUT: Prepared statement handle */
  58   char **pzErrMsg                 /* OUT: Error message (if any) */
  59 ){
  60   int rc = SQLITE_OK;
  61
  62   /* If there is no %_docsize table, there should be no requests for
  63   ** statements to operate on it.  */
  64   assert( p->pConfig->bColumnsize || (
  65         eStmt!=FTS5_STMT_REPLACE_DOCSIZE
  66      && eStmt!=FTS5_STMT_DELETE_DOCSIZE
  67      && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE
  68   ));
  69
  70   assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
  71   if( p->aStmt[eStmt]==0 ){
  72     const char *azStmt[] = {
  73       "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
  74       "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
  75       "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
  76
  77       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
  78       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
  79       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
  80       "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)",     /* REPLACE_DOCSIZE  */
  81       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
  82
  83       "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?",    /* LOOKUP_DOCSIZE  */
  84
  85       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
  86       "SELECT %s FROM %s AS T",                         /* SCAN */
  87     };
  88     Fts5Config *pC = p->pConfig;
  89     char *zSql = 0;
  90
  91     switch( eStmt ){
  92       case FTS5_STMT_SCAN:
  93         zSql = sqlite3_mprintf(azStmt[eStmt],
  94             pC->zContentExprlist, pC->zContent
  95         );
  96         break;
  97
  98       case FTS5_STMT_SCAN_ASC:
  99       case FTS5_STMT_SCAN_DESC:
 100         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist,
 101             pC->zContent, pC->zContentRowid, pC->zContentRowid,
 102             pC->zContentRowid
 103         );
 104         break;
 105
 106       case FTS5_STMT_LOOKUP:
 107         zSql = sqlite3_mprintf(azStmt[eStmt],
 108             pC->zContentExprlist, pC->zContent, pC->zContentRowid
 109         );
 110         break;
 111
 112       case FTS5_STMT_INSERT_CONTENT:
 113       case FTS5_STMT_REPLACE_CONTENT: {
 114         int nCol = pC->nCol + 1;
 115         char *zBind;
 116         int i;
 117
 118         zBind = sqlite3_malloc64(1 + nCol*2);
 119         if( zBind ){
 120           for(i=0; i<nCol; i++){
 121             zBind[i*2] = '?';
 122             zBind[i*2 + 1] = ',';
 123           }
 124           zBind[i*2-1] = '\0';
 125           zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
 126           sqlite3_free(zBind);
 127         }
 128         break;
 129       }
 130
 131       case FTS5_STMT_REPLACE_DOCSIZE:
 132         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
 133           (pC->bContentlessDelete ? ",?" : "")
 134         );
 135         break;
 136
 137       case FTS5_STMT_LOOKUP_DOCSIZE:
 138         zSql = sqlite3_mprintf(azStmt[eStmt],
 139             (pC->bContentlessDelete ? ",origin" : ""),
 140             pC->zDb, pC->zName
 141         );
 142         break;
 143
 144       default:
 145         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
 146         break;
 147     }
 148
 149     if( zSql==0 ){
 150       rc = SQLITE_NOMEM;
 151     }else{
 152       int f = SQLITE_PREPARE_PERSISTENT;
 153       if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
 154       p->pConfig->bLock++;
 155       rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
 156       p->pConfig->bLock--;
 157       sqlite3_free(zSql);
 158       if( rc!=SQLITE_OK && pzErrMsg ){
 159         *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
 160       }
 161     }
 162   }
 163
 164   *ppStmt = p->aStmt[eStmt];
 165   sqlite3_reset(*ppStmt);
 166   return rc;
 167 }
 168
 169
 170 static int fts5ExecPrintf(
 171   sqlite3 *db,
 172   char **pzErr,
 173   const char *zFormat,
 174   ...
 175 ){
 176   int rc;
 177   va_list ap;                     /* ... printf arguments */
 178   char *zSql;
 179
 180   va_start(ap, zFormat);
 181   zSql = sqlite3_vmprintf(zFormat, ap);
 182
 183   if( zSql==0 ){
 184     rc = SQLITE_NOMEM;
 185   }else{
 186     rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
 187     sqlite3_free(zSql);
 188   }
 189
 190   va_end(ap);
 191   return rc;
 192 }
 193
 194 /*
 195 ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
 196 ** code otherwise.
 197 */
 198 int sqlite3Fts5DropAll(Fts5Config *pConfig){
 199   int rc = fts5ExecPrintf(pConfig->db, 0,
 200       "DROP TABLE IF EXISTS %Q.'%q_data';"
 201       "DROP TABLE IF EXISTS %Q.'%q_idx';"
 202       "DROP TABLE IF EXISTS %Q.'%q_config';",
 203       pConfig->zDb, pConfig->zName,
 204       pConfig->zDb, pConfig->zName,
 205       pConfig->zDb, pConfig->zName
 206   );
 207   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 208     rc = fts5ExecPrintf(pConfig->db, 0,
 209         "DROP TABLE IF EXISTS %Q.'%q_docsize';",
 210         pConfig->zDb, pConfig->zName
 211     );
 212   }
 213   if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
 214     rc = fts5ExecPrintf(pConfig->db, 0,
 215         "DROP TABLE IF EXISTS %Q.'%q_content';",
 216         pConfig->zDb, pConfig->zName
 217     );
 218   }
 219   return rc;
 220 }
 221
 222 static void fts5StorageRenameOne(
 223   Fts5Config *pConfig,            /* Current FTS5 configuration */
 224   int *pRc,                       /* IN/OUT: Error code */
 225   const char *zTail,              /* Tail of table name e.g. "data", "config" */
 226   const char *zName               /* New name of FTS5 table */
 227 ){
 228   if( *pRc==SQLITE_OK ){
 229     *pRc = fts5ExecPrintf(pConfig->db, 0,
 230         "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
 231         pConfig->zDb, pConfig->zName, zTail, zName, zTail
 232     );
 233   }
 234 }
 235
 236 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
 237   Fts5Config *pConfig = pStorage->pConfig;
 238   int rc = sqlite3Fts5StorageSync(pStorage);
 239
 240   fts5StorageRenameOne(pConfig, &rc, "data", zName);
 241   fts5StorageRenameOne(pConfig, &rc, "idx", zName);
 242   fts5StorageRenameOne(pConfig, &rc, "config", zName);
 243   if( pConfig->bColumnsize ){
 244     fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
 245   }
 246   if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 247     fts5StorageRenameOne(pConfig, &rc, "content", zName);
 248   }
 249   return rc;
 250 }
 251
 252 /*
 253 ** Create the shadow table named zPost, with definition zDefn. Return
 254 ** SQLITE_OK if successful, or an SQLite error code otherwise.
 255 */
 256 int sqlite3Fts5CreateTable(
 257   Fts5Config *pConfig,            /* FTS5 configuration */
 258   const char *zPost,              /* Shadow table to create (e.g. "content") */
 259   const char *zDefn,              /* Columns etc. for shadow table */
 260   int bWithout,                   /* True for without rowid */
 261   char **pzErr                    /* OUT: Error message */
 262 ){
 263   int rc;
 264   char *zErr = 0;
 265
 266   rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
 267       pConfig->zDb, pConfig->zName, zPost, zDefn,
 268 #ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
 269       bWithout?" WITHOUT ROWID":
 270 #endif
 271       ""
 272   );
 273   if( zErr ){
 274     *pzErr = sqlite3_mprintf(
 275         "fts5: error creating shadow table %q_%s: %s",
 276         pConfig->zName, zPost, zErr
 277     );
 278     sqlite3_free(zErr);
 279   }
 280
 281   return rc;
 282 }
 283
 284 /*
 285 ** Open a new Fts5Index handle. If the bCreate argument is true, create
 286 ** and initialize the underlying tables
 287 **
 288 ** If successful, set *pp to point to the new object and return SQLITE_OK.
 289 ** Otherwise, set *pp to NULL and return an SQLite error code.
 290 */
 291 int sqlite3Fts5StorageOpen(
 292   Fts5Config *pConfig,
 293   Fts5Index *pIndex,
 294   int bCreate,
 295   Fts5Storage **pp,
 296   char **pzErr                    /* OUT: Error message */
 297 ){
 298   int rc = SQLITE_OK;
 299   Fts5Storage *p;                 /* New object */
 300   sqlite3_int64 nByte;            /* Bytes of space to allocate */
 301
 302   nByte = sizeof(Fts5Storage)               /* Fts5Storage object */
 303         + pConfig->nCol * sizeof(i64);      /* Fts5Storage.aTotalSize[] */
 304   *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte);
 305   if( !p ) return SQLITE_NOMEM;
 306
 307   memset(p, 0, (size_t)nByte);
 308   p->aTotalSize = (i64*)&p[1];
 309   p->pConfig = pConfig;
 310   p->pIndex = pIndex;
 311
 312   if( bCreate ){
 313     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 314       int nDefn = 32 + pConfig->nCol*10;
 315       char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
 316       if( zDefn==0 ){
 317         rc = SQLITE_NOMEM;
 318       }else{
 319         int i;
 320         int iOff;
 321         sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
 322         iOff = (int)strlen(zDefn);
 323         for(i=0; i<pConfig->nCol; i++){
 324           sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
 325           iOff += (int)strlen(&zDefn[iOff]);
 326         }
 327         rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
 328       }
 329       sqlite3_free(zDefn);
 330     }
 331
 332     if( rc==SQLITE_OK && pConfig->bColumnsize ){
 333       const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
 334       if( pConfig->bContentlessDelete ){
 335         zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
 336       }
 337       rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
 338     }
 339     if( rc==SQLITE_OK ){
 340       rc = sqlite3Fts5CreateTable(
 341           pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
 342       );
 343     }
 344     if( rc==SQLITE_OK ){
 345       rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
 346     }
 347   }
 348
 349   if( rc ){
 350     sqlite3Fts5StorageClose(p);
 351     *pp = 0;
 352   }
 353   return rc;
 354 }
 355
 356 /*
 357 ** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
 358 */
 359 int sqlite3Fts5StorageClose(Fts5Storage *p){
 360   int rc = SQLITE_OK;
 361   if( p ){
 362     int i;
 363
 364     /* Finalize all SQL statements */
 365     for(i=0; i<ArraySize(p->aStmt); i++){
 366       sqlite3_finalize(p->aStmt[i]);
 367     }
 368
 369     sqlite3_free(p);
 370   }
 371   return rc;
 372 }
 373
 374 typedef struct Fts5InsertCtx Fts5InsertCtx;
 375 struct Fts5InsertCtx {
 376   Fts5Storage *pStorage;
 377   int iCol;
 378   int szCol;                      /* Size of column value in tokens */
 379 };
 380
 381 /*
 382 ** Tokenization callback used when inserting tokens into the FTS index.
 383 */
 384 static int fts5StorageInsertCallback(
 385   void *pContext,                 /* Pointer to Fts5InsertCtx object */
 386   int tflags,
 387   const char *pToken,             /* Buffer containing token */
 388   int nToken,                     /* Size of token in bytes */
 389   int iUnused1,                   /* Start offset of token */
 390   int iUnused2                    /* End offset of token */
 391 ){
 392   Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
 393   Fts5Index *pIdx = pCtx->pStorage->pIndex;
 394   UNUSED_PARAM2(iUnused1, iUnused2);
 395   if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
 396   if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
 397     pCtx->szCol++;
 398   }
 399   return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 400 }
 401
 402 /*
 403 ** If a row with rowid iDel is present in the %_content table, add the
 404 ** delete-markers to the FTS index necessary to delete it. Do not actually
 405 ** remove the %_content row at this time though.
 406 */
 407 static int fts5StorageDeleteFromIndex(
 408   Fts5Storage *p,
 409   i64 iDel,
 410   sqlite3_value **apVal
 411 ){
 412   Fts5Config *pConfig = p->pConfig;
 413   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
 414   int rc = SQLITE_OK;             /* Return code */
 415   int rc2;                        /* sqlite3_reset() return code */
 416   int iCol;
 417   Fts5InsertCtx ctx;
 418
 419   if( apVal==0 ){
 420     rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
 421     if( rc!=SQLITE_OK ) return rc;
 422     sqlite3_bind_int64(pSeek, 1, iDel);
 423     if( sqlite3_step(pSeek)!=SQLITE_ROW ){
 424       return sqlite3_reset(pSeek);
 425     }
 426   }
 427
 428   ctx.pStorage = p;
 429   ctx.iCol = -1;
 430   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
 431     if( pConfig->abUnindexed[iCol-1]==0 ){
 432       const char *zText;
 433       int nText;
 434       assert( pSeek==0 || apVal==0 );
 435       assert( pSeek!=0 || apVal!=0 );
 436       if( pSeek ){
 437         zText = (const char*)sqlite3_column_text(pSeek, iCol);
 438         nText = sqlite3_column_bytes(pSeek, iCol);
 439       }else if( ALWAYS(apVal) ){
 440         zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
 441         nText = sqlite3_value_bytes(apVal[iCol-1]);
 442       }else{
 443         continue;
 444       }
 445       ctx.szCol = 0;
 446       rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
 447           zText, nText, (void*)&ctx, fts5StorageInsertCallback
 448       );
 449       p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
 450       if( p->aTotalSize[iCol-1]<0 ){
 451         rc = FTS5_CORRUPT;
 452       }
 453     }
 454   }
 455   if( rc==SQLITE_OK && p->nTotalRow<1 ){
 456     rc = FTS5_CORRUPT;
 457   }else{
 458     p->nTotalRow--;
 459   }
 460
 461   rc2 = sqlite3_reset(pSeek);
 462   if( rc==SQLITE_OK ) rc = rc2;
 463   return rc;
 464 }
 465
 466 /*
 467 ** This function is called to process a DELETE on a contentless_delete=1
 468 ** table. It adds the tombstone required to delete the entry with rowid
 469 ** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
 470 ** an SQLite error code.
 471 */
 472 static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
 473   i64 iOrigin = 0;
 474   sqlite3_stmt *pLookup = 0;
 475   int rc = SQLITE_OK;
 476
 477   assert( p->pConfig->bContentlessDelete );
 478   assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
 479
 480   /* Look up the origin of the document in the %_docsize table. Store
 481   ** this in stack variable iOrigin.  */
 482   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
 483   if( rc==SQLITE_OK ){
 484     sqlite3_bind_int64(pLookup, 1, iDel);
 485     if( SQLITE_ROW==sqlite3_step(pLookup) ){
 486       iOrigin = sqlite3_column_int64(pLookup, 1);
 487     }
 488     rc = sqlite3_reset(pLookup);
 489   }
 490
 491   if( rc==SQLITE_OK && iOrigin!=0 ){
 492     rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
 493   }
 494
 495   return rc;
 496 }
 497
 498 /*
 499 ** Insert a record into the %_docsize table. Specifically, do:
 500 **
 501 **   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
 502 **
 503 ** If there is no %_docsize table (as happens if the columnsize=0 option
 504 ** is specified when the FTS5 table is created), this function is a no-op.
 505 */
 506 static int fts5StorageInsertDocsize(
 507   Fts5Storage *p,                 /* Storage module to write to */
 508   i64 iRowid,                     /* id value */
 509   Fts5Buffer *pBuf                /* sz value */
 510 ){
 511   int rc = SQLITE_OK;
 512   if( p->pConfig->bColumnsize ){
 513     sqlite3_stmt *pReplace = 0;
 514     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
 515     if( rc==SQLITE_OK ){
 516       sqlite3_bind_int64(pReplace, 1, iRowid);
 517       if( p->pConfig->bContentlessDelete ){
 518         i64 iOrigin = 0;
 519         rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
 520         sqlite3_bind_int64(pReplace, 3, iOrigin);
 521       }
 522       if( rc==SQLITE_OK ){
 523         sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
 524         sqlite3_step(pReplace);
 525         rc = sqlite3_reset(pReplace);
 526         sqlite3_bind_null(pReplace, 2);
 527       }
 528     }
 529   }
 530   return rc;
 531 }
 532
 533 /*
 534 ** Load the contents of the "averages" record from disk into the
 535 ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
 536 ** argument bCache is true, set the p->bTotalsValid flag to indicate
 537 ** that the contents of aTotalSize[] and nTotalRow are valid until
 538 ** further notice.
 539 **
 540 ** Return SQLITE_OK if successful, or an SQLite error code if an error
 541 ** occurs.
 542 */
 543 static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
 544   int rc = SQLITE_OK;
 545   if( p->bTotalsValid==0 ){
 546     rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
 547     p->bTotalsValid = bCache;
 548   }
 549   return rc;
 550 }
 551
 552 /*
 553 ** Store the current contents of the p->nTotalRow and p->aTotalSize[]
 554 ** variables in the "averages" record on disk.
 555 **
 556 ** Return SQLITE_OK if successful, or an SQLite error code if an error
 557 ** occurs.
 558 */
 559 static int fts5StorageSaveTotals(Fts5Storage *p){
 560   int nCol = p->pConfig->nCol;
 561   int i;
 562   Fts5Buffer buf;
 563   int rc = SQLITE_OK;
 564   memset(&buf, 0, sizeof(buf));
 565
 566   sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
 567   for(i=0; i<nCol; i++){
 568     sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
 569   }
 570   if( rc==SQLITE_OK ){
 571     rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
 572   }
 573   sqlite3_free(buf.p);
 574
 575   return rc;
 576 }
 577
 578 /*
 579 ** Remove a row from the FTS table.
 580 */
 581 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 582   Fts5Config *pConfig = p->pConfig;
 583   int rc;
 584   sqlite3_stmt *pDel = 0;
 585
 586   assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
 587   rc = fts5StorageLoadTotals(p, 1);
 588
 589   /* Delete the index records */
 590   if( rc==SQLITE_OK ){
 591     rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
 592   }
 593
 594   if( rc==SQLITE_OK ){
 595     if( p->pConfig->bContentlessDelete ){
 596       rc = fts5StorageContentlessDelete(p, iDel);
 597     }else{
 598       rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
 599     }
 600   }
 601
 602   /* Delete the %_docsize record */
 603   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 604     rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
 605     if( rc==SQLITE_OK ){
 606       sqlite3_bind_int64(pDel, 1, iDel);
 607       sqlite3_step(pDel);
 608       rc = sqlite3_reset(pDel);
 609     }
 610   }
 611
 612   /* Delete the %_content record */
 613   if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
 614     if( rc==SQLITE_OK ){
 615       rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
 616     }
 617     if( rc==SQLITE_OK ){
 618       sqlite3_bind_int64(pDel, 1, iDel);
 619       sqlite3_step(pDel);
 620       rc = sqlite3_reset(pDel);
 621     }
 622   }
 623
 624   return rc;
 625 }
 626
 627 /*
 628 ** Delete all entries in the FTS5 index.
 629 */
 630 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
 631   Fts5Config *pConfig = p->pConfig;
 632   int rc;
 633
 634   p->bTotalsValid = 0;
 635
 636   /* Delete the contents of the %_data and %_docsize tables. */
 637   rc = fts5ExecPrintf(pConfig->db, 0,
 638       "DELETE FROM %Q.'%q_data';"
 639       "DELETE FROM %Q.'%q_idx';",
 640       pConfig->zDb, pConfig->zName,
 641       pConfig->zDb, pConfig->zName
 642   );
 643   if( rc==SQLITE_OK && pConfig->bColumnsize ){
 644     rc = fts5ExecPrintf(pConfig->db, 0,
 645         "DELETE FROM %Q.'%q_docsize';",
 646         pConfig->zDb, pConfig->zName
 647     );
 648   }
 649
 650   /* Reinitialize the %_data table. This call creates the initial structure
 651   ** and averages records.  */
 652   if( rc==SQLITE_OK ){
 653     rc = sqlite3Fts5IndexReinit(p->pIndex);
 654   }
 655   if( rc==SQLITE_OK ){
 656     rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
 657   }
 658   return rc;
 659 }
 660
 661 int sqlite3Fts5StorageRebuild(Fts5Storage *p){
 662   Fts5Buffer buf = {0,0,0};
 663   Fts5Config *pConfig = p->pConfig;
 664   sqlite3_stmt *pScan = 0;
 665   Fts5InsertCtx ctx;
 666   int rc, rc2;
 667
 668   memset(&ctx, 0, sizeof(Fts5InsertCtx));
 669   ctx.pStorage = p;
 670   rc = sqlite3Fts5StorageDeleteAll(p);
 671   if( rc==SQLITE_OK ){
 672     rc = fts5StorageLoadTotals(p, 1);
 673   }
 674
 675   if( rc==SQLITE_OK ){
 676     rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg);
 677   }
 678
 679   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
 680     i64 iRowid = sqlite3_column_int64(pScan, 0);
 681
 682     sqlite3Fts5BufferZero(&buf);
 683     rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
 684     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
 685       ctx.szCol = 0;
 686       if( pConfig->abUnindexed[ctx.iCol]==0 ){
 687         const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
 688         int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
 689         rc = sqlite3Fts5Tokenize(pConfig,
 690             FTS5_TOKENIZE_DOCUMENT,
 691             zText, nText,
 692             (void*)&ctx,
 693             fts5StorageInsertCallback
 694         );
 695       }
 696       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
 697       p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
 698     }
 699     p->nTotalRow++;
 700
 701     if( rc==SQLITE_OK ){
 702       rc = fts5StorageInsertDocsize(p, iRowid, &buf);
 703     }
 704   }
 705   sqlite3_free(buf.p);
 706   rc2 = sqlite3_reset(pScan);
 707   if( rc==SQLITE_OK ) rc = rc2;
 708
 709   /* Write the averages record */
 710   if( rc==SQLITE_OK ){
 711     rc = fts5StorageSaveTotals(p);
 712   }
 713   return rc;
 714 }
 715
 716 int sqlite3Fts5StorageOptimize(Fts5Storage *p){
 717   return sqlite3Fts5IndexOptimize(p->pIndex);
 718 }
 719
 720 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
 721   return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
 722 }
 723
 724 int sqlite3Fts5StorageReset(Fts5Storage *p){
 725   return sqlite3Fts5IndexReset(p->pIndex);
 726 }
 727
 728 /*
 729 ** Allocate a new rowid. This is used for "external content" tables when
 730 ** a NULL value is inserted into the rowid column. The new rowid is allocated
 731 ** by inserting a dummy row into the %_docsize table. The dummy will be
 732 ** overwritten later.
 733 **
 734 ** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
 735 ** this case the user is required to provide a rowid explicitly.
 736 */
 737 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
 738   int rc = SQLITE_MISMATCH;
 739   if( p->pConfig->bColumnsize ){
 740     sqlite3_stmt *pReplace = 0;
 741     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
 742     if( rc==SQLITE_OK ){
 743       sqlite3_bind_null(pReplace, 1);
 744       sqlite3_bind_null(pReplace, 2);
 745       sqlite3_step(pReplace);
 746       rc = sqlite3_reset(pReplace);
 747     }
 748     if( rc==SQLITE_OK ){
 749       *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
 750     }
 751   }
 752   return rc;
 753 }
 754
 755 /*
 756 ** Insert a new row into the FTS content table.
 757 */
 758 int sqlite3Fts5StorageContentInsert(
 759   Fts5Storage *p,
 760   sqlite3_value **apVal,
 761   i64 *piRowid
 762 ){
 763   Fts5Config *pConfig = p->pConfig;
 764   int rc = SQLITE_OK;
 765
 766   /* Insert the new row into the %_content table. */
 767   if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
 768     if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
 769       *piRowid = sqlite3_value_int64(apVal[1]);
 770     }else{
 771       rc = fts5StorageNewRowid(p, piRowid);
 772     }
 773   }else{
 774     sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
 775     int i;                        /* Counter variable */
 776     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
 777     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
 778       rc = sqlite3_bind_value(pInsert, i, apVal[i]);
 779     }
 780     if( rc==SQLITE_OK ){
 781       sqlite3_step(pInsert);
 782       rc = sqlite3_reset(pInsert);
 783     }
 784     *piRowid = sqlite3_last_insert_rowid(pConfig->db);
 785   }
 786
 787   return rc;
 788 }
 789
 790 /*
 791 ** Insert new entries into the FTS index and %_docsize table.
 792 */
 793 int sqlite3Fts5StorageIndexInsert(
 794   Fts5Storage *p,
 795   sqlite3_value **apVal,
 796   i64 iRowid
 797 ){
 798   Fts5Config *pConfig = p->pConfig;
 799   int rc = SQLITE_OK;             /* Return code */
 800   Fts5InsertCtx ctx;              /* Tokenization callback context object */
 801   Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
 802
 803   memset(&buf, 0, sizeof(Fts5Buffer));
 804   ctx.pStorage = p;
 805   rc = fts5StorageLoadTotals(p, 1);
 806
 807   if( rc==SQLITE_OK ){
 808     rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
 809   }
 810   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
 811     ctx.szCol = 0;
 812     if( pConfig->abUnindexed[ctx.iCol]==0 ){
 813       const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
 814       int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
 815       rc = sqlite3Fts5Tokenize(pConfig,
 816           FTS5_TOKENIZE_DOCUMENT,
 817           zText, nText,
 818           (void*)&ctx,
 819           fts5StorageInsertCallback
 820       );
 821     }
 822     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
 823     p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
 824   }
 825   p->nTotalRow++;
 826
 827   /* Write the %_docsize record */
 828   if( rc==SQLITE_OK ){
 829     rc = fts5StorageInsertDocsize(p, iRowid, &buf);
 830   }
 831   sqlite3_free(buf.p);
 832
 833   return rc;
 834 }
 835
 836 static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
 837   Fts5Config *pConfig = p->pConfig;
 838   char *zSql;
 839   int rc;
 840
 841   zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'",
 842       pConfig->zDb, pConfig->zName, zSuffix
 843   );
 844   if( zSql==0 ){
 845     rc = SQLITE_NOMEM;
 846   }else{
 847     sqlite3_stmt *pCnt = 0;
 848     rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
 849     if( rc==SQLITE_OK ){
 850       if( SQLITE_ROW==sqlite3_step(pCnt) ){
 851         *pnRow = sqlite3_column_int64(pCnt, 0);
 852       }
 853       rc = sqlite3_finalize(pCnt);
 854     }
 855   }
 856
 857   sqlite3_free(zSql);
 858   return rc;
 859 }
 860
 861 /*
 862 ** Context object used by sqlite3Fts5StorageIntegrity().
 863 */
 864 typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
 865 struct Fts5IntegrityCtx {
 866   i64 iRowid;
 867   int iCol;
 868   int szCol;
 869   u64 cksum;
 870   Fts5Termset *pTermset;
 871   Fts5Config *pConfig;
 872 };
 873
 874
 875 /*
 876 ** Tokenization callback used by integrity check.
 877 */
 878 static int fts5StorageIntegrityCallback(
 879   void *pContext,                 /* Pointer to Fts5IntegrityCtx object */
 880   int tflags,
 881   const char *pToken,             /* Buffer containing token */
 882   int nToken,                     /* Size of token in bytes */
 883   int iUnused1,                   /* Start offset of token */
 884   int iUnused2                    /* End offset of token */
 885 ){
 886   Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
 887   Fts5Termset *pTermset = pCtx->pTermset;
 888   int bPresent;
 889   int ii;
 890   int rc = SQLITE_OK;
 891   int iPos;
 892   int iCol;
 893
 894   UNUSED_PARAM2(iUnused1, iUnused2);
 895   if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
 896
 897   if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
 898     pCtx->szCol++;
 899   }
 900
 901   switch( pCtx->pConfig->eDetail ){
 902     case FTS5_DETAIL_FULL:
 903       iPos = pCtx->szCol-1;
 904       iCol = pCtx->iCol;
 905       break;
 906
 907     case FTS5_DETAIL_COLUMNS:
 908       iPos = pCtx->iCol;
 909       iCol = 0;
 910       break;
 911
 912     default:
 913       assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
 914       iPos = 0;
 915       iCol = 0;
 916       break;
 917   }
 918
 919   rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
 920   if( rc==SQLITE_OK && bPresent==0 ){
 921     pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
 922         pCtx->iRowid, iCol, iPos, 0, pToken, nToken
 923     );
 924   }
 925
 926   for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
 927     const int nChar = pCtx->pConfig->aPrefix[ii];
 928     int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
 929     if( nByte ){
 930       rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
 931       if( bPresent==0 ){
 932         pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
 933             pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
 934         );
 935       }
 936     }
 937   }
 938
 939   return rc;
 940 }
 941
 942 /*
 943 ** Check that the contents of the FTS index match that of the %_content
 944 ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
 945 ** some other SQLite error code if an error occurs while attempting to
 946 ** determine this.
 947 */
 948 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
 949   Fts5Config *pConfig = p->pConfig;
 950   int rc = SQLITE_OK;             /* Return code */
 951   int *aColSize;                  /* Array of size pConfig->nCol */
 952   i64 *aTotalSize;                /* Array of size pConfig->nCol */
 953   Fts5IntegrityCtx ctx;
 954   sqlite3_stmt *pScan;
 955   int bUseCksum;
 956
 957   memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
 958   ctx.pConfig = p->pConfig;
 959   aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64)));
 960   if( !aTotalSize ) return SQLITE_NOMEM;
 961   aColSize = (int*)&aTotalSize[pConfig->nCol];
 962   memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
 963
 964   bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL
 965            || (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg)
 966   );
 967   if( bUseCksum ){
 968     /* Generate the expected index checksum based on the contents of the
 969     ** %_content table. This block stores the checksum in ctx.cksum. */
 970     rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
 971     if( rc==SQLITE_OK ){
 972       int rc2;
 973       while( SQLITE_ROW==sqlite3_step(pScan) ){
 974         int i;
 975         ctx.iRowid = sqlite3_column_int64(pScan, 0);
 976         ctx.szCol = 0;
 977         if( pConfig->bColumnsize ){
 978           rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
 979         }
 980         if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
 981           rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
 982         }
 983         for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
 984           if( pConfig->abUnindexed[i] ) continue;
 985           ctx.iCol = i;
 986           ctx.szCol = 0;
 987           if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
 988             rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
 989           }
 990           if( rc==SQLITE_OK ){
 991             const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
 992             int nText = sqlite3_column_bytes(pScan, i+1);
 993             rc = sqlite3Fts5Tokenize(pConfig,
 994                 FTS5_TOKENIZE_DOCUMENT,
 995                 zText, nText,
 996                 (void*)&ctx,
 997                 fts5StorageIntegrityCallback
 998             );
 999           }
1000           if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
1001             rc = FTS5_CORRUPT;
1002           }
1003           aTotalSize[i] += ctx.szCol;
1004           if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
1005             sqlite3Fts5TermsetFree(ctx.pTermset);
1006             ctx.pTermset = 0;
1007           }
1008         }
1009         sqlite3Fts5TermsetFree(ctx.pTermset);
1010         ctx.pTermset = 0;
1011
1012         if( rc!=SQLITE_OK ) break;
1013       }
1014       rc2 = sqlite3_reset(pScan);
1015       if( rc==SQLITE_OK ) rc = rc2;
1016     }
1017
1018     /* Test that the "totals" (sometimes called "averages") record looks Ok */
1019     if( rc==SQLITE_OK ){
1020       int i;
1021       rc = fts5StorageLoadTotals(p, 0);
1022       for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
1023         if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT;
1024       }
1025     }
1026
1027     /* Check that the %_docsize and %_content tables contain the expected
1028     ** number of rows.  */
1029     if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
1030       i64 nRow = 0;
1031       rc = fts5StorageCount(p, "content", &nRow);
1032       if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
1033     }
1034     if( rc==SQLITE_OK && pConfig->bColumnsize ){
1035       i64 nRow = 0;
1036       rc = fts5StorageCount(p, "docsize", &nRow);
1037       if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
1038     }
1039   }
1040
1041   /* Pass the expected checksum down to the FTS index module. It will
1042   ** verify, amongst other things, that it matches the checksum generated by
1043   ** inspecting the index itself.  */
1044   if( rc==SQLITE_OK ){
1045     rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum);
1046   }
1047
1048   sqlite3_free(aTotalSize);
1049   return rc;
1050 }
1051
1052 /*
1053 ** Obtain an SQLite statement handle that may be used to read data from the
1054 ** %_content table.
1055 */
1056 int sqlite3Fts5StorageStmt(
1057   Fts5Storage *p,
1058   int eStmt,
1059   sqlite3_stmt **pp,
1060   char **pzErrMsg
1061 ){
1062   int rc;
1063   assert( eStmt==FTS5_STMT_SCAN_ASC
1064        || eStmt==FTS5_STMT_SCAN_DESC
1065        || eStmt==FTS5_STMT_LOOKUP
1066   );
1067   rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
1068   if( rc==SQLITE_OK ){
1069     assert( p->aStmt[eStmt]==*pp );
1070     p->aStmt[eStmt] = 0;
1071   }
1072   return rc;
1073 }
1074
1075 /*
1076 ** Release an SQLite statement handle obtained via an earlier call to
1077 ** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
1078 ** must match that passed to the sqlite3Fts5StorageStmt() call.
1079 */
1080 void sqlite3Fts5StorageStmtRelease(
1081   Fts5Storage *p,
1082   int eStmt,
1083   sqlite3_stmt *pStmt
1084 ){
1085   assert( eStmt==FTS5_STMT_SCAN_ASC
1086        || eStmt==FTS5_STMT_SCAN_DESC
1087        || eStmt==FTS5_STMT_LOOKUP
1088   );
1089   if( p->aStmt[eStmt]==0 ){
1090     sqlite3_reset(pStmt);
1091     p->aStmt[eStmt] = pStmt;
1092   }else{
1093     sqlite3_finalize(pStmt);
1094   }
1095 }
1096
1097 static int fts5StorageDecodeSizeArray(
1098   int *aCol, int nCol,            /* Array to populate */
1099   const u8 *aBlob, int nBlob      /* Record to read varints from */
1100 ){
1101   int i;
1102   int iOff = 0;
1103   for(i=0; i<nCol; i++){
1104     if( iOff>=nBlob ) return 1;
1105     iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]);
1106   }
1107   return (iOff!=nBlob);
1108 }
1109
1110 /*
1111 ** Argument aCol points to an array of integers containing one entry for
1112 ** each table column. This function reads the %_docsize record for the
1113 ** specified rowid and populates aCol[] with the results.
1114 **
1115 ** An SQLite error code is returned if an error occurs, or SQLITE_OK
1116 ** otherwise.
1117 */
1118 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
1119   int nCol = p->pConfig->nCol;    /* Number of user columns in table */
1120   sqlite3_stmt *pLookup = 0;      /* Statement to query %_docsize */
1121   int rc;                         /* Return Code */
1122
1123   assert( p->pConfig->bColumnsize );
1124   rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
1125   if( pLookup ){
1126     int bCorrupt = 1;
1127     assert( rc==SQLITE_OK );
1128     sqlite3_bind_int64(pLookup, 1, iRowid);
1129     if( SQLITE_ROW==sqlite3_step(pLookup) ){
1130       const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
1131       int nBlob = sqlite3_column_bytes(pLookup, 0);
1132       if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
1133         bCorrupt = 0;
1134       }
1135     }
1136     rc = sqlite3_reset(pLookup);
1137     if( bCorrupt && rc==SQLITE_OK ){
1138       rc = FTS5_CORRUPT;
1139     }
1140   }else{
1141     assert( rc!=SQLITE_OK );
1142   }
1143
1144   return rc;
1145 }
1146
1147 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
1148   int rc = fts5StorageLoadTotals(p, 0);
1149   if( rc==SQLITE_OK ){
1150     *pnToken = 0;
1151     if( iCol<0 ){
1152       int i;
1153       for(i=0; i<p->pConfig->nCol; i++){
1154         *pnToken += p->aTotalSize[i];
1155       }
1156     }else if( iCol<p->pConfig->nCol ){
1157       *pnToken = p->aTotalSize[iCol];
1158     }else{
1159       rc = SQLITE_RANGE;
1160     }
1161   }
1162   return rc;
1163 }
1164
1165 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
1166   int rc = fts5StorageLoadTotals(p, 0);
1167   if( rc==SQLITE_OK ){
1168     /* nTotalRow being zero does not necessarily indicate a corrupt
1169     ** database - it might be that the FTS5 table really does contain zero
1170     ** rows. However this function is only called from the xRowCount() API,
1171     ** and there is no way for that API to be invoked if the table contains
1172     ** no rows. Hence the FTS5_CORRUPT return.  */
1173     *pnRow = p->nTotalRow;
1174     if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT;
1175   }
1176   return rc;
1177 }
1178
1179 /*
1180 ** Flush any data currently held in-memory to disk.
1181 */
1182 int sqlite3Fts5StorageSync(Fts5Storage *p){
1183   int rc = SQLITE_OK;
1184   i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
1185   if( p->bTotalsValid ){
1186     rc = fts5StorageSaveTotals(p);
1187     if( rc==SQLITE_OK ){
1188       p->bTotalsValid = 0;
1189     }
1190   }
1191   if( rc==SQLITE_OK ){
1192     rc = sqlite3Fts5IndexSync(p->pIndex);
1193   }
1194   sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
1195   return rc;
1196 }
1197
1198 int sqlite3Fts5StorageRollback(Fts5Storage *p){
1199   p->bTotalsValid = 0;
1200   return sqlite3Fts5IndexRollback(p->pIndex);
1201 }
1202
1203 int sqlite3Fts5StorageConfigValue(
1204   Fts5Storage *p,
1205   const char *z,
1206   sqlite3_value *pVal,
1207   int iVal
1208 ){
1209   sqlite3_stmt *pReplace = 0;
1210   int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
1211   if( rc==SQLITE_OK ){
1212     sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
1213     if( pVal ){
1214       sqlite3_bind_value(pReplace, 2, pVal);
1215     }else{
1216       sqlite3_bind_int(pReplace, 2, iVal);
1217     }
1218     sqlite3_step(pReplace);
1219     rc = sqlite3_reset(pReplace);
1220     sqlite3_bind_null(pReplace, 1);
1221   }
1222   if( rc==SQLITE_OK && pVal ){
1223     int iNew = p->pConfig->iCookie + 1;
1224     rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
1225     if( rc==SQLITE_OK ){
1226       p->pConfig->iCookie = iNew;
1227     }
1228   }
1229   return rc;
1230 }