ext/fts5/fts5Int.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTS5INT_H
  15 #define _FTS5INT_H
  16
  17 #include "fts5.h"
  18 #include "sqlite3ext.h"
  19 SQLITE_EXTENSION_INIT1
  20
  21 #include <string.h>
  22 #include <assert.h>
  23
  24 #ifndef SQLITE_AMALGAMATION
  25
  26 typedef unsigned char  u8;
  27 typedef unsigned int   u32;
  28 typedef unsigned short u16;
  29 typedef short i16;
  30 typedef sqlite3_int64 i64;
  31 typedef sqlite3_uint64 u64;
  32
  33 #ifndef ArraySize
  34 # define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
  35 #endif
  36
  37 #define testcase(x)
  38
  39 #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
  40 # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
  41 #endif
  42 #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
  43 # define ALWAYS(X)      (1)
  44 # define NEVER(X)       (0)
  45 #elif !defined(NDEBUG)
  46 # define ALWAYS(X)      ((X)?1:(assert(0),0))
  47 # define NEVER(X)       ((X)?(assert(0),1):0)
  48 #else
  49 # define ALWAYS(X)      (X)
  50 # define NEVER(X)       (X)
  51 #endif
  52
  53 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
  54 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
  55
  56 /*
  57 ** Constants for the largest and smallest possible 64-bit signed integers.
  58 */
  59 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
  60 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
  61
  62 #endif
  63
  64 /* Truncate very long tokens to this many bytes. Hard limit is
  65 ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
  66 ** field that occurs at the start of each leaf page (see fts5_index.c). */
  67 #define FTS5_MAX_TOKEN_SIZE 32768
  68
  69 /*
  70 ** Maximum number of prefix indexes on single FTS5 table. This must be
  71 ** less than 32. If it is set to anything large than that, an #error
  72 ** directive in fts5_index.c will cause the build to fail.
  73 */
  74 #define FTS5_MAX_PREFIX_INDEXES 31
  75
  76 /*
  77 ** Maximum segments permitted in a single index
  78 */
  79 #define FTS5_MAX_SEGMENT 2000
  80
  81 #define FTS5_DEFAULT_NEARDIST 10
  82 #define FTS5_DEFAULT_RANK     "bm25"
  83
  84 /* Name of rank and rowid columns */
  85 #define FTS5_RANK_NAME "rank"
  86 #define FTS5_ROWID_NAME "rowid"
  87
  88 #ifdef SQLITE_DEBUG
  89 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
  90 int sqlite3Fts5Corrupt(void);
  91 #else
  92 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
  93 #endif
  94
  95 /*
  96 ** The assert_nc() macro is similar to the assert() macro, except that it
  97 ** is used for assert() conditions that are true only if it can be
  98 ** guranteed that the database is not corrupt.
  99 */
 100 #ifdef SQLITE_DEBUG
 101 extern int sqlite3_fts5_may_be_corrupt;
 102 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
 103 #else
 104 # define assert_nc(x) assert(x)
 105 #endif
 106
 107 /*
 108 ** A version of memcmp() that does not cause asan errors if one of the pointer
 109 ** parameters is NULL and the number of bytes to compare is zero.
 110 */
 111 #define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
 112
 113 /* Mark a function parameter as unused, to suppress nuisance compiler
 114 ** warnings. */
 115 #ifndef UNUSED_PARAM
 116 # define UNUSED_PARAM(X)  (void)(X)
 117 #endif
 118
 119 #ifndef UNUSED_PARAM2
 120 # define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
 121 #endif
 122
 123 typedef struct Fts5Global Fts5Global;
 124 typedef struct Fts5Colset Fts5Colset;
 125
 126 /* If a NEAR() clump or phrase may only match a specific set of columns,
 127 ** then an object of the following type is used to record the set of columns.
 128 ** Each entry in the aiCol[] array is a column that may be matched.
 129 **
 130 ** This object is used by fts5_expr.c and fts5_index.c.
 131 */
 132 struct Fts5Colset {
 133   int nCol;
 134   int aiCol[1];
 135 };
 136
 137
 138
 139 /**************************************************************************
 140 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 141 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 142 */
 143
 144 typedef struct Fts5Config Fts5Config;
 145
 146 /*
 147 ** An instance of the following structure encodes all information that can
 148 ** be gleaned from the CREATE VIRTUAL TABLE statement.
 149 **
 150 ** And all information loaded from the %_config table.
 151 **
 152 ** nAutomerge:
 153 **   The minimum number of segments that an auto-merge operation should
 154 **   attempt to merge together. A value of 1 sets the object to use the
 155 **   compile time default. Zero disables auto-merge altogether.
 156 **
 157 ** zContent:
 158 **
 159 ** zContentRowid:
 160 **   The value of the content_rowid= option, if one was specified. Or
 161 **   the string "rowid" otherwise. This text is not quoted - if it is
 162 **   used as part of an SQL statement it needs to be quoted appropriately.
 163 **
 164 ** zContentExprlist:
 165 **
 166 ** pzErrmsg:
 167 **   This exists in order to allow the fts5_index.c module to return a
 168 **   decent error message if it encounters a file-format version it does
 169 **   not understand.
 170 **
 171 ** bColumnsize:
 172 **   True if the %_docsize table is created.
 173 **
 174 ** bPrefixIndex:
 175 **   This is only used for debugging. If set to false, any prefix indexes
 176 **   are ignored. This value is configured using:
 177 **
 178 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 179 **
 180 */
 181 struct Fts5Config {
 182   sqlite3 *db;                    /* Database handle */
 183   char *zDb;                      /* Database holding FTS index (e.g. "main") */
 184   char *zName;                    /* Name of FTS index */
 185   int nCol;                       /* Number of columns */
 186   char **azCol;                   /* Column names */
 187   u8 *abUnindexed;                /* True for unindexed columns */
 188   int nPrefix;                    /* Number of prefix indexes */
 189   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
 190   int eContent;                   /* An FTS5_CONTENT value */
 191   char *zContent;                 /* content table */
 192   char *zContentRowid;            /* "content_rowid=" option value */
 193   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
 194   int eDetail;                    /* FTS5_DETAIL_XXX value */
 195   char *zContentExprlist;
 196   Fts5Tokenizer *pTok;
 197   fts5_tokenizer *pTokApi;
 198   int bLock;                      /* True when table is preparing statement */
 199   int ePattern;                   /* FTS_PATTERN_XXX constant */
 200
 201   /* Values loaded from the %_config table */
 202   int iVersion;                   /* fts5 file format 'version' */
 203   int iCookie;                    /* Incremented when %_config is modified */
 204   int pgsz;                       /* Approximate page size used in %_data */
 205   int nAutomerge;                 /* 'automerge' setting */
 206   int nCrisisMerge;               /* Maximum allowed segments per level */
 207   int nUsermerge;                 /* 'usermerge' setting */
 208   int nHashSize;                  /* Bytes of memory for in-memory hash */
 209   char *zRank;                    /* Name of rank function */
 210   char *zRankArgs;                /* Arguments to rank function */
 211   int bSecureDelete;              /* 'secure-delete' */
 212
 213   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
 214   char **pzErrmsg;
 215
 216 #ifdef SQLITE_DEBUG
 217   int bPrefixIndex;               /* True to use prefix-indexes */
 218 #endif
 219 };
 220
 221 /* Current expected value of %_config table 'version' field. And
 222 ** the expected version if the 'secure-delete' option has ever been
 223 ** set on the table.  */
 224 #define FTS5_CURRENT_VERSION               4
 225 #define FTS5_CURRENT_VERSION_SECUREDELETE  5
 226
 227 #define FTS5_CONTENT_NORMAL   0
 228 #define FTS5_CONTENT_NONE     1
 229 #define FTS5_CONTENT_EXTERNAL 2
 230
 231 #define FTS5_DETAIL_FULL      0
 232 #define FTS5_DETAIL_NONE      1
 233 #define FTS5_DETAIL_COLUMNS   2
 234
 235 #define FTS5_PATTERN_NONE     0
 236 #define FTS5_PATTERN_LIKE     65  /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
 237 #define FTS5_PATTERN_GLOB     66  /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
 238
 239 int sqlite3Fts5ConfigParse(
 240     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 241 );
 242 void sqlite3Fts5ConfigFree(Fts5Config*);
 243
 244 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 245
 246 int sqlite3Fts5Tokenize(
 247   Fts5Config *pConfig,            /* FTS5 Configuration object */
 248   int flags,                      /* FTS5_TOKENIZE_* flags */
 249   const char *pText, int nText,   /* Text to tokenize */
 250   void *pCtx,                     /* Context passed to xToken() */
 251   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 252 );
 253
 254 void sqlite3Fts5Dequote(char *z);
 255
 256 /* Load the contents of the %_config table */
 257 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
 258
 259 /* Set the value of a single config attribute */
 260 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 261
 262 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 263
 264 /*
 265 ** End of interface to code in fts5_config.c.
 266 **************************************************************************/
 267
 268 /**************************************************************************
 269 ** Interface to code in fts5_buffer.c.
 270 */
 271
 272 /*
 273 ** Buffer object for the incremental building of string data.
 274 */
 275 typedef struct Fts5Buffer Fts5Buffer;
 276 struct Fts5Buffer {
 277   u8 *p;
 278   int n;
 279   int nSpace;
 280 };
 281
 282 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 283 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 284 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 285 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 286 void sqlite3Fts5BufferFree(Fts5Buffer*);
 287 void sqlite3Fts5BufferZero(Fts5Buffer*);
 288 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 289 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 290
 291 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 292
 293 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 294 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
 295 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 296 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 297 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 298
 299 #define fts5BufferGrow(pRc,pBuf,nn) ( \
 300   (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
 301     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 302 )
 303
 304 /* Write and decode big-endian 32-bit integer values */
 305 void sqlite3Fts5Put32(u8*, int);
 306 int sqlite3Fts5Get32(const u8*);
 307
 308 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 309 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
 310
 311 typedef struct Fts5PoslistReader Fts5PoslistReader;
 312 struct Fts5PoslistReader {
 313   /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
 314   const u8 *a;                    /* Position list to iterate through */
 315   int n;                          /* Size of buffer at a[] in bytes */
 316   int i;                          /* Current offset in a[] */
 317
 318   u8 bFlag;                       /* For client use (any custom purpose) */
 319
 320   /* Output variables */
 321   u8 bEof;                        /* Set to true at EOF */
 322   i64 iPos;                       /* (iCol<<32) + iPos */
 323 };
 324 int sqlite3Fts5PoslistReaderInit(
 325   const u8 *a, int n,             /* Poslist buffer to iterate through */
 326   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 327 );
 328 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 329
 330 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 331 struct Fts5PoslistWriter {
 332   i64 iPrev;
 333 };
 334 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
 335 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 336
 337 int sqlite3Fts5PoslistNext64(
 338   const u8 *a, int n,             /* Buffer containing poslist */
 339   int *pi,                        /* IN/OUT: Offset within a[] */
 340   i64 *piOff                      /* IN/OUT: Current offset */
 341 );
 342
 343 /* Malloc utility */
 344 void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
 345 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 346
 347 /* Character set tests (like isspace(), isalpha() etc.) */
 348 int sqlite3Fts5IsBareword(char t);
 349
 350
 351 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
 352 typedef struct Fts5Termset Fts5Termset;
 353 int sqlite3Fts5TermsetNew(Fts5Termset**);
 354 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
 355 void sqlite3Fts5TermsetFree(Fts5Termset*);
 356
 357 /*
 358 ** End of interface to code in fts5_buffer.c.
 359 **************************************************************************/
 360
 361 /**************************************************************************
 362 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 363 ** to access the data stored in the %_data table.
 364 */
 365
 366 typedef struct Fts5Index Fts5Index;
 367 typedef struct Fts5IndexIter Fts5IndexIter;
 368
 369 struct Fts5IndexIter {
 370   i64 iRowid;
 371   const u8 *pData;
 372   int nData;
 373   u8 bEof;
 374 };
 375
 376 #define sqlite3Fts5IterEof(x) ((x)->bEof)
 377
 378 /*
 379 ** Values used as part of the flags argument passed to IndexQuery().
 380 */
 381 #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
 382 #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
 383 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 384 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 385
 386 /* The following are used internally by the fts5_index.c module. They are
 387 ** defined here only to make it easier to avoid clashes with the flags
 388 ** above. */
 389 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 390 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
 391 #define FTS5INDEX_QUERY_SKIPHASH   0x0040
 392
 393 /*
 394 ** Create/destroy an Fts5Index object.
 395 */
 396 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 397 int sqlite3Fts5IndexClose(Fts5Index *p);
 398
 399 /*
 400 ** Return a simple checksum value based on the arguments.
 401 */
 402 u64 sqlite3Fts5IndexEntryCksum(
 403   i64 iRowid,
 404   int iCol,
 405   int iPos,
 406   int iIdx,
 407   const char *pTerm,
 408   int nTerm
 409 );
 410
 411 /*
 412 ** Argument p points to a buffer containing utf-8 text that is n bytes in
 413 ** size. Return the number of bytes in the nChar character prefix of the
 414 ** buffer, or 0 if there are less than nChar characters in total.
 415 */
 416 int sqlite3Fts5IndexCharlenToBytelen(
 417   const char *p,
 418   int nByte,
 419   int nChar
 420 );
 421
 422 /*
 423 ** Open a new iterator to iterate though all rowids that match the
 424 ** specified token or token prefix.
 425 */
 426 int sqlite3Fts5IndexQuery(
 427   Fts5Index *p,                   /* FTS index to query */
 428   const char *pToken, int nToken, /* Token (or prefix) to query for */
 429   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
 430   Fts5Colset *pColset,            /* Match these columns only */
 431   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 432 );
 433
 434 /*
 435 ** The various operations on open token or token prefix iterators opened
 436 ** using sqlite3Fts5IndexQuery().
 437 */
 438 int sqlite3Fts5IterNext(Fts5IndexIter*);
 439 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 440
 441 /*
 442 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 443 */
 444 void sqlite3Fts5IterClose(Fts5IndexIter*);
 445
 446 /*
 447 ** Close the reader blob handle, if it is open.
 448 */
 449 void sqlite3Fts5IndexCloseReader(Fts5Index*);
 450
 451 /*
 452 ** This interface is used by the fts5vocab module.
 453 */
 454 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 455 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 456 void *sqlite3Fts5StructureRef(Fts5Index*);
 457 void sqlite3Fts5StructureRelease(void*);
 458 int sqlite3Fts5StructureTest(Fts5Index*, void*);
 459
 460
 461 /*
 462 ** Insert or remove data to or from the index. Each time a document is
 463 ** added to or removed from the index, this function is called one or more
 464 ** times.
 465 **
 466 ** For an insert, it must be called once for each token in the new document.
 467 ** If the operation is a delete, it must be called (at least) once for each
 468 ** unique token in the document with an iCol value less than zero. The iPos
 469 ** argument is ignored for a delete.
 470 */
 471 int sqlite3Fts5IndexWrite(
 472   Fts5Index *p,                   /* Index to write to */
 473   int iCol,                       /* Column token appears in (-ve -> delete) */
 474   int iPos,                       /* Position of token within column */
 475   const char *pToken, int nToken  /* Token to add or remove to or from index */
 476 );
 477
 478 /*
 479 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 480 ** document iDocid.
 481 */
 482 int sqlite3Fts5IndexBeginWrite(
 483   Fts5Index *p,                   /* Index to write to */
 484   int bDelete,                    /* True if current operation is a delete */
 485   i64 iDocid                      /* Docid to add or remove data from */
 486 );
 487
 488 /*
 489 ** Flush any data stored in the in-memory hash tables to the database.
 490 ** Also close any open blob handles.
 491 */
 492 int sqlite3Fts5IndexSync(Fts5Index *p);
 493
 494 /*
 495 ** Discard any data stored in the in-memory hash tables. Do not write it
 496 ** to the database. Additionally, assume that the contents of the %_data
 497 ** table may have changed on disk. So any in-memory caches of %_data
 498 ** records must be invalidated.
 499 */
 500 int sqlite3Fts5IndexRollback(Fts5Index *p);
 501
 502 /*
 503 ** Get or set the "averages" values.
 504 */
 505 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 506 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 507
 508 /*
 509 ** Functions called by the storage module as part of integrity-check.
 510 */
 511 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
 512
 513 /*
 514 ** Called during virtual module initialization to register UDF
 515 ** fts5_decode() with SQLite
 516 */
 517 int sqlite3Fts5IndexInit(sqlite3*);
 518
 519 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 520
 521 /*
 522 ** Return the total number of entries read from the %_data table by
 523 ** this connection since it was created.
 524 */
 525 int sqlite3Fts5IndexReads(Fts5Index *p);
 526
 527 int sqlite3Fts5IndexReinit(Fts5Index *p);
 528 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 529 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 530 int sqlite3Fts5IndexReset(Fts5Index *p);
 531
 532 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 533
 534 /*
 535 ** End of interface to code in fts5_index.c.
 536 **************************************************************************/
 537
 538 /**************************************************************************
 539 ** Interface to code in fts5_varint.c.
 540 */
 541 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 542 int sqlite3Fts5GetVarintLen(u32 iVal);
 543 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 544 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 545
 546 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
 547 #define fts5GetVarint    sqlite3Fts5GetVarint
 548
 549 #define fts5FastGetVarint32(a, iOff, nVal) {      \
 550   nVal = (a)[iOff++];                             \
 551   if( nVal & 0x80 ){                              \
 552     iOff--;                                       \
 553     iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
 554   }                                               \
 555 }
 556
 557
 558 /*
 559 ** End of interface to code in fts5_varint.c.
 560 **************************************************************************/
 561
 562
 563 /**************************************************************************
 564 ** Interface to code in fts5_main.c.
 565 */
 566
 567 /*
 568 ** Virtual-table object.
 569 */
 570 typedef struct Fts5Table Fts5Table;
 571 struct Fts5Table {
 572   sqlite3_vtab base;              /* Base class used by SQLite core */
 573   Fts5Config *pConfig;            /* Virtual table configuration */
 574   Fts5Index *pIndex;              /* Full-text index */
 575 };
 576
 577 int sqlite3Fts5GetTokenizer(
 578   Fts5Global*,
 579   const char **azArg,
 580   int nArg,
 581   Fts5Config*,
 582   char **pzErr
 583 );
 584
 585 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 586
 587 int sqlite3Fts5FlushToDisk(Fts5Table*);
 588
 589 /*
 590 ** End of interface to code in fts5.c.
 591 **************************************************************************/
 592
 593 /**************************************************************************
 594 ** Interface to code in fts5_hash.c.
 595 */
 596 typedef struct Fts5Hash Fts5Hash;
 597
 598 /*
 599 ** Create a hash table, free a hash table.
 600 */
 601 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 602 void sqlite3Fts5HashFree(Fts5Hash*);
 603
 604 int sqlite3Fts5HashWrite(
 605   Fts5Hash*,
 606   i64 iRowid,                     /* Rowid for this entry */
 607   int iCol,                       /* Column token appears in (-ve -> delete) */
 608   int iPos,                       /* Position of token within column */
 609   char bByte,
 610   const char *pToken, int nToken  /* Token to add or remove to or from index */
 611 );
 612
 613 /*
 614 ** Empty (but do not delete) a hash table.
 615 */
 616 void sqlite3Fts5HashClear(Fts5Hash*);
 617
 618 int sqlite3Fts5HashQuery(
 619   Fts5Hash*,                      /* Hash table to query */
 620   int nPre,
 621   const char *pTerm, int nTerm,   /* Query term */
 622   void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
 623   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 624 );
 625
 626 int sqlite3Fts5HashScanInit(
 627   Fts5Hash*,                      /* Hash table to query */
 628   const char *pTerm, int nTerm    /* Query prefix */
 629 );
 630 void sqlite3Fts5HashScanNext(Fts5Hash*);
 631 int sqlite3Fts5HashScanEof(Fts5Hash*);
 632 void sqlite3Fts5HashScanEntry(Fts5Hash *,
 633   const char **pzTerm,            /* OUT: term (nul-terminated) */
 634   const u8 **ppDoclist,           /* OUT: pointer to doclist */
 635   int *pnDoclist                  /* OUT: size of doclist in bytes */
 636 );
 637
 638
 639 /*
 640 ** End of interface to code in fts5_hash.c.
 641 **************************************************************************/
 642
 643 /**************************************************************************
 644 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
 645 ** code to access the data stored in the %_content and %_docsize tables.
 646 */
 647
 648 #define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
 649 #define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
 650 #define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
 651
 652 typedef struct Fts5Storage Fts5Storage;
 653
 654 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 655 int sqlite3Fts5StorageClose(Fts5Storage *p);
 656 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 657
 658 int sqlite3Fts5DropAll(Fts5Config*);
 659 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 660
 661 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 662 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 663 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 664
 665 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
 666
 667 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 668 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 669
 670 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 671 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 672 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 673
 674 int sqlite3Fts5StorageSync(Fts5Storage *p);
 675 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 676
 677 int sqlite3Fts5StorageConfigValue(
 678     Fts5Storage *p, const char*, sqlite3_value*, int
 679 );
 680
 681 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 682 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 683 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 684 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 685 int sqlite3Fts5StorageReset(Fts5Storage *p);
 686
 687 /*
 688 ** End of interface to code in fts5_storage.c.
 689 **************************************************************************/
 690
 691
 692 /**************************************************************************
 693 ** Interface to code in fts5_expr.c.
 694 */
 695 typedef struct Fts5Expr Fts5Expr;
 696 typedef struct Fts5ExprNode Fts5ExprNode;
 697 typedef struct Fts5Parse Fts5Parse;
 698 typedef struct Fts5Token Fts5Token;
 699 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
 700 typedef struct Fts5ExprNearset Fts5ExprNearset;
 701
 702 struct Fts5Token {
 703   const char *p;                  /* Token text (not NULL terminated) */
 704   int n;                          /* Size of buffer p in bytes */
 705 };
 706
 707 /* Parse a MATCH expression. */
 708 int sqlite3Fts5ExprNew(
 709   Fts5Config *pConfig,
 710   int bPhraseToAnd,
 711   int iCol,                       /* Column on LHS of MATCH operator */
 712   const char *zExpr,
 713   Fts5Expr **ppNew,
 714   char **pzErr
 715 );
 716 int sqlite3Fts5ExprPattern(
 717   Fts5Config *pConfig,
 718   int bGlob,
 719   int iCol,
 720   const char *zText,
 721   Fts5Expr **pp
 722 );
 723
 724 /*
 725 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
 726 **     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
 727 **     rc = sqlite3Fts5ExprNext(pExpr)
 728 ** ){
 729 **   // The document with rowid iRowid matches the expression!
 730 **   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
 731 ** }
 732 */
 733 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
 734 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
 735 int sqlite3Fts5ExprEof(Fts5Expr*);
 736 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 737
 738 void sqlite3Fts5ExprFree(Fts5Expr*);
 739 int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
 740
 741 /* Called during startup to register a UDF with SQLite */
 742 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 743
 744 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 745 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 746 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 747
 748 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
 749 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 750 int sqlite3Fts5ExprPopulatePoslists(
 751     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 752 );
 753 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 754
 755 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 756
 757 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 758
 759 /*******************************************
 760 ** The fts5_expr.c API above this point is used by the other hand-written
 761 ** C code in this module. The interfaces below this point are called by
 762 ** the parser code in fts5parse.y.  */
 763
 764 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 765
 766 Fts5ExprNode *sqlite3Fts5ParseNode(
 767   Fts5Parse *pParse,
 768   int eType,
 769   Fts5ExprNode *pLeft,
 770   Fts5ExprNode *pRight,
 771   Fts5ExprNearset *pNear
 772 );
 773
 774 Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
 775   Fts5Parse *pParse,
 776   Fts5ExprNode *pLeft,
 777   Fts5ExprNode *pRight
 778 );
 779
 780 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 781   Fts5Parse *pParse,
 782   Fts5ExprPhrase *pPhrase,
 783   Fts5Token *pToken,
 784   int bPrefix
 785 );
 786
 787 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
 788
 789 Fts5ExprNearset *sqlite3Fts5ParseNearset(
 790   Fts5Parse*,
 791   Fts5ExprNearset*,
 792   Fts5ExprPhrase*
 793 );
 794
 795 Fts5Colset *sqlite3Fts5ParseColset(
 796   Fts5Parse*,
 797   Fts5Colset*,
 798   Fts5Token *
 799 );
 800
 801 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 802 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 803 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 804
 805 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 806 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 807 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 808 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 809 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 810
 811 /*
 812 ** End of interface to code in fts5_expr.c.
 813 **************************************************************************/
 814
 815
 816
 817 /**************************************************************************
 818 ** Interface to code in fts5_aux.c.
 819 */
 820
 821 int sqlite3Fts5AuxInit(fts5_api*);
 822 /*
 823 ** End of interface to code in fts5_aux.c.
 824 **************************************************************************/
 825
 826 /**************************************************************************
 827 ** Interface to code in fts5_tokenizer.c.
 828 */
 829
 830 int sqlite3Fts5TokenizerInit(fts5_api*);
 831 int sqlite3Fts5TokenizerPattern(
 832     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
 833     Fts5Tokenizer *pTok
 834 );
 835 /*
 836 ** End of interface to code in fts5_tokenizer.c.
 837 **************************************************************************/
 838
 839 /**************************************************************************
 840 ** Interface to code in fts5_vocab.c.
 841 */
 842
 843 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 844
 845 /*
 846 ** End of interface to code in fts5_vocab.c.
 847 **************************************************************************/
 848
 849
 850 /**************************************************************************
 851 ** Interface to automatically generated code in fts5_unicode2.c.
 852 */
 853 int sqlite3Fts5UnicodeIsdiacritic(int c);
 854 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 855
 856 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
 857 int sqlite3Fts5UnicodeCategory(u32 iCode);
 858 void sqlite3Fts5UnicodeAscii(u8*, u8*);
 859 /*
 860 ** End of interface to code in fts5_unicode2.c.
 861 **************************************************************************/
 862
 863 #endif