ext/fts5/fts5Int.h

   1 /*
   2 ** 2014 May 31
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 */
  14 #ifndef _FTS5INT_H
  15 #define _FTS5INT_H
  16
  17 #include "fts5.h"
  18 #include "sqlite3ext.h"
  19 SQLITE_EXTENSION_INIT1
  20
  21 #include <string.h>
  22 #include <assert.h>
  23
  24 #ifndef SQLITE_AMALGAMATION
  25
  26 typedef unsigned char  u8;
  27 typedef unsigned int   u32;
  28 typedef unsigned short u16;
  29 typedef short i16;
  30 typedef sqlite3_int64 i64;
  31 typedef sqlite3_uint64 u64;
  32
  33 #ifndef ArraySize
  34 # define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
  35 #endif
  36
  37 #define testcase(x)
  38
  39 #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
  40 # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
  41 #endif
  42 #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
  43 # define ALWAYS(X)      (1)
  44 # define NEVER(X)       (0)
  45 #elif !defined(NDEBUG)
  46 # define ALWAYS(X)      ((X)?1:(assert(0),0))
  47 # define NEVER(X)       ((X)?(assert(0),1):0)
  48 #else
  49 # define ALWAYS(X)      (X)
  50 # define NEVER(X)       (X)
  51 #endif
  52
  53 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
  54 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
  55
  56 /*
  57 ** Constants for the largest and smallest possible 64-bit signed integers.
  58 */
  59 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
  60 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
  61
  62 #endif
  63
  64 /* Truncate very long tokens to this many bytes. Hard limit is
  65 ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
  66 ** field that occurs at the start of each leaf page (see fts5_index.c). */
  67 #define FTS5_MAX_TOKEN_SIZE 32768
  68
  69 /*
  70 ** Maximum number of prefix indexes on single FTS5 table. This must be
  71 ** less than 32. If it is set to anything large than that, an #error
  72 ** directive in fts5_index.c will cause the build to fail.
  73 */
  74 #define FTS5_MAX_PREFIX_INDEXES 31
  75
  76 /*
  77 ** Maximum segments permitted in a single index
  78 */
  79 #define FTS5_MAX_SEGMENT 2000
  80
  81 #define FTS5_DEFAULT_NEARDIST 10
  82 #define FTS5_DEFAULT_RANK     "bm25"
  83
  84 /* Name of rank and rowid columns */
  85 #define FTS5_RANK_NAME "rank"
  86 #define FTS5_ROWID_NAME "rowid"
  87
  88 #ifdef SQLITE_DEBUG
  89 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
  90 int sqlite3Fts5Corrupt(void);
  91 #else
  92 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
  93 #endif
  94
  95 /*
  96 ** The assert_nc() macro is similar to the assert() macro, except that it
  97 ** is used for assert() conditions that are true only if it can be
  98 ** guranteed that the database is not corrupt.
  99 */
 100 #ifdef SQLITE_DEBUG
 101 extern int sqlite3_fts5_may_be_corrupt;
 102 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
 103 #else
 104 # define assert_nc(x) assert(x)
 105 #endif
 106
 107 /*
 108 ** A version of memcmp() that does not cause asan errors if one of the pointer
 109 ** parameters is NULL and the number of bytes to compare is zero.
 110 */
 111 #define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
 112
 113 /* Mark a function parameter as unused, to suppress nuisance compiler
 114 ** warnings. */
 115 #ifndef UNUSED_PARAM
 116 # define UNUSED_PARAM(X)  (void)(X)
 117 #endif
 118
 119 #ifndef UNUSED_PARAM2
 120 # define UNUSED_PARAM2(X, Y)  (void)(X), (void)(Y)
 121 #endif
 122
 123 typedef struct Fts5Global Fts5Global;
 124 typedef struct Fts5Colset Fts5Colset;
 125
 126 /* If a NEAR() clump or phrase may only match a specific set of columns,
 127 ** then an object of the following type is used to record the set of columns.
 128 ** Each entry in the aiCol[] array is a column that may be matched.
 129 **
 130 ** This object is used by fts5_expr.c and fts5_index.c.
 131 */
 132 struct Fts5Colset {
 133   int nCol;
 134   int aiCol[1];
 135 };
 136
 137
 138
 139 /**************************************************************************
 140 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
 141 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
 142 */
 143
 144 typedef struct Fts5Config Fts5Config;
 145
 146 /*
 147 ** An instance of the following structure encodes all information that can
 148 ** be gleaned from the CREATE VIRTUAL TABLE statement.
 149 **
 150 ** And all information loaded from the %_config table.
 151 **
 152 ** nAutomerge:
 153 **   The minimum number of segments that an auto-merge operation should
 154 **   attempt to merge together. A value of 1 sets the object to use the
 155 **   compile time default. Zero disables auto-merge altogether.
 156 **
 157 ** zContent:
 158 **
 159 ** zContentRowid:
 160 **   The value of the content_rowid= option, if one was specified. Or
 161 **   the string "rowid" otherwise. This text is not quoted - if it is
 162 **   used as part of an SQL statement it needs to be quoted appropriately.
 163 **
 164 ** zContentExprlist:
 165 **
 166 ** pzErrmsg:
 167 **   This exists in order to allow the fts5_index.c module to return a
 168 **   decent error message if it encounters a file-format version it does
 169 **   not understand.
 170 **
 171 ** bColumnsize:
 172 **   True if the %_docsize table is created.
 173 **
 174 ** bPrefixIndex:
 175 **   This is only used for debugging. If set to false, any prefix indexes
 176 **   are ignored. This value is configured using:
 177 **
 178 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 179 **
 180 */
 181 struct Fts5Config {
 182   sqlite3 *db;                    /* Database handle */
 183   char *zDb;                      /* Database holding FTS index (e.g. "main") */
 184   char *zName;                    /* Name of FTS index */
 185   int nCol;                       /* Number of columns */
 186   char **azCol;                   /* Column names */
 187   u8 *abUnindexed;                /* True for unindexed columns */
 188   int nPrefix;                    /* Number of prefix indexes */
 189   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
 190   int eContent;                   /* An FTS5_CONTENT value */
 191   char *zContent;                 /* content table */
 192   char *zContentRowid;            /* "content_rowid=" option value */
 193   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
 194   int eDetail;                    /* FTS5_DETAIL_XXX value */
 195   char *zContentExprlist;
 196   Fts5Tokenizer *pTok;
 197   fts5_tokenizer *pTokApi;
 198   int bLock;                      /* True when table is preparing statement */
 199   int ePattern;                   /* FTS_PATTERN_XXX constant */
 200
 201   /* Values loaded from the %_config table */
 202   int iCookie;                    /* Incremented when %_config is modified */
 203   int pgsz;                       /* Approximate page size used in %_data */
 204   int nAutomerge;                 /* 'automerge' setting */
 205   int nCrisisMerge;               /* Maximum allowed segments per level */
 206   int nUsermerge;                 /* 'usermerge' setting */
 207   int nHashSize;                  /* Bytes of memory for in-memory hash */
 208   char *zRank;                    /* Name of rank function */
 209   char *zRankArgs;                /* Arguments to rank function */
 210
 211   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
 212   char **pzErrmsg;
 213
 214 #ifdef SQLITE_DEBUG
 215   int bPrefixIndex;               /* True to use prefix-indexes */
 216 #endif
 217 };
 218
 219 /* Current expected value of %_config table 'version' field */
 220 #define FTS5_CURRENT_VERSION  4
 221
 222 #define FTS5_CONTENT_NORMAL   0
 223 #define FTS5_CONTENT_NONE     1
 224 #define FTS5_CONTENT_EXTERNAL 2
 225
 226 #define FTS5_DETAIL_FULL      0
 227 #define FTS5_DETAIL_NONE      1
 228 #define FTS5_DETAIL_COLUMNS   2
 229
 230 #define FTS5_PATTERN_NONE     0
 231 #define FTS5_PATTERN_LIKE     65  /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
 232 #define FTS5_PATTERN_GLOB     66  /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
 233
 234 int sqlite3Fts5ConfigParse(
 235     Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
 236 );
 237 void sqlite3Fts5ConfigFree(Fts5Config*);
 238
 239 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
 240
 241 int sqlite3Fts5Tokenize(
 242   Fts5Config *pConfig,            /* FTS5 Configuration object */
 243   int flags,                      /* FTS5_TOKENIZE_* flags */
 244   const char *pText, int nText,   /* Text to tokenize */
 245   void *pCtx,                     /* Context passed to xToken() */
 246   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 247 );
 248
 249 void sqlite3Fts5Dequote(char *z);
 250
 251 /* Load the contents of the %_config table */
 252 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
 253
 254 /* Set the value of a single config attribute */
 255 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 256
 257 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 258
 259 /*
 260 ** End of interface to code in fts5_config.c.
 261 **************************************************************************/
 262
 263 /**************************************************************************
 264 ** Interface to code in fts5_buffer.c.
 265 */
 266
 267 /*
 268 ** Buffer object for the incremental building of string data.
 269 */
 270 typedef struct Fts5Buffer Fts5Buffer;
 271 struct Fts5Buffer {
 272   u8 *p;
 273   int n;
 274   int nSpace;
 275 };
 276
 277 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
 278 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
 279 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
 280 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
 281 void sqlite3Fts5BufferFree(Fts5Buffer*);
 282 void sqlite3Fts5BufferZero(Fts5Buffer*);
 283 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
 284 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
 285
 286 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 287
 288 #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
 289 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
 290 #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
 291 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
 292 #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
 293
 294 #define fts5BufferGrow(pRc,pBuf,nn) ( \
 295   (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
 296     sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
 297 )
 298
 299 /* Write and decode big-endian 32-bit integer values */
 300 void sqlite3Fts5Put32(u8*, int);
 301 int sqlite3Fts5Get32(const u8*);
 302
 303 #define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 304 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
 305
 306 typedef struct Fts5PoslistReader Fts5PoslistReader;
 307 struct Fts5PoslistReader {
 308   /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
 309   const u8 *a;                    /* Position list to iterate through */
 310   int n;                          /* Size of buffer at a[] in bytes */
 311   int i;                          /* Current offset in a[] */
 312
 313   u8 bFlag;                       /* For client use (any custom purpose) */
 314
 315   /* Output variables */
 316   u8 bEof;                        /* Set to true at EOF */
 317   i64 iPos;                       /* (iCol<<32) + iPos */
 318 };
 319 int sqlite3Fts5PoslistReaderInit(
 320   const u8 *a, int n,             /* Poslist buffer to iterate through */
 321   Fts5PoslistReader *pIter        /* Iterator object to initialize */
 322 );
 323 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
 324
 325 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
 326 struct Fts5PoslistWriter {
 327   i64 iPrev;
 328 };
 329 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
 330 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
 331
 332 int sqlite3Fts5PoslistNext64(
 333   const u8 *a, int n,             /* Buffer containing poslist */
 334   int *pi,                        /* IN/OUT: Offset within a[] */
 335   i64 *piOff                      /* IN/OUT: Current offset */
 336 );
 337
 338 /* Malloc utility */
 339 void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
 340 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
 341
 342 /* Character set tests (like isspace(), isalpha() etc.) */
 343 int sqlite3Fts5IsBareword(char t);
 344
 345
 346 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
 347 typedef struct Fts5Termset Fts5Termset;
 348 int sqlite3Fts5TermsetNew(Fts5Termset**);
 349 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
 350 void sqlite3Fts5TermsetFree(Fts5Termset*);
 351
 352 /*
 353 ** End of interface to code in fts5_buffer.c.
 354 **************************************************************************/
 355
 356 /**************************************************************************
 357 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
 358 ** to access the data stored in the %_data table.
 359 */
 360
 361 typedef struct Fts5Index Fts5Index;
 362 typedef struct Fts5IndexIter Fts5IndexIter;
 363
 364 struct Fts5IndexIter {
 365   i64 iRowid;
 366   const u8 *pData;
 367   int nData;
 368   u8 bEof;
 369 };
 370
 371 #define sqlite3Fts5IterEof(x) ((x)->bEof)
 372
 373 /*
 374 ** Values used as part of the flags argument passed to IndexQuery().
 375 */
 376 #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
 377 #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
 378 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
 379 #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 380
 381 /* The following are used internally by the fts5_index.c module. They are
 382 ** defined here only to make it easier to avoid clashes with the flags
 383 ** above. */
 384 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 385 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
 386
 387 /*
 388 ** Create/destroy an Fts5Index object.
 389 */
 390 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
 391 int sqlite3Fts5IndexClose(Fts5Index *p);
 392
 393 /*
 394 ** Return a simple checksum value based on the arguments.
 395 */
 396 u64 sqlite3Fts5IndexEntryCksum(
 397   i64 iRowid,
 398   int iCol,
 399   int iPos,
 400   int iIdx,
 401   const char *pTerm,
 402   int nTerm
 403 );
 404
 405 /*
 406 ** Argument p points to a buffer containing utf-8 text that is n bytes in
 407 ** size. Return the number of bytes in the nChar character prefix of the
 408 ** buffer, or 0 if there are less than nChar characters in total.
 409 */
 410 int sqlite3Fts5IndexCharlenToBytelen(
 411   const char *p,
 412   int nByte,
 413   int nChar
 414 );
 415
 416 /*
 417 ** Open a new iterator to iterate though all rowids that match the
 418 ** specified token or token prefix.
 419 */
 420 int sqlite3Fts5IndexQuery(
 421   Fts5Index *p,                   /* FTS index to query */
 422   const char *pToken, int nToken, /* Token (or prefix) to query for */
 423   int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
 424   Fts5Colset *pColset,            /* Match these columns only */
 425   Fts5IndexIter **ppIter          /* OUT: New iterator object */
 426 );
 427
 428 /*
 429 ** The various operations on open token or token prefix iterators opened
 430 ** using sqlite3Fts5IndexQuery().
 431 */
 432 int sqlite3Fts5IterNext(Fts5IndexIter*);
 433 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 434
 435 /*
 436 ** Close an iterator opened by sqlite3Fts5IndexQuery().
 437 */
 438 void sqlite3Fts5IterClose(Fts5IndexIter*);
 439
 440 /*
 441 ** Close the reader blob handle, if it is open.
 442 */
 443 void sqlite3Fts5IndexCloseReader(Fts5Index*);
 444
 445 /*
 446 ** This interface is used by the fts5vocab module.
 447 */
 448 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
 449 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
 450 void *sqlite3Fts5StructureRef(Fts5Index*);
 451 void sqlite3Fts5StructureRelease(void*);
 452 int sqlite3Fts5StructureTest(Fts5Index*, void*);
 453
 454
 455 /*
 456 ** Insert or remove data to or from the index. Each time a document is
 457 ** added to or removed from the index, this function is called one or more
 458 ** times.
 459 **
 460 ** For an insert, it must be called once for each token in the new document.
 461 ** If the operation is a delete, it must be called (at least) once for each
 462 ** unique token in the document with an iCol value less than zero. The iPos
 463 ** argument is ignored for a delete.
 464 */
 465 int sqlite3Fts5IndexWrite(
 466   Fts5Index *p,                   /* Index to write to */
 467   int iCol,                       /* Column token appears in (-ve -> delete) */
 468   int iPos,                       /* Position of token within column */
 469   const char *pToken, int nToken  /* Token to add or remove to or from index */
 470 );
 471
 472 /*
 473 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
 474 ** document iDocid.
 475 */
 476 int sqlite3Fts5IndexBeginWrite(
 477   Fts5Index *p,                   /* Index to write to */
 478   int bDelete,                    /* True if current operation is a delete */
 479   i64 iDocid                      /* Docid to add or remove data from */
 480 );
 481
 482 /*
 483 ** Flush any data stored in the in-memory hash tables to the database.
 484 ** Also close any open blob handles.
 485 */
 486 int sqlite3Fts5IndexSync(Fts5Index *p);
 487
 488 /*
 489 ** Discard any data stored in the in-memory hash tables. Do not write it
 490 ** to the database. Additionally, assume that the contents of the %_data
 491 ** table may have changed on disk. So any in-memory caches of %_data
 492 ** records must be invalidated.
 493 */
 494 int sqlite3Fts5IndexRollback(Fts5Index *p);
 495
 496 /*
 497 ** Get or set the "averages" values.
 498 */
 499 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
 500 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
 501
 502 /*
 503 ** Functions called by the storage module as part of integrity-check.
 504 */
 505 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
 506
 507 /*
 508 ** Called during virtual module initialization to register UDF
 509 ** fts5_decode() with SQLite
 510 */
 511 int sqlite3Fts5IndexInit(sqlite3*);
 512
 513 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
 514
 515 /*
 516 ** Return the total number of entries read from the %_data table by
 517 ** this connection since it was created.
 518 */
 519 int sqlite3Fts5IndexReads(Fts5Index *p);
 520
 521 int sqlite3Fts5IndexReinit(Fts5Index *p);
 522 int sqlite3Fts5IndexOptimize(Fts5Index *p);
 523 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
 524 int sqlite3Fts5IndexReset(Fts5Index *p);
 525
 526 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 527
 528 /*
 529 ** End of interface to code in fts5_index.c.
 530 **************************************************************************/
 531
 532 /**************************************************************************
 533 ** Interface to code in fts5_varint.c.
 534 */
 535 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 536 int sqlite3Fts5GetVarintLen(u32 iVal);
 537 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 538 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 539
 540 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
 541 #define fts5GetVarint    sqlite3Fts5GetVarint
 542
 543 #define fts5FastGetVarint32(a, iOff, nVal) {      \
 544   nVal = (a)[iOff++];                             \
 545   if( nVal & 0x80 ){                              \
 546     iOff--;                                       \
 547     iOff += fts5GetVarint32(&(a)[iOff], nVal);    \
 548   }                                               \
 549 }
 550
 551
 552 /*
 553 ** End of interface to code in fts5_varint.c.
 554 **************************************************************************/
 555
 556
 557 /**************************************************************************
 558 ** Interface to code in fts5_main.c.
 559 */
 560
 561 /*
 562 ** Virtual-table object.
 563 */
 564 typedef struct Fts5Table Fts5Table;
 565 struct Fts5Table {
 566   sqlite3_vtab base;              /* Base class used by SQLite core */
 567   Fts5Config *pConfig;            /* Virtual table configuration */
 568   Fts5Index *pIndex;              /* Full-text index */
 569 };
 570
 571 int sqlite3Fts5GetTokenizer(
 572   Fts5Global*,
 573   const char **azArg,
 574   int nArg,
 575   Fts5Config*,
 576   char **pzErr
 577 );
 578
 579 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 580
 581 int sqlite3Fts5FlushToDisk(Fts5Table*);
 582
 583 /*
 584 ** End of interface to code in fts5.c.
 585 **************************************************************************/
 586
 587 /**************************************************************************
 588 ** Interface to code in fts5_hash.c.
 589 */
 590 typedef struct Fts5Hash Fts5Hash;
 591
 592 /*
 593 ** Create a hash table, free a hash table.
 594 */
 595 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
 596 void sqlite3Fts5HashFree(Fts5Hash*);
 597
 598 int sqlite3Fts5HashWrite(
 599   Fts5Hash*,
 600   i64 iRowid,                     /* Rowid for this entry */
 601   int iCol,                       /* Column token appears in (-ve -> delete) */
 602   int iPos,                       /* Position of token within column */
 603   char bByte,
 604   const char *pToken, int nToken  /* Token to add or remove to or from index */
 605 );
 606
 607 /*
 608 ** Empty (but do not delete) a hash table.
 609 */
 610 void sqlite3Fts5HashClear(Fts5Hash*);
 611
 612 int sqlite3Fts5HashQuery(
 613   Fts5Hash*,                      /* Hash table to query */
 614   int nPre,
 615   const char *pTerm, int nTerm,   /* Query term */
 616   void **ppObj,                   /* OUT: Pointer to doclist for pTerm */
 617   int *pnDoclist                  /* OUT: Size of doclist in bytes */
 618 );
 619
 620 int sqlite3Fts5HashScanInit(
 621   Fts5Hash*,                      /* Hash table to query */
 622   const char *pTerm, int nTerm    /* Query prefix */
 623 );
 624 void sqlite3Fts5HashScanNext(Fts5Hash*);
 625 int sqlite3Fts5HashScanEof(Fts5Hash*);
 626 void sqlite3Fts5HashScanEntry(Fts5Hash *,
 627   const char **pzTerm,            /* OUT: term (nul-terminated) */
 628   const u8 **ppDoclist,           /* OUT: pointer to doclist */
 629   int *pnDoclist                  /* OUT: size of doclist in bytes */
 630 );
 631
 632
 633 /*
 634 ** End of interface to code in fts5_hash.c.
 635 **************************************************************************/
 636
 637 /**************************************************************************
 638 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
 639 ** code to access the data stored in the %_content and %_docsize tables.
 640 */
 641
 642 #define FTS5_STMT_SCAN_ASC  0     /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
 643 #define FTS5_STMT_SCAN_DESC 1     /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
 644 #define FTS5_STMT_LOOKUP    2     /* SELECT rowid, * FROM ... WHERE rowid=? */
 645
 646 typedef struct Fts5Storage Fts5Storage;
 647
 648 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
 649 int sqlite3Fts5StorageClose(Fts5Storage *p);
 650 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 651
 652 int sqlite3Fts5DropAll(Fts5Config*);
 653 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 654
 655 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 656 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 657 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 658
 659 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
 660
 661 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
 662 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
 663
 664 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
 665 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
 666 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
 667
 668 int sqlite3Fts5StorageSync(Fts5Storage *p);
 669 int sqlite3Fts5StorageRollback(Fts5Storage *p);
 670
 671 int sqlite3Fts5StorageConfigValue(
 672     Fts5Storage *p, const char*, sqlite3_value*, int
 673 );
 674
 675 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
 676 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
 677 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 678 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 679 int sqlite3Fts5StorageReset(Fts5Storage *p);
 680
 681 /*
 682 ** End of interface to code in fts5_storage.c.
 683 **************************************************************************/
 684
 685
 686 /**************************************************************************
 687 ** Interface to code in fts5_expr.c.
 688 */
 689 typedef struct Fts5Expr Fts5Expr;
 690 typedef struct Fts5ExprNode Fts5ExprNode;
 691 typedef struct Fts5Parse Fts5Parse;
 692 typedef struct Fts5Token Fts5Token;
 693 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
 694 typedef struct Fts5ExprNearset Fts5ExprNearset;
 695
 696 struct Fts5Token {
 697   const char *p;                  /* Token text (not NULL terminated) */
 698   int n;                          /* Size of buffer p in bytes */
 699 };
 700
 701 /* Parse a MATCH expression. */
 702 int sqlite3Fts5ExprNew(
 703   Fts5Config *pConfig,
 704   int bPhraseToAnd,
 705   int iCol,                       /* Column on LHS of MATCH operator */
 706   const char *zExpr,
 707   Fts5Expr **ppNew,
 708   char **pzErr
 709 );
 710 int sqlite3Fts5ExprPattern(
 711   Fts5Config *pConfig,
 712   int bGlob,
 713   int iCol,
 714   const char *zText,
 715   Fts5Expr **pp
 716 );
 717
 718 /*
 719 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
 720 **     rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
 721 **     rc = sqlite3Fts5ExprNext(pExpr)
 722 ** ){
 723 **   // The document with rowid iRowid matches the expression!
 724 **   i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
 725 ** }
 726 */
 727 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
 728 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
 729 int sqlite3Fts5ExprEof(Fts5Expr*);
 730 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
 731
 732 void sqlite3Fts5ExprFree(Fts5Expr*);
 733 int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
 734
 735 /* Called during startup to register a UDF with SQLite */
 736 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
 737
 738 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 739 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 740 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 741
 742 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
 743 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
 744 int sqlite3Fts5ExprPopulatePoslists(
 745     Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
 746 );
 747 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
 748
 749 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 750
 751 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 752
 753 /*******************************************
 754 ** The fts5_expr.c API above this point is used by the other hand-written
 755 ** C code in this module. The interfaces below this point are called by
 756 ** the parser code in fts5parse.y.  */
 757
 758 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
 759
 760 Fts5ExprNode *sqlite3Fts5ParseNode(
 761   Fts5Parse *pParse,
 762   int eType,
 763   Fts5ExprNode *pLeft,
 764   Fts5ExprNode *pRight,
 765   Fts5ExprNearset *pNear
 766 );
 767
 768 Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
 769   Fts5Parse *pParse,
 770   Fts5ExprNode *pLeft,
 771   Fts5ExprNode *pRight
 772 );
 773
 774 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 775   Fts5Parse *pParse,
 776   Fts5ExprPhrase *pPhrase,
 777   Fts5Token *pToken,
 778   int bPrefix
 779 );
 780
 781 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
 782
 783 Fts5ExprNearset *sqlite3Fts5ParseNearset(
 784   Fts5Parse*,
 785   Fts5ExprNearset*,
 786   Fts5ExprPhrase*
 787 );
 788
 789 Fts5Colset *sqlite3Fts5ParseColset(
 790   Fts5Parse*,
 791   Fts5Colset*,
 792   Fts5Token *
 793 );
 794
 795 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
 796 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
 797 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
 798
 799 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
 800 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
 801 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
 802 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
 803 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
 804
 805 /*
 806 ** End of interface to code in fts5_expr.c.
 807 **************************************************************************/
 808
 809
 810
 811 /**************************************************************************
 812 ** Interface to code in fts5_aux.c.
 813 */
 814
 815 int sqlite3Fts5AuxInit(fts5_api*);
 816 /*
 817 ** End of interface to code in fts5_aux.c.
 818 **************************************************************************/
 819
 820 /**************************************************************************
 821 ** Interface to code in fts5_tokenizer.c.
 822 */
 823
 824 int sqlite3Fts5TokenizerInit(fts5_api*);
 825 int sqlite3Fts5TokenizerPattern(
 826     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
 827     Fts5Tokenizer *pTok
 828 );
 829 /*
 830 ** End of interface to code in fts5_tokenizer.c.
 831 **************************************************************************/
 832
 833 /**************************************************************************
 834 ** Interface to code in fts5_vocab.c.
 835 */
 836
 837 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
 838
 839 /*
 840 ** End of interface to code in fts5_vocab.c.
 841 **************************************************************************/
 842
 843
 844 /**************************************************************************
 845 ** Interface to automatically generated code in fts5_unicode2.c.
 846 */
 847 int sqlite3Fts5UnicodeIsdiacritic(int c);
 848 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
 849
 850 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
 851 int sqlite3Fts5UnicodeCategory(u32 iCode);
 852 void sqlite3Fts5UnicodeAscii(u8*, u8*);
 853 /*
 854 ** End of interface to code in fts5_unicode2.c.
 855 **************************************************************************/
 856
 857 #endif