third_party/sqlite/src/ext/fts3/fts3_tokenizer.c

   1 /*
   2 ** 2007 June 22
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This is part of an SQLite module implementing full-text search.
  14 ** This particular file implements the generic tokenizer interface.
  15 */
  16
  17 /*
  18 ** The code in this file is only compiled if:
  19 **
  20 **     * The FTS3 module is being built as an extension
  21 **       (in which case SQLITE_CORE is not defined), or
  22 **
  23 **     * The FTS3 module is being built into the core of
  24 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
  25 */
  26 #include "fts3Int.h"
  27 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
  28
  29 #include <assert.h>
  30 #include <string.h>
  31
  32 /*
  33 ** Implementation of the SQL scalar function for accessing the underlying
  34 ** hash table. This function may be called as follows:
  35 **
  36 **   SELECT <function-name>(<key-name>);
  37 **   SELECT <function-name>(<key-name>, <pointer>);
  38 **
  39 ** where <function-name> is the name passed as the second argument
  40 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer').
  41 **
  42 ** If the <pointer> argument is specified, it must be a blob value
  43 ** containing a pointer to be stored as the hash data corresponding
  44 ** to the string <key-name>. If <pointer> is not specified, then
  45 ** the string <key-name> must already exist in the has table. Otherwise,
  46 ** an error is returned.
  47 **
  48 ** Whether or not the <pointer> argument is specified, the value returned
  49 ** is a blob containing the pointer stored as the hash data corresponding
  50 ** to string <key-name> (after the hash-table is updated, if applicable).
  51 */
  52 static void scalarFunc(
  53   sqlite3_context *context,
  54   int argc,
  55   sqlite3_value **argv
  56 ){
  57   Fts3Hash *pHash;
  58   void *pPtr = 0;
  59   const unsigned char *zName;
  60   int nName;
  61
  62   assert( argc==1 || argc==2 );
  63
  64   pHash = (Fts3Hash *)sqlite3_user_data(context);
  65
  66   zName = sqlite3_value_text(argv[0]);
  67   nName = sqlite3_value_bytes(argv[0])+1;
  68
  69   if( argc==2 ){
  70     void *pOld;
  71     int n = sqlite3_value_bytes(argv[1]);
  72     if( n!=sizeof(pPtr) ){
  73       sqlite3_result_error(context, "argument type mismatch", -1);
  74       return;
  75     }
  76     pPtr = *(void **)sqlite3_value_blob(argv[1]);
  77     pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr);
  78     if( pOld==pPtr ){
  79       sqlite3_result_error(context, "out of memory", -1);
  80       return;
  81     }
  82   }else{
  83     pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
  84     if( !pPtr ){
  85       char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
  86       sqlite3_result_error(context, zErr, -1);
  87       sqlite3_free(zErr);
  88       return;
  89     }
  90   }
  91
  92   sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
  93 }
  94
  95 int sqlite3Fts3IsIdChar(char c){
  96   static const char isFtsIdChar[] = {
  97       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
  98       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
  99       0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
 100       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
 101       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
 102       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
 103       0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
 104       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
 105   };
 106   return (c&0x80 || isFtsIdChar[(int)(c)]);
 107 }
 108
 109 const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
 110   const char *z1;
 111   const char *z2 = 0;
 112
 113   /* Find the start of the next token. */
 114   z1 = zStr;
 115   while( z2==0 ){
 116     char c = *z1;
 117     switch( c ){
 118       case '\0': return 0;        /* No more tokens here */
 119       case '\'':
 120       case '"':
 121       case '`': {
 122         z2 = z1;
 123         while( *++z2 && (*z2!=c || *++z2==c) );
 124         break;
 125       }
 126       case '[':
 127         z2 = &z1[1];
 128         while( *z2 && z2[0]!=']' ) z2++;
 129         if( *z2 ) z2++;
 130         break;
 131
 132       default:
 133         if( sqlite3Fts3IsIdChar(*z1) ){
 134           z2 = &z1[1];
 135           while( sqlite3Fts3IsIdChar(*z2) ) z2++;
 136         }else{
 137           z1++;
 138         }
 139     }
 140   }
 141
 142   *pn = (int)(z2-z1);
 143   return z1;
 144 }
 145
 146 int sqlite3Fts3InitTokenizer(
 147   Fts3Hash *pHash,                /* Tokenizer hash table */
 148   const char *zArg,               /* Tokenizer name */
 149   sqlite3_tokenizer **ppTok,      /* OUT: Tokenizer (if applicable) */
 150   char **pzErr                    /* OUT: Set to malloced error message */
 151 ){
 152   int rc;
 153   char *z = (char *)zArg;
 154   int n = 0;
 155   char *zCopy;
 156   char *zEnd;                     /* Pointer to nul-term of zCopy */
 157   sqlite3_tokenizer_module *m;
 158
 159   zCopy = sqlite3_mprintf("%s", zArg);
 160   if( !zCopy ) return SQLITE_NOMEM;
 161   zEnd = &zCopy[strlen(zCopy)];
 162
 163   z = (char *)sqlite3Fts3NextToken(zCopy, &n);
 164   z[n] = '\0';
 165   sqlite3Fts3Dequote(z);
 166
 167   m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
 168   if( !m ){
 169     *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
 170     rc = SQLITE_ERROR;
 171   }else{
 172     char const **aArg = 0;
 173     int iArg = 0;
 174     z = &z[n+1];
 175     while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
 176       int nNew = sizeof(char *)*(iArg+1);
 177       char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
 178       if( !aNew ){
 179         sqlite3_free(zCopy);
 180         sqlite3_free((void *)aArg);
 181         return SQLITE_NOMEM;
 182       }
 183       aArg = aNew;
 184       aArg[iArg++] = z;
 185       z[n] = '\0';
 186       sqlite3Fts3Dequote(z);
 187       z = &z[n+1];
 188     }
 189     rc = m->xCreate(iArg, aArg, ppTok);
 190     assert( rc!=SQLITE_OK || *ppTok );
 191     if( rc!=SQLITE_OK ){
 192       *pzErr = sqlite3_mprintf("unknown tokenizer");
 193     }else{
 194       (*ppTok)->pModule = m;
 195     }
 196     sqlite3_free((void *)aArg);
 197   }
 198
 199   sqlite3_free(zCopy);
 200   return rc;
 201 }
 202
 203
 204 #ifdef SQLITE_TEST
 205
 206 #include <tcl.h>
 207 #include <string.h>
 208
 209 /*
 210 ** Implementation of a special SQL scalar function for testing tokenizers
 211 ** designed to be used in concert with the Tcl testing framework. This
 212 ** function must be called with two or more arguments:
 213 **
 214 **   SELECT <function-name>(<key-name>, ..., <input-string>);
 215 **
 216 ** where <function-name> is the name passed as the second argument
 217 ** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer')
 218 ** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test').
 219 **
 220 ** The return value is a string that may be interpreted as a Tcl
 221 ** list. For each token in the <input-string>, three elements are
 222 ** added to the returned list. The first is the token position, the
 223 ** second is the token text (folded, stemmed, etc.) and the third is the
 224 ** substring of <input-string> associated with the token. For example,
 225 ** using the built-in "simple" tokenizer:
 226 **
 227 **   SELECT fts_tokenizer_test('simple', 'I don't see how');
 228 **
 229 ** will return the string:
 230 **
 231 **   "{0 i I 1 dont don't 2 see see 3 how how}"
 232 **
 233 */
 234 static void testFunc(
 235   sqlite3_context *context,
 236   int argc,
 237   sqlite3_value **argv
 238 ){
 239   Fts3Hash *pHash;
 240   sqlite3_tokenizer_module *p;
 241   sqlite3_tokenizer *pTokenizer = 0;
 242   sqlite3_tokenizer_cursor *pCsr = 0;
 243
 244   const char *zErr = 0;
 245
 246   const char *zName;
 247   int nName;
 248   const char *zInput;
 249   int nInput;
 250
 251   const char *azArg[64];
 252
 253   const char *zToken;
 254   int nToken = 0;
 255   int iStart = 0;
 256   int iEnd = 0;
 257   int iPos = 0;
 258   int i;
 259
 260   Tcl_Obj *pRet;
 261
 262   if( argc<2 ){
 263     sqlite3_result_error(context, "insufficient arguments", -1);
 264     return;
 265   }
 266
 267   nName = sqlite3_value_bytes(argv[0]);
 268   zName = (const char *)sqlite3_value_text(argv[0]);
 269   nInput = sqlite3_value_bytes(argv[argc-1]);
 270   zInput = (const char *)sqlite3_value_text(argv[argc-1]);
 271
 272   pHash = (Fts3Hash *)sqlite3_user_data(context);
 273   p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
 274
 275   if( !p ){
 276     char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
 277     sqlite3_result_error(context, zErr, -1);
 278     sqlite3_free(zErr);
 279     return;
 280   }
 281
 282   pRet = Tcl_NewObj();
 283   Tcl_IncrRefCount(pRet);
 284
 285   for(i=1; i<argc-1; i++){
 286     azArg[i-1] = (const char *)sqlite3_value_text(argv[i]);
 287   }
 288
 289   if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){
 290     zErr = "error in xCreate()";
 291     goto finish;
 292   }
 293   pTokenizer->pModule = p;
 294   if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){
 295     zErr = "error in xOpen()";
 296     goto finish;
 297   }
 298
 299   while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
 300     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
 301     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 302     zToken = &zInput[iStart];
 303     nToken = iEnd-iStart;
 304     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 305   }
 306
 307   if( SQLITE_OK!=p->xClose(pCsr) ){
 308     zErr = "error in xClose()";
 309     goto finish;
 310   }
 311   if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
 312     zErr = "error in xDestroy()";
 313     goto finish;
 314   }
 315
 316 finish:
 317   if( zErr ){
 318     sqlite3_result_error(context, zErr, -1);
 319   }else{
 320     sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
 321   }
 322   Tcl_DecrRefCount(pRet);
 323 }
 324
 325 static
 326 int registerTokenizer(
 327   sqlite3 *db,
 328   char *zName,
 329   const sqlite3_tokenizer_module *p
 330 ){
 331   int rc;
 332   sqlite3_stmt *pStmt;
 333   const char zSql[] = "SELECT fts3_tokenizer(?, ?)";
 334
 335   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 336   if( rc!=SQLITE_OK ){
 337     return rc;
 338   }
 339
 340   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 341   sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
 342   sqlite3_step(pStmt);
 343
 344   return sqlite3_finalize(pStmt);
 345 }
 346
 347 static
 348 int queryTokenizer(
 349   sqlite3 *db,
 350   char *zName,
 351   const sqlite3_tokenizer_module **pp
 352 ){
 353   int rc;
 354   sqlite3_stmt *pStmt;
 355   const char zSql[] = "SELECT fts3_tokenizer(?)";
 356
 357   *pp = 0;
 358   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 359   if( rc!=SQLITE_OK ){
 360     return rc;
 361   }
 362
 363   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 364   if( SQLITE_ROW==sqlite3_step(pStmt) ){
 365     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
 366       memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
 367     }
 368   }
 369
 370   return sqlite3_finalize(pStmt);
 371 }
 372
 373 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 374
 375 /*
 376 ** Implementation of the scalar function fts3_tokenizer_internal_test().
 377 ** This function is used for testing only, it is not included in the
 378 ** build unless SQLITE_TEST is defined.
 379 **
 380 ** The purpose of this is to test that the fts3_tokenizer() function
 381 ** can be used as designed by the C-code in the queryTokenizer and
 382 ** registerTokenizer() functions above. These two functions are repeated
 383 ** in the README.tokenizer file as an example, so it is important to
 384 ** test them.
 385 **
 386 ** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar
 387 ** function with no arguments. An assert() will fail if a problem is
 388 ** detected. i.e.:
 389 **
 390 **     SELECT fts3_tokenizer_internal_test();
 391 **
 392 */
 393 static void intTestFunc(
 394   sqlite3_context *context,
 395   int argc,
 396   sqlite3_value **argv
 397 ){
 398   int rc;
 399   const sqlite3_tokenizer_module *p1;
 400   const sqlite3_tokenizer_module *p2;
 401   sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
 402
 403   UNUSED_PARAMETER(argc);
 404   UNUSED_PARAMETER(argv);
 405
 406   /* Test the query function */
 407   sqlite3Fts3SimpleTokenizerModule(&p1);
 408   rc = queryTokenizer(db, "simple", &p2);
 409   assert( rc==SQLITE_OK );
 410   assert( p1==p2 );
 411   rc = queryTokenizer(db, "nosuchtokenizer", &p2);
 412   assert( rc==SQLITE_ERROR );
 413   assert( p2==0 );
 414   assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
 415
 416   /* Test the storage function */
 417   rc = registerTokenizer(db, "nosuchtokenizer", p1);
 418   assert( rc==SQLITE_OK );
 419   rc = queryTokenizer(db, "nosuchtokenizer", &p2);
 420   assert( rc==SQLITE_OK );
 421   assert( p2==p1 );
 422
 423   sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 424 }
 425
 426 #endif
 427
 428 /*
 429 ** Set up SQL objects in database db used to access the contents of
 430 ** the hash table pointed to by argument pHash. The hash table must
 431 ** been initialized to use string keys, and to take a private copy
 432 ** of the key when a value is inserted. i.e. by a call similar to:
 433 **
 434 **    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
 435 **
 436 ** This function adds a scalar function (see header comment above
 437 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is
 438 ** defined at compilation time, a temporary virtual table (see header
 439 ** comment above struct HashTableVtab) to the database schema. Both
 440 ** provide read/write access to the contents of *pHash.
 441 **
 442 ** The third argument to this function, zName, is used as the name
 443 ** of both the scalar and, if created, the virtual table.
 444 */
 445 int sqlite3Fts3InitHashTable(
 446   sqlite3 *db,
 447   Fts3Hash *pHash,
 448   const char *zName
 449 ){
 450   int rc = SQLITE_OK;
 451   void *p = (void *)pHash;
 452   const int any = SQLITE_ANY;
 453
 454 #ifdef SQLITE_TEST
 455   char *zTest = 0;
 456   char *zTest2 = 0;
 457   void *pdb = (void *)db;
 458   zTest = sqlite3_mprintf("%s_test", zName);
 459   zTest2 = sqlite3_mprintf("%s_internal_test", zName);
 460   if( !zTest || !zTest2 ){
 461     rc = SQLITE_NOMEM;
 462   }
 463 #endif
 464
 465   if( SQLITE_OK==rc ){
 466     rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0);
 467   }
 468   if( SQLITE_OK==rc ){
 469     rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0);
 470   }
 471 #ifdef SQLITE_TEST
 472   if( SQLITE_OK==rc ){
 473     rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0);
 474   }
 475   if( SQLITE_OK==rc ){
 476     rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0);
 477   }
 478 #endif
 479
 480 #ifdef SQLITE_TEST
 481   sqlite3_free(zTest);
 482   sqlite3_free(zTest2);
 483 #endif
 484
 485   return rc;
 486 }
 487
 488 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */