third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2_tokenizer.c

   1 /*
   2 ** 2007 June 22
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** This is part of an SQLite module implementing full-text search.
  14 ** This particular file implements the generic tokenizer interface.
  15 */
  16
  17 /*
  18 ** The code in this file is only compiled if:
  19 **
  20 **     * The FTS2 module is being built as an extension
  21 **       (in which case SQLITE_CORE is not defined), or
  22 **
  23 **     * The FTS2 module is being built into the core of
  24 **       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
  25 */
  26 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
  27
  28
  29 #include "sqlite3.h"
  30 #include "sqlite3ext.h"
  31 SQLITE_EXTENSION_INIT3
  32
  33 #include "fts2_hash.h"
  34 #include "fts2_tokenizer.h"
  35 #include <assert.h>
  36
  37 /*
  38 ** Implementation of the SQL scalar function for accessing the underlying
  39 ** hash table. This function may be called as follows:
  40 **
  41 **   SELECT <function-name>(<key-name>);
  42 **   SELECT <function-name>(<key-name>, <pointer>);
  43 **
  44 ** where <function-name> is the name passed as the second argument
  45 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
  46 **
  47 ** If the <pointer> argument is specified, it must be a blob value
  48 ** containing a pointer to be stored as the hash data corresponding
  49 ** to the string <key-name>. If <pointer> is not specified, then
  50 ** the string <key-name> must already exist in the has table. Otherwise,
  51 ** an error is returned.
  52 **
  53 ** Whether or not the <pointer> argument is specified, the value returned
  54 ** is a blob containing the pointer stored as the hash data corresponding
  55 ** to string <key-name> (after the hash-table is updated, if applicable).
  56 */
  57 static void scalarFunc(
  58   sqlite3_context *context,
  59   int argc,
  60   sqlite3_value **argv
  61 ){
  62   fts2Hash *pHash;
  63   void *pPtr = 0;
  64   const unsigned char *zName;
  65   int nName;
  66
  67   assert( argc==1 || argc==2 );
  68
  69   pHash = (fts2Hash *)sqlite3_user_data(context);
  70
  71   zName = sqlite3_value_text(argv[0]);
  72   nName = sqlite3_value_bytes(argv[0])+1;
  73
  74   if( argc==2 ){
  75     void *pOld;
  76     int n = sqlite3_value_bytes(argv[1]);
  77     if( n!=sizeof(pPtr) ){
  78       sqlite3_result_error(context, "argument type mismatch", -1);
  79       return;
  80     }
  81     pPtr = *(void **)sqlite3_value_blob(argv[1]);
  82     pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
  83     if( pOld==pPtr ){
  84       sqlite3_result_error(context, "out of memory", -1);
  85       return;
  86     }
  87   }else{
  88     pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
  89     if( !pPtr ){
  90       char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
  91       sqlite3_result_error(context, zErr, -1);
  92       sqlite3_free(zErr);
  93       return;
  94     }
  95   }
  96
  97   sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
  98 }
  99
 100 #ifdef SQLITE_TEST
 101
 102 #include <tcl.h>
 103 #include <string.h>
 104
 105 /*
 106 ** Implementation of a special SQL scalar function for testing tokenizers
 107 ** designed to be used in concert with the Tcl testing framework. This
 108 ** function must be called with two arguments:
 109 **
 110 **   SELECT <function-name>(<key-name>, <input-string>);
 111 **   SELECT <function-name>(<key-name>, <pointer>);
 112 **
 113 ** where <function-name> is the name passed as the second argument
 114 ** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
 115 ** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
 116 **
 117 ** The return value is a string that may be interpreted as a Tcl
 118 ** list. For each token in the <input-string>, three elements are
 119 ** added to the returned list. The first is the token position, the
 120 ** second is the token text (folded, stemmed, etc.) and the third is the
 121 ** substring of <input-string> associated with the token. For example,
 122 ** using the built-in "simple" tokenizer:
 123 **
 124 **   SELECT fts_tokenizer_test('simple', 'I don't see how');
 125 **
 126 ** will return the string:
 127 **
 128 **   "{0 i I 1 dont don't 2 see see 3 how how}"
 129 **
 130 */
 131 static void testFunc(
 132   sqlite3_context *context,
 133   int argc,
 134   sqlite3_value **argv
 135 ){
 136   fts2Hash *pHash;
 137   sqlite3_tokenizer_module *p;
 138   sqlite3_tokenizer *pTokenizer = 0;
 139   sqlite3_tokenizer_cursor *pCsr = 0;
 140
 141   const char *zErr = 0;
 142
 143   const char *zName;
 144   int nName;
 145   const char *zInput;
 146   int nInput;
 147
 148   const char *zArg = 0;
 149
 150   const char *zToken;
 151   int nToken;
 152   int iStart;
 153   int iEnd;
 154   int iPos;
 155
 156   Tcl_Obj *pRet;
 157
 158   assert( argc==2 || argc==3 );
 159
 160   nName = sqlite3_value_bytes(argv[0]);
 161   zName = (const char *)sqlite3_value_text(argv[0]);
 162   nInput = sqlite3_value_bytes(argv[argc-1]);
 163   zInput = (const char *)sqlite3_value_text(argv[argc-1]);
 164
 165   if( argc==3 ){
 166     zArg = (const char *)sqlite3_value_text(argv[1]);
 167   }
 168
 169   pHash = (fts2Hash *)sqlite3_user_data(context);
 170   p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
 171
 172   if( !p ){
 173     char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
 174     sqlite3_result_error(context, zErr, -1);
 175     sqlite3_free(zErr);
 176     return;
 177   }
 178
 179   pRet = Tcl_NewObj();
 180   Tcl_IncrRefCount(pRet);
 181
 182   if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
 183     zErr = "error in xCreate()";
 184     goto finish;
 185   }
 186   pTokenizer->pModule = p;
 187   if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
 188     zErr = "error in xOpen()";
 189     goto finish;
 190   }
 191   pCsr->pTokenizer = pTokenizer;
 192
 193   while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
 194     Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
 195     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 196     zToken = &zInput[iStart];
 197     nToken = iEnd-iStart;
 198     Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
 199   }
 200
 201   if( SQLITE_OK!=p->xClose(pCsr) ){
 202     zErr = "error in xClose()";
 203     goto finish;
 204   }
 205   if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
 206     zErr = "error in xDestroy()";
 207     goto finish;
 208   }
 209
 210 finish:
 211   if( zErr ){
 212     sqlite3_result_error(context, zErr, -1);
 213   }else{
 214     sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
 215   }
 216   Tcl_DecrRefCount(pRet);
 217 }
 218
 219 static
 220 int registerTokenizer(
 221   sqlite3 *db,
 222   char *zName,
 223   const sqlite3_tokenizer_module *p
 224 ){
 225   int rc;
 226   sqlite3_stmt *pStmt;
 227   const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
 228
 229   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 230   if( rc!=SQLITE_OK ){
 231     return rc;
 232   }
 233
 234   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 235   sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
 236   sqlite3_step(pStmt);
 237
 238   return sqlite3_finalize(pStmt);
 239 }
 240
 241 static
 242 int queryFts2Tokenizer(
 243   sqlite3 *db,
 244   char *zName,
 245   const sqlite3_tokenizer_module **pp
 246 ){
 247   int rc;
 248   sqlite3_stmt *pStmt;
 249   const char zSql[] = "SELECT fts2_tokenizer(?)";
 250
 251   *pp = 0;
 252   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
 253   if( rc!=SQLITE_OK ){
 254     return rc;
 255   }
 256
 257   sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
 258   if( SQLITE_ROW==sqlite3_step(pStmt) ){
 259     if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
 260       memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
 261     }
 262   }
 263
 264   return sqlite3_finalize(pStmt);
 265 }
 266
 267 void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 268
 269 /*
 270 ** Implementation of the scalar function fts2_tokenizer_internal_test().
 271 ** This function is used for testing only, it is not included in the
 272 ** build unless SQLITE_TEST is defined.
 273 **
 274 ** The purpose of this is to test that the fts2_tokenizer() function
 275 ** can be used as designed by the C-code in the queryFts2Tokenizer and
 276 ** registerTokenizer() functions above. These two functions are repeated
 277 ** in the README.tokenizer file as an example, so it is important to
 278 ** test them.
 279 **
 280 ** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
 281 ** function with no arguments. An assert() will fail if a problem is
 282 ** detected. i.e.:
 283 **
 284 **     SELECT fts2_tokenizer_internal_test();
 285 **
 286 */
 287 static void intTestFunc(
 288   sqlite3_context *context,
 289   int argc,
 290   sqlite3_value **argv
 291 ){
 292   int rc;
 293   const sqlite3_tokenizer_module *p1;
 294   const sqlite3_tokenizer_module *p2;
 295   sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
 296
 297   /* Test the query function */
 298   sqlite3Fts2SimpleTokenizerModule(&p1);
 299   rc = queryFts2Tokenizer(db, "simple", &p2);
 300   assert( rc==SQLITE_OK );
 301   assert( p1==p2 );
 302   rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
 303   assert( rc==SQLITE_ERROR );
 304   assert( p2==0 );
 305   assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
 306
 307   /* Test the storage function */
 308   rc = registerTokenizer(db, "nosuchtokenizer", p1);
 309   assert( rc==SQLITE_OK );
 310   rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
 311   assert( rc==SQLITE_OK );
 312   assert( p2==p1 );
 313
 314   sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
 315 }
 316
 317 #endif
 318
 319 /*
 320 ** Set up SQL objects in database db used to access the contents of
 321 ** the hash table pointed to by argument pHash. The hash table must
 322 ** been initialized to use string keys, and to take a private copy
 323 ** of the key when a value is inserted. i.e. by a call similar to:
 324 **
 325 **    sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
 326 **
 327 ** This function adds a scalar function (see header comment above
 328 ** scalarFunc() in this file for details) and, if ENABLE_TABLE is
 329 ** defined at compilation time, a temporary virtual table (see header
 330 ** comment above struct HashTableVtab) to the database schema. Both
 331 ** provide read/write access to the contents of *pHash.
 332 **
 333 ** The third argument to this function, zName, is used as the name
 334 ** of both the scalar and, if created, the virtual table.
 335 */
 336 int sqlite3Fts2InitHashTable(
 337   sqlite3 *db,
 338   fts2Hash *pHash,
 339   const char *zName
 340 ){
 341   int rc = SQLITE_OK;
 342   void *p = (void *)pHash;
 343   const int any = SQLITE_ANY;
 344   char *zTest = 0;
 345   char *zTest2 = 0;
 346
 347 #ifdef SQLITE_TEST
 348   void *pdb = (void *)db;
 349   zTest = sqlite3_mprintf("%s_test", zName);
 350   zTest2 = sqlite3_mprintf("%s_internal_test", zName);
 351   if( !zTest || !zTest2 ){
 352     rc = SQLITE_NOMEM;
 353   }
 354 #endif
 355
 356   if( rc!=SQLITE_OK
 357    || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
 358    || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
 359 #ifdef SQLITE_TEST
 360    || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
 361    || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
 362    || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
 363 #endif
 364   );
 365
 366   sqlite3_free(zTest);
 367   sqlite3_free(zTest2);
 368   return rc;
 369 }
 370
 371 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */