third_party/sqlite/sqlite-src-3080704/ext/fts2/fts2_tokenizer1.c

   1 /*
   2 ** 2006 Oct 10
   3 **
   4 ** The author disclaims copyright to this source code.  In place of
   5 ** a legal notice, here is a blessing:
   6 **
   7 **    May you do good and not evil.
   8 **    May you find forgiveness for yourself and forgive others.
   9 **    May you share freely, never taking more than you give.
  10 **
  11 ******************************************************************************
  12 **
  13 ** Implementation of the "simple" full-text-search tokenizer.
  14 */
  15
  16 /*
  17 ** The code in this file is only compiled if:
  18 **
  19 **     * The FTS2 module is being built as an extension
  20 **       (in which case SQLITE_CORE is not defined), or
  21 **
  22 **     * The FTS2 module is being built into the core of
  23 **       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
  24 */
  25 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
  26
  27
  28 #include <assert.h>
  29 #include <stdlib.h>
  30 #include <stdio.h>
  31 #include <string.h>
  32
  33 #include "sqlite3.h"
  34 #include "sqlite3ext.h"
  35 SQLITE_EXTENSION_INIT3
  36 #include "fts2_tokenizer.h"
  37
  38 typedef struct simple_tokenizer {
  39   sqlite3_tokenizer base;
  40   char delim[128];             /* flag ASCII delimiters */
  41 } simple_tokenizer;
  42
  43 typedef struct simple_tokenizer_cursor {
  44   sqlite3_tokenizer_cursor base;
  45   const char *pInput;          /* input we are tokenizing */
  46   int nBytes;                  /* size of the input */
  47   int iOffset;                 /* current position in pInput */
  48   int iToken;                  /* index of next token to be returned */
  49   char *pToken;                /* storage for current token */
  50   int nTokenAllocated;         /* space allocated to zToken buffer */
  51 } simple_tokenizer_cursor;
  52
  53
  54 /* Forward declaration */
  55 static const sqlite3_tokenizer_module simpleTokenizerModule;
  56
  57 static int simpleDelim(simple_tokenizer *t, unsigned char c){
  58   return c<0x80 && t->delim[c];
  59 }
  60
  61 /*
  62 ** Create a new tokenizer instance.
  63 */
  64 static int simpleCreate(
  65   int argc, const char * const *argv,
  66   sqlite3_tokenizer **ppTokenizer
  67 ){
  68   simple_tokenizer *t;
  69
  70   t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
  71   if( t==NULL ) return SQLITE_NOMEM;
  72   memset(t, 0, sizeof(*t));
  73
  74   /* TODO(shess) Delimiters need to remain the same from run to run,
  75   ** else we need to reindex.  One solution would be a meta-table to
  76   ** track such information in the database, then we'd only want this
  77   ** information on the initial create.
  78   */
  79   if( argc>1 ){
  80     int i, n = strlen(argv[1]);
  81     for(i=0; i<n; i++){
  82       unsigned char ch = argv[1][i];
  83       /* We explicitly don't support UTF-8 delimiters for now. */
  84       if( ch>=0x80 ){
  85         sqlite3_free(t);
  86         return SQLITE_ERROR;
  87       }
  88       t->delim[ch] = 1;
  89     }
  90   } else {
  91     /* Mark non-alphanumeric ASCII characters as delimiters */
  92     int i;
  93     for(i=1; i<0x80; i++){
  94       t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
  95                       (i>='a' && i<='z'));
  96     }
  97   }
  98
  99   *ppTokenizer = &t->base;
 100   return SQLITE_OK;
 101 }
 102
 103 /*
 104 ** Destroy a tokenizer
 105 */
 106 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
 107   sqlite3_free(pTokenizer);
 108   return SQLITE_OK;
 109 }
 110
 111 /*
 112 ** Prepare to begin tokenizing a particular string.  The input
 113 ** string to be tokenized is pInput[0..nBytes-1].  A cursor
 114 ** used to incrementally tokenize this string is returned in
 115 ** *ppCursor.
 116 */
 117 static int simpleOpen(
 118   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
 119   const char *pInput, int nBytes,        /* String to be tokenized */
 120   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
 121 ){
 122   simple_tokenizer_cursor *c;
 123
 124   c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
 125   if( c==NULL ) return SQLITE_NOMEM;
 126
 127   c->pInput = pInput;
 128   if( pInput==0 ){
 129     c->nBytes = 0;
 130   }else if( nBytes<0 ){
 131     c->nBytes = (int)strlen(pInput);
 132   }else{
 133     c->nBytes = nBytes;
 134   }
 135   c->iOffset = 0;                 /* start tokenizing at the beginning */
 136   c->iToken = 0;
 137   c->pToken = NULL;               /* no space allocated, yet. */
 138   c->nTokenAllocated = 0;
 139
 140   *ppCursor = &c->base;
 141   return SQLITE_OK;
 142 }
 143
 144 /*
 145 ** Close a tokenization cursor previously opened by a call to
 146 ** simpleOpen() above.
 147 */
 148 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
 149   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 150   sqlite3_free(c->pToken);
 151   sqlite3_free(c);
 152   return SQLITE_OK;
 153 }
 154
 155 /*
 156 ** Extract the next token from a tokenization cursor.  The cursor must
 157 ** have been opened by a prior call to simpleOpen().
 158 */
 159 static int simpleNext(
 160   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
 161   const char **ppToken,               /* OUT: *ppToken is the token text */
 162   int *pnBytes,                       /* OUT: Number of bytes in token */
 163   int *piStartOffset,                 /* OUT: Starting offset of token */
 164   int *piEndOffset,                   /* OUT: Ending offset of token */
 165   int *piPosition                     /* OUT: Position integer of token */
 166 ){
 167   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 168   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
 169   unsigned char *p = (unsigned char *)c->pInput;
 170
 171   while( c->iOffset<c->nBytes ){
 172     int iStartOffset;
 173
 174     /* Scan past delimiter characters */
 175     while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
 176       c->iOffset++;
 177     }
 178
 179     /* Count non-delimiter characters. */
 180     iStartOffset = c->iOffset;
 181     while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
 182       c->iOffset++;
 183     }
 184
 185     if( c->iOffset>iStartOffset ){
 186       int i, n = c->iOffset-iStartOffset;
 187       if( n>c->nTokenAllocated ){
 188         c->nTokenAllocated = n+20;
 189         c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
 190         if( c->pToken==NULL ) return SQLITE_NOMEM;
 191       }
 192       for(i=0; i<n; i++){
 193         /* TODO(shess) This needs expansion to handle UTF-8
 194         ** case-insensitivity.
 195         */
 196         unsigned char ch = p[iStartOffset+i];
 197         c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
 198       }
 199       *ppToken = c->pToken;
 200       *pnBytes = n;
 201       *piStartOffset = iStartOffset;
 202       *piEndOffset = c->iOffset;
 203       *piPosition = c->iToken++;
 204
 205       return SQLITE_OK;
 206     }
 207   }
 208   return SQLITE_DONE;
 209 }
 210
 211 /*
 212 ** The set of routines that implement the simple tokenizer
 213 */
 214 static const sqlite3_tokenizer_module simpleTokenizerModule = {
 215   0,
 216   simpleCreate,
 217   simpleDestroy,
 218   simpleOpen,
 219   simpleClose,
 220   simpleNext,
 221 };
 222
 223 /*
 224 ** Allocate a new simple tokenizer.  Return a pointer to the new
 225 ** tokenizer in *ppModule
 226 */
 227 void sqlite3Fts2SimpleTokenizerModule(
 228   sqlite3_tokenizer_module const**ppModule
 229 ){
 230   *ppModule = &simpleTokenizerModule;
 231 }
 232
 233 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */