ext/fts1/fts1_tokenizer1.c

   1 /*
   2 ** The author disclaims copyright to this source code.
   3 **
   4 *************************************************************************
   5 ** Implementation of the "simple" full-text-search tokenizer.
   6 */
   7
   8 /*
   9 ** The code in this file is only compiled if:
  10 **
  11 **     * The FTS1 module is being built as an extension
  12 **       (in which case SQLITE_CORE is not defined), or
  13 **
  14 **     * The FTS1 module is being built into the core of
  15 **       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
  16 */
  17 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
  18
  19
  20 #include <assert.h>
  21 #include <stdlib.h>
  22 #include <stdio.h>
  23 #include <string.h>
  24 #include <ctype.h>
  25
  26 #include "fts1_tokenizer.h"
  27
  28 typedef struct simple_tokenizer {
  29   sqlite3_tokenizer base;
  30   char delim[128];             /* flag ASCII delimiters */
  31 } simple_tokenizer;
  32
  33 typedef struct simple_tokenizer_cursor {
  34   sqlite3_tokenizer_cursor base;
  35   const char *pInput;          /* input we are tokenizing */
  36   int nBytes;                  /* size of the input */
  37   int iOffset;                 /* current position in pInput */
  38   int iToken;                  /* index of next token to be returned */
  39   char *pToken;                /* storage for current token */
  40   int nTokenAllocated;         /* space allocated to zToken buffer */
  41 } simple_tokenizer_cursor;
  42
  43
  44 /* Forward declaration */
  45 static const sqlite3_tokenizer_module simpleTokenizerModule;
  46
  47 static int isDelim(simple_tokenizer *t, unsigned char c){
  48   return c<0x80 && t->delim[c];
  49 }
  50
  51 /*
  52 ** Create a new tokenizer instance.
  53 */
  54 static int simpleCreate(
  55   int argc, const char * const *argv,
  56   sqlite3_tokenizer **ppTokenizer
  57 ){
  58   simple_tokenizer *t;
  59
  60   t = (simple_tokenizer *) calloc(sizeof(*t), 1);
  61   if( t==NULL ) return SQLITE_NOMEM;
  62
  63   /* TODO(shess) Delimiters need to remain the same from run to run,
  64   ** else we need to reindex.  One solution would be a meta-table to
  65   ** track such information in the database, then we'd only want this
  66   ** information on the initial create.
  67   */
  68   if( argc>1 ){
  69     int i, n = strlen(argv[1]);
  70     for(i=0; i<n; i++){
  71       unsigned char ch = argv[1][i];
  72       /* We explicitly don't support UTF-8 delimiters for now. */
  73       if( ch>=0x80 ){
  74         free(t);
  75         return SQLITE_ERROR;
  76       }
  77       t->delim[ch] = 1;
  78     }
  79   } else {
  80     /* Mark non-alphanumeric ASCII characters as delimiters */
  81     int i;
  82     for(i=1; i<0x80; i++){
  83       t->delim[i] = !isalnum(i);
  84     }
  85   }
  86
  87   *ppTokenizer = &t->base;
  88   return SQLITE_OK;
  89 }
  90
  91 /*
  92 ** Destroy a tokenizer
  93 */
  94 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
  95   free(pTokenizer);
  96   return SQLITE_OK;
  97 }
  98
  99 /*
 100 ** Prepare to begin tokenizing a particular string.  The input
 101 ** string to be tokenized is pInput[0..nBytes-1].  A cursor
 102 ** used to incrementally tokenize this string is returned in
 103 ** *ppCursor.
 104 */
 105 static int simpleOpen(
 106   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
 107   const char *pInput, int nBytes,        /* String to be tokenized */
 108   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
 109 ){
 110   simple_tokenizer_cursor *c;
 111
 112   c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
 113   if( c==NULL ) return SQLITE_NOMEM;
 114
 115   c->pInput = pInput;
 116   if( pInput==0 ){
 117     c->nBytes = 0;
 118   }else if( nBytes<0 ){
 119     c->nBytes = (int)strlen(pInput);
 120   }else{
 121     c->nBytes = nBytes;
 122   }
 123   c->iOffset = 0;                 /* start tokenizing at the beginning */
 124   c->iToken = 0;
 125   c->pToken = NULL;               /* no space allocated, yet. */
 126   c->nTokenAllocated = 0;
 127
 128   *ppCursor = &c->base;
 129   return SQLITE_OK;
 130 }
 131
 132 /*
 133 ** Close a tokenization cursor previously opened by a call to
 134 ** simpleOpen() above.
 135 */
 136 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
 137   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 138   free(c->pToken);
 139   free(c);
 140   return SQLITE_OK;
 141 }
 142
 143 /*
 144 ** Extract the next token from a tokenization cursor.  The cursor must
 145 ** have been opened by a prior call to simpleOpen().
 146 */
 147 static int simpleNext(
 148   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
 149   const char **ppToken,               /* OUT: *ppToken is the token text */
 150   int *pnBytes,                       /* OUT: Number of bytes in token */
 151   int *piStartOffset,                 /* OUT: Starting offset of token */
 152   int *piEndOffset,                   /* OUT: Ending offset of token */
 153   int *piPosition                     /* OUT: Position integer of token */
 154 ){
 155   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
 156   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
 157   unsigned char *p = (unsigned char *)c->pInput;
 158
 159   while( c->iOffset<c->nBytes ){
 160     int iStartOffset;
 161
 162     /* Scan past delimiter characters */
 163     while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
 164       c->iOffset++;
 165     }
 166
 167     /* Count non-delimiter characters. */
 168     iStartOffset = c->iOffset;
 169     while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
 170       c->iOffset++;
 171     }
 172
 173     if( c->iOffset>iStartOffset ){
 174       int i, n = c->iOffset-iStartOffset;
 175       if( n>c->nTokenAllocated ){
 176         c->nTokenAllocated = n+20;
 177         c->pToken = realloc(c->pToken, c->nTokenAllocated);
 178         if( c->pToken==NULL ) return SQLITE_NOMEM;
 179       }
 180       for(i=0; i<n; i++){
 181         /* TODO(shess) This needs expansion to handle UTF-8
 182         ** case-insensitivity.
 183         */
 184         unsigned char ch = p[iStartOffset+i];
 185         c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
 186       }
 187       *ppToken = c->pToken;
 188       *pnBytes = n;
 189       *piStartOffset = iStartOffset;
 190       *piEndOffset = c->iOffset;
 191       *piPosition = c->iToken++;
 192
 193       return SQLITE_OK;
 194     }
 195   }
 196   return SQLITE_DONE;
 197 }
 198
 199 /*
 200 ** The set of routines that implement the simple tokenizer
 201 */
 202 static const sqlite3_tokenizer_module simpleTokenizerModule = {
 203   0,
 204   simpleCreate,
 205   simpleDestroy,
 206   simpleOpen,
 207   simpleClose,
 208   simpleNext,
 209 };
 210
 211 /*
 212 ** Allocate a new simple tokenizer.  Return a pointer to the new
 213 ** tokenizer in *ppModule
 214 */
 215 void sqlite3Fts1SimpleTokenizerModule(
 216   sqlite3_tokenizer_module const**ppModule
 217 ){
 218   *ppModule = &simpleTokenizerModule;
 219 }
 220
 221 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */