1 /*
2 ** 2006 Oct 10
3 **
4 ** The author disclaims copyright to this source code.  In place of
5 ** a legal notice, here is a blessing:
6 **
7 **    May you do good and not evil.
8 **    May you find forgiveness for yourself and forgive others.
9 **    May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** Implementation of the "simple" full-text-search tokenizer.
14 */
15 
16 /*
17 ** The code in this file is only compiled if:
18 **
19 **     * The FTS3 module is being built as an extension
20 **       (in which case SQLITE_CORE is not defined), or
21 **
22 **     * The FTS3 module is being built into the core of
23 **       SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
24 */
25 #include "fts3Int.h"
26 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
27 
28 #include <assert.h>
29 #include <stdlib.h>
30 #include <stdio.h>
31 #include <string.h>
32 
33 #include "fts3_tokenizer.h"
34 
35 typedef struct simple_tokenizer {
36   sqlite3_tokenizer base;
37   char delim[128];             /* flag ASCII delimiters */
38 } simple_tokenizer;
39 
40 typedef struct simple_tokenizer_cursor {
41   sqlite3_tokenizer_cursor base;
42   const char *pInput;          /* input we are tokenizing */
43   int nBytes;                  /* size of the input */
44   int iOffset;                 /* current position in pInput */
45   int iToken;                  /* index of next token to be returned */
46   char *pToken;                /* storage for current token */
47   int nTokenAllocated;         /* space allocated to zToken buffer */
48 } simple_tokenizer_cursor;
49 
50 
51 static int simpleDelim(simple_tokenizer *t, unsigned char c){
52   return c<0x80 && t->delim[c];
53 }
54 static int fts3_isalnum(int x){
55   return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
56 }
57 
58 /*
59 ** Create a new tokenizer instance.
60 */
61 static int simpleCreate(
62   int argc, const char * const *argv,
63   sqlite3_tokenizer **ppTokenizer
64 ){
65   simple_tokenizer *t;
66 
67   t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
68   if( t==NULL ) return SQLITE_NOMEM;
69   memset(t, 0, sizeof(*t));
70 
71   /* TODO(shess) Delimiters need to remain the same from run to run,
72   ** else we need to reindex.  One solution would be a meta-table to
73   ** track such information in the database, then we'd only want this
74   ** information on the initial create.
75   */
76   if( argc>1 ){
77     int i, n = (int)strlen(argv[1]);
78     for(i=0; i<n; i++){
79       unsigned char ch = argv[1][i];
80       /* We explicitly don't support UTF-8 delimiters for now. */
81       if( ch>=0x80 ){
82         sqlite3_free(t);
83         return SQLITE_ERROR;
84       }
85       t->delim[ch] = 1;
86     }
87   } else {
88     /* Mark non-alphanumeric ASCII characters as delimiters */
89     int i;
90     for(i=1; i<0x80; i++){
91       t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
92     }
93   }
94 
95   *ppTokenizer = &t->base;
96   return SQLITE_OK;
97 }
98 
99 /*
100 ** Destroy a tokenizer
101 */
102 static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
103   sqlite3_free(pTokenizer);
104   return SQLITE_OK;
105 }
106 
107 /*
108 ** Prepare to begin tokenizing a particular string.  The input
109 ** string to be tokenized is pInput[0..nBytes-1].  A cursor
110 ** used to incrementally tokenize this string is returned in
111 ** *ppCursor.
112 */
113 static int simpleOpen(
114   sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
115   const char *pInput, int nBytes,        /* String to be tokenized */
116   sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
117 ){
118   simple_tokenizer_cursor *c;
119 
120   UNUSED_PARAMETER(pTokenizer);
121 
122   c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
123   if( c==NULL ) return SQLITE_NOMEM;
124 
125   c->pInput = pInput;
126   if( pInput==0 ){
127     c->nBytes = 0;
128   }else if( nBytes<0 ){
129     c->nBytes = (int)strlen(pInput);
130   }else{
131     c->nBytes = nBytes;
132   }
133   c->iOffset = 0;                 /* start tokenizing at the beginning */
134   c->iToken = 0;
135   c->pToken = NULL;               /* no space allocated, yet. */
136   c->nTokenAllocated = 0;
137 
138   *ppCursor = &c->base;
139   return SQLITE_OK;
140 }
141 
142 /*
143 ** Close a tokenization cursor previously opened by a call to
144 ** simpleOpen() above.
145 */
146 static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
147   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
148   sqlite3_free(c->pToken);
149   sqlite3_free(c);
150   return SQLITE_OK;
151 }
152 
153 /*
154 ** Extract the next token from a tokenization cursor.  The cursor must
155 ** have been opened by a prior call to simpleOpen().
156 */
157 static int simpleNext(
158   sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
159   const char **ppToken,               /* OUT: *ppToken is the token text */
160   int *pnBytes,                       /* OUT: Number of bytes in token */
161   int *piStartOffset,                 /* OUT: Starting offset of token */
162   int *piEndOffset,                   /* OUT: Ending offset of token */
163   int *piPosition                     /* OUT: Position integer of token */
164 ){
165   simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
166   simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
167   unsigned char *p = (unsigned char *)c->pInput;
168 
169   while( c->iOffset<c->nBytes ){
170     int iStartOffset;
171 
172     /* Scan past delimiter characters */
173     while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
174       c->iOffset++;
175     }
176 
177     /* Count non-delimiter characters. */
178     iStartOffset = c->iOffset;
179     while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
180       c->iOffset++;
181     }
182 
183     if( c->iOffset>iStartOffset ){
184       int i, n = c->iOffset-iStartOffset;
185       if( n>c->nTokenAllocated ){
186         char *pNew;
187         c->nTokenAllocated = n+20;
188         pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated);
189         if( !pNew ) return SQLITE_NOMEM;
190         c->pToken = pNew;
191       }
192       for(i=0; i<n; i++){
193         /* TODO(shess) This needs expansion to handle UTF-8
194         ** case-insensitivity.
195         */
196         unsigned char ch = p[iStartOffset+i];
197         c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
198       }
199       *ppToken = c->pToken;
200       *pnBytes = n;
201       *piStartOffset = iStartOffset;
202       *piEndOffset = c->iOffset;
203       *piPosition = c->iToken++;
204 
205       return SQLITE_OK;
206     }
207   }
208   return SQLITE_DONE;
209 }
210 
211 /*
212 ** The set of routines that implement the simple tokenizer
213 */
214 static const sqlite3_tokenizer_module simpleTokenizerModule = {
215   0,
216   simpleCreate,
217   simpleDestroy,
218   simpleOpen,
219   simpleClose,
220   simpleNext,
221   0,
222 };
223 
224 /*
225 ** Allocate a new simple tokenizer.  Return a pointer to the new
226 ** tokenizer in *ppModule
227 */
228 void sqlite3Fts3SimpleTokenizerModule(
229   sqlite3_tokenizer_module const**ppModule
230 ){
231   *ppModule = &simpleTokenizerModule;
232 }
233 
234 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
235