xref: /sqlite-3.40.0/src/tokenize.c (revision 5314c4d7)
175897234Sdrh /*
2b19a2bc6Sdrh ** 2001 September 15
375897234Sdrh **
4b19a2bc6Sdrh ** The author disclaims copyright to this source code.  In place of
5b19a2bc6Sdrh ** a legal notice, here is a blessing:
675897234Sdrh **
7b19a2bc6Sdrh **    May you do good and not evil.
8b19a2bc6Sdrh **    May you find forgiveness for yourself and forgive others.
9b19a2bc6Sdrh **    May you share freely, never taking more than you give.
1075897234Sdrh **
1175897234Sdrh *************************************************************************
1275897234Sdrh ** An tokenizer for SQL
1375897234Sdrh **
1475897234Sdrh ** This file contains C code that splits an SQL input string up into
1575897234Sdrh ** individual tokens and sends those tokens one-by-one over to the
1675897234Sdrh ** parser for analysis.
1775897234Sdrh **
18*5314c4d7Sdanielk1977 ** $Id: tokenize.c,v 1.78 2004/06/18 06:02:35 danielk1977 Exp $
1975897234Sdrh */
2075897234Sdrh #include "sqliteInt.h"
21ad75e987Sdrh #include "os.h"
2275897234Sdrh #include <ctype.h>
23dcc581ccSdrh #include <stdlib.h>
2475897234Sdrh 
2575897234Sdrh /*
2675897234Sdrh ** All the keywords of the SQL language are stored as in a hash
2775897234Sdrh ** table composed of instances of the following structure.
2875897234Sdrh */
2975897234Sdrh typedef struct Keyword Keyword;
3075897234Sdrh struct Keyword {
3175897234Sdrh   char *zName;             /* The keyword name */
32ba21256fSdrh   u8 tokenType;            /* Token value for this keyword */
33ba21256fSdrh   u8 len;                  /* Length of this keyword */
34ba21256fSdrh   u8 iNext;                /* Index in aKeywordTable[] of next with same hash */
3575897234Sdrh };
3675897234Sdrh 
3775897234Sdrh /*
3875897234Sdrh ** These are the keywords
3975897234Sdrh */
4075897234Sdrh static Keyword aKeywordTable[] = {
41ba21256fSdrh   { "ABORT",             TK_ABORT,        },
42ba21256fSdrh   { "AFTER",             TK_AFTER,        },
43ba21256fSdrh   { "ALL",               TK_ALL,          },
44ba21256fSdrh   { "AND",               TK_AND,          },
45ba21256fSdrh   { "AS",                TK_AS,           },
46ba21256fSdrh   { "ASC",               TK_ASC,          },
47ba21256fSdrh   { "ATTACH",            TK_ATTACH,       },
48ba21256fSdrh   { "BEFORE",            TK_BEFORE,       },
49ba21256fSdrh   { "BEGIN",             TK_BEGIN,        },
50ba21256fSdrh   { "BETWEEN",           TK_BETWEEN,      },
51ba21256fSdrh   { "BY",                TK_BY,           },
52ba21256fSdrh   { "CASCADE",           TK_CASCADE,      },
53ba21256fSdrh   { "CASE",              TK_CASE,         },
54ba21256fSdrh   { "CHECK",             TK_CHECK,        },
55ba21256fSdrh   { "CLUSTER",           TK_CLUSTER,      },
56ba21256fSdrh   { "COLLATE",           TK_COLLATE,      },
57ba21256fSdrh   { "COMMIT",            TK_COMMIT,       },
58ba21256fSdrh   { "CONFLICT",          TK_CONFLICT,     },
59ba21256fSdrh   { "CONSTRAINT",        TK_CONSTRAINT,   },
60ba21256fSdrh   { "CREATE",            TK_CREATE,       },
61ba21256fSdrh   { "CROSS",             TK_JOIN_KW,      },
62ba21256fSdrh   { "DATABASE",          TK_DATABASE,     },
63ba21256fSdrh   { "DEFAULT",           TK_DEFAULT,      },
64ba21256fSdrh   { "DEFERRED",          TK_DEFERRED,     },
65ba21256fSdrh   { "DEFERRABLE",        TK_DEFERRABLE,   },
66ba21256fSdrh   { "DELETE",            TK_DELETE,       },
67ba21256fSdrh   { "DESC",              TK_DESC,         },
68ba21256fSdrh   { "DETACH",            TK_DETACH,       },
69ba21256fSdrh   { "DISTINCT",          TK_DISTINCT,     },
70ba21256fSdrh   { "DROP",              TK_DROP,         },
71ba21256fSdrh   { "END",               TK_END,          },
72ba21256fSdrh   { "EACH",              TK_EACH,         },
73ba21256fSdrh   { "ELSE",              TK_ELSE,         },
74ba21256fSdrh   { "EXCEPT",            TK_EXCEPT,       },
75ba21256fSdrh   { "EXPLAIN",           TK_EXPLAIN,      },
76ba21256fSdrh   { "FAIL",              TK_FAIL,         },
77ba21256fSdrh   { "FOR",               TK_FOR,          },
78ba21256fSdrh   { "FOREIGN",           TK_FOREIGN,      },
79ba21256fSdrh   { "FROM",              TK_FROM,         },
80ba21256fSdrh   { "FULL",              TK_JOIN_KW,      },
81ba21256fSdrh   { "GLOB",              TK_GLOB,         },
82ba21256fSdrh   { "GROUP",             TK_GROUP,        },
83ba21256fSdrh   { "HAVING",            TK_HAVING,       },
84ba21256fSdrh   { "IGNORE",            TK_IGNORE,       },
85ba21256fSdrh   { "IMMEDIATE",         TK_IMMEDIATE,    },
86ba21256fSdrh   { "IN",                TK_IN,           },
87ba21256fSdrh   { "INDEX",             TK_INDEX,        },
88ba21256fSdrh   { "INITIALLY",         TK_INITIALLY,    },
89ba21256fSdrh   { "INNER",             TK_JOIN_KW,      },
90ba21256fSdrh   { "INSERT",            TK_INSERT,       },
91ba21256fSdrh   { "INSTEAD",           TK_INSTEAD,      },
92ba21256fSdrh   { "INTERSECT",         TK_INTERSECT,    },
93ba21256fSdrh   { "INTO",              TK_INTO,         },
94ba21256fSdrh   { "IS",                TK_IS,           },
95ba21256fSdrh   { "ISNULL",            TK_ISNULL,       },
96ba21256fSdrh   { "JOIN",              TK_JOIN,         },
97ba21256fSdrh   { "KEY",               TK_KEY,          },
98ba21256fSdrh   { "LEFT",              TK_JOIN_KW,      },
99ba21256fSdrh   { "LIKE",              TK_LIKE,         },
100ba21256fSdrh   { "LIMIT",             TK_LIMIT,        },
101ba21256fSdrh   { "MATCH",             TK_MATCH,        },
102ba21256fSdrh   { "NATURAL",           TK_JOIN_KW,      },
103ba21256fSdrh   { "NOT",               TK_NOT,          },
104ba21256fSdrh   { "NOTNULL",           TK_NOTNULL,      },
105ba21256fSdrh   { "NULL",              TK_NULL,         },
106ba21256fSdrh   { "OF",                TK_OF,           },
107ba21256fSdrh   { "OFFSET",            TK_OFFSET,       },
108ba21256fSdrh   { "ON",                TK_ON,           },
109ba21256fSdrh   { "OR",                TK_OR,           },
110ba21256fSdrh   { "ORDER",             TK_ORDER,        },
111ba21256fSdrh   { "OUTER",             TK_JOIN_KW,      },
112ba21256fSdrh   { "PRAGMA",            TK_PRAGMA,       },
113ba21256fSdrh   { "PRIMARY",           TK_PRIMARY,      },
114ba21256fSdrh   { "RAISE",             TK_RAISE,        },
115ba21256fSdrh   { "REFERENCES",        TK_REFERENCES,   },
116ba21256fSdrh   { "REPLACE",           TK_REPLACE,      },
117ba21256fSdrh   { "RESTRICT",          TK_RESTRICT,     },
118ba21256fSdrh   { "RIGHT",             TK_JOIN_KW,      },
119ba21256fSdrh   { "ROLLBACK",          TK_ROLLBACK,     },
120ba21256fSdrh   { "ROW",               TK_ROW,          },
121ba21256fSdrh   { "SELECT",            TK_SELECT,       },
122ba21256fSdrh   { "SET",               TK_SET,          },
123ba21256fSdrh   { "STATEMENT",         TK_STATEMENT,    },
124ba21256fSdrh   { "TABLE",             TK_TABLE,        },
125ba21256fSdrh   { "TEMP",              TK_TEMP,         },
126ba21256fSdrh   { "TEMPORARY",         TK_TEMP,         },
127ba21256fSdrh   { "THEN",              TK_THEN,         },
128ba21256fSdrh   { "TRANSACTION",       TK_TRANSACTION,  },
129ba21256fSdrh   { "TRIGGER",           TK_TRIGGER,      },
130ba21256fSdrh   { "UNION",             TK_UNION,        },
131ba21256fSdrh   { "UNIQUE",            TK_UNIQUE,       },
132ba21256fSdrh   { "UPDATE",            TK_UPDATE,       },
133ba21256fSdrh   { "USING",             TK_USING,        },
134ba21256fSdrh   { "VACUUM",            TK_VACUUM,       },
135ba21256fSdrh   { "VALUES",            TK_VALUES,       },
136ba21256fSdrh   { "VIEW",              TK_VIEW,         },
137ba21256fSdrh   { "WHEN",              TK_WHEN,         },
138ba21256fSdrh   { "WHERE",             TK_WHERE,        },
13975897234Sdrh };
14075897234Sdrh 
14175897234Sdrh /*
14275897234Sdrh ** This is the hash table
14375897234Sdrh */
144ba21256fSdrh #define KEY_HASH_SIZE 101
145ba21256fSdrh static u8 aiHashTable[KEY_HASH_SIZE];
14675897234Sdrh 
14775897234Sdrh 
14875897234Sdrh /*
14975897234Sdrh ** This function looks up an identifier to determine if it is a
15075897234Sdrh ** keyword.  If it is a keyword, the token code of that keyword is
15175897234Sdrh ** returned.  If the input is not a keyword, TK_ID is returned.
15275897234Sdrh */
1534adee20fSdanielk1977 int sqlite3KeywordCode(const char *z, int n){
154ba21256fSdrh   int h, i;
15575897234Sdrh   Keyword *p;
15693a5c6bdSdrh   static char needInit = 1;
15793a5c6bdSdrh   if( needInit ){
15875897234Sdrh     /* Initialize the keyword hash table */
1594adee20fSdanielk1977     sqlite3OsEnterMutex();
16093a5c6bdSdrh     if( needInit ){
161ba21256fSdrh       int nk;
162ba21256fSdrh       nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
163ba21256fSdrh       for(i=0; i<nk; i++){
16475897234Sdrh         aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
1654adee20fSdanielk1977         h = sqlite3HashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
16675897234Sdrh         h %= KEY_HASH_SIZE;
167ba21256fSdrh         aKeywordTable[i].iNext = aiHashTable[h];
168ba21256fSdrh         aiHashTable[h] = i+1;
16975897234Sdrh       }
17093a5c6bdSdrh       needInit = 0;
17175897234Sdrh     }
1724adee20fSdanielk1977     sqlite3OsLeaveMutex();
173ad75e987Sdrh   }
1744adee20fSdanielk1977   h = sqlite3HashNoCase(z, n) % KEY_HASH_SIZE;
175ba21256fSdrh   for(i=aiHashTable[h]; i; i=p->iNext){
176ba21256fSdrh     p = &aKeywordTable[i-1];
1774adee20fSdanielk1977     if( p->len==n && sqlite3StrNICmp(p->zName, z, n)==0 ){
17875897234Sdrh       return p->tokenType;
17975897234Sdrh     }
18075897234Sdrh   }
18175897234Sdrh   return TK_ID;
18275897234Sdrh }
18375897234Sdrh 
18498808babSdrh 
18598808babSdrh /*
186ba21256fSdrh ** If X is a character that can be used in an identifier and
187ba21256fSdrh ** X&0x80==0 then isIdChar[X] will be 1.  If X&0x80==0x80 then
188ba21256fSdrh ** X is always an identifier character.  (Hence all UTF-8
189ba21256fSdrh ** characters can be part of an identifier).  isIdChar[X] will
190ba21256fSdrh ** be 0 for every character in the lower 128 ASCII characters
191ba21256fSdrh ** that cannot be used as part of an identifier.
19298808babSdrh **
19398808babSdrh ** In this implementation, an identifier can be a string of
19498808babSdrh ** alphabetic characters, digits, and "_" plus any character
19598808babSdrh ** with the high-order bit set.  The latter rule means that
19698808babSdrh ** any sequence of UTF-8 characters or characters taken from
19798808babSdrh ** an extended ISO8859 character set can form an identifier.
19898808babSdrh */
19998808babSdrh static const char isIdChar[] = {
20098808babSdrh /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
20198808babSdrh     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
20298808babSdrh     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 1x */
20398808babSdrh     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
20498808babSdrh     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
20598808babSdrh     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
20698808babSdrh     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
20798808babSdrh     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
20898808babSdrh     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
20998808babSdrh };
21098808babSdrh 
21198808babSdrh 
21275897234Sdrh /*
21361b487d0Sdrh ** Return the length of the token that begins at z[0].
21461b487d0Sdrh ** Store the token type in *tokenType before returning.
21575897234Sdrh */
21698808babSdrh static int sqliteGetToken(const unsigned char *z, int *tokenType){
21775897234Sdrh   int i;
21875897234Sdrh   switch( *z ){
21930cab809Sdrh     case ' ': case '\t': case '\n': case '\f': case '\r': {
22032eb7b47Sdrh       for(i=1; isspace(z[i]); i++){}
22175897234Sdrh       *tokenType = TK_SPACE;
22275897234Sdrh       return i;
22375897234Sdrh     }
22475897234Sdrh     case '-': {
22575897234Sdrh       if( z[1]=='-' ){
22675897234Sdrh         for(i=2; z[i] && z[i]!='\n'; i++){}
22775897234Sdrh         *tokenType = TK_COMMENT;
22875897234Sdrh         return i;
22975897234Sdrh       }
23075897234Sdrh       *tokenType = TK_MINUS;
23175897234Sdrh       return 1;
23275897234Sdrh     }
23375897234Sdrh     case '(': {
23475897234Sdrh       *tokenType = TK_LP;
23575897234Sdrh       return 1;
23675897234Sdrh     }
23775897234Sdrh     case ')': {
23875897234Sdrh       *tokenType = TK_RP;
23975897234Sdrh       return 1;
24075897234Sdrh     }
24175897234Sdrh     case ';': {
24275897234Sdrh       *tokenType = TK_SEMI;
24375897234Sdrh       return 1;
24475897234Sdrh     }
24575897234Sdrh     case '+': {
24675897234Sdrh       *tokenType = TK_PLUS;
24775897234Sdrh       return 1;
24875897234Sdrh     }
24975897234Sdrh     case '*': {
25075897234Sdrh       *tokenType = TK_STAR;
25175897234Sdrh       return 1;
25275897234Sdrh     }
25375897234Sdrh     case '/': {
25466105a8eSdrh       if( z[1]!='*' || z[2]==0 ){
25575897234Sdrh         *tokenType = TK_SLASH;
25675897234Sdrh         return 1;
25775897234Sdrh       }
25866105a8eSdrh       for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
25966105a8eSdrh       if( z[i] ) i++;
26066105a8eSdrh       *tokenType = TK_COMMENT;
26166105a8eSdrh       return i;
26266105a8eSdrh     }
263bf4133cbSdrh     case '%': {
264bf4133cbSdrh       *tokenType = TK_REM;
265bf4133cbSdrh       return 1;
266bf4133cbSdrh     }
26775897234Sdrh     case '=': {
26875897234Sdrh       *tokenType = TK_EQ;
26975897234Sdrh       return 1 + (z[1]=='=');
27075897234Sdrh     }
27175897234Sdrh     case '<': {
27275897234Sdrh       if( z[1]=='=' ){
27375897234Sdrh         *tokenType = TK_LE;
27475897234Sdrh         return 2;
27575897234Sdrh       }else if( z[1]=='>' ){
27675897234Sdrh         *tokenType = TK_NE;
27775897234Sdrh         return 2;
278bf4133cbSdrh       }else if( z[1]=='<' ){
279bf4133cbSdrh         *tokenType = TK_LSHIFT;
280bf4133cbSdrh         return 2;
28175897234Sdrh       }else{
28275897234Sdrh         *tokenType = TK_LT;
28375897234Sdrh         return 1;
28475897234Sdrh       }
28575897234Sdrh     }
28675897234Sdrh     case '>': {
28775897234Sdrh       if( z[1]=='=' ){
28875897234Sdrh         *tokenType = TK_GE;
28975897234Sdrh         return 2;
290bf4133cbSdrh       }else if( z[1]=='>' ){
291bf4133cbSdrh         *tokenType = TK_RSHIFT;
292bf4133cbSdrh         return 2;
29375897234Sdrh       }else{
29475897234Sdrh         *tokenType = TK_GT;
29575897234Sdrh         return 1;
29675897234Sdrh       }
29775897234Sdrh     }
29875897234Sdrh     case '!': {
29975897234Sdrh       if( z[1]!='=' ){
30075897234Sdrh         *tokenType = TK_ILLEGAL;
301c837e709Sdrh         return 2;
30275897234Sdrh       }else{
30375897234Sdrh         *tokenType = TK_NE;
30475897234Sdrh         return 2;
30575897234Sdrh       }
30675897234Sdrh     }
3070040077dSdrh     case '|': {
3080040077dSdrh       if( z[1]!='|' ){
309bf4133cbSdrh         *tokenType = TK_BITOR;
3100040077dSdrh         return 1;
3110040077dSdrh       }else{
3120040077dSdrh         *tokenType = TK_CONCAT;
3130040077dSdrh         return 2;
3140040077dSdrh       }
3150040077dSdrh     }
31675897234Sdrh     case ',': {
31775897234Sdrh       *tokenType = TK_COMMA;
31875897234Sdrh       return 1;
31975897234Sdrh     }
320bf4133cbSdrh     case '&': {
321bf4133cbSdrh       *tokenType = TK_BITAND;
322bf4133cbSdrh       return 1;
323bf4133cbSdrh     }
324bf4133cbSdrh     case '~': {
325bf4133cbSdrh       *tokenType = TK_BITNOT;
326bf4133cbSdrh       return 1;
327bf4133cbSdrh     }
32875897234Sdrh     case '\'': case '"': {
32975897234Sdrh       int delim = z[0];
33075897234Sdrh       for(i=1; z[i]; i++){
33175897234Sdrh         if( z[i]==delim ){
33275897234Sdrh           if( z[i+1]==delim ){
33375897234Sdrh             i++;
33475897234Sdrh           }else{
33575897234Sdrh             break;
33675897234Sdrh           }
33775897234Sdrh         }
33875897234Sdrh       }
33975897234Sdrh       if( z[i] ) i++;
34075897234Sdrh       *tokenType = TK_STRING;
34175897234Sdrh       return i;
34275897234Sdrh     }
34375897234Sdrh     case '.': {
34475897234Sdrh       *tokenType = TK_DOT;
34575897234Sdrh       return 1;
34675897234Sdrh     }
34775897234Sdrh     case '0': case '1': case '2': case '3': case '4':
34875897234Sdrh     case '5': case '6': case '7': case '8': case '9': {
349c837e709Sdrh       *tokenType = TK_INTEGER;
35032eb7b47Sdrh       for(i=1; isdigit(z[i]); i++){}
351bb07e9a3Sdrh       if( z[i]=='.' && isdigit(z[i+1]) ){
352bb07e9a3Sdrh         i += 2;
35332eb7b47Sdrh         while( isdigit(z[i]) ){ i++; }
354c837e709Sdrh         *tokenType = TK_FLOAT;
355c837e709Sdrh       }
35675897234Sdrh       if( (z[i]=='e' || z[i]=='E') &&
35775897234Sdrh            ( isdigit(z[i+1])
35875897234Sdrh             || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
35975897234Sdrh            )
36075897234Sdrh       ){
36175897234Sdrh         i += 2;
36232eb7b47Sdrh         while( isdigit(z[i]) ){ i++; }
36375897234Sdrh         *tokenType = TK_FLOAT;
36475897234Sdrh       }
36575897234Sdrh       return i;
36675897234Sdrh     }
3672f4392ffSdrh     case '[': {
3682f4392ffSdrh       for(i=1; z[i] && z[i-1]!=']'; i++){}
3692f4392ffSdrh       *tokenType = TK_ID;
3702f4392ffSdrh       return i;
3712f4392ffSdrh     }
3727c972decSdrh     case '?': {
37350457896Sdrh       *tokenType = TK_VARIABLE;
3747c972decSdrh       return 1;
37550457896Sdrh     }
376c572ef7fSdanielk1977     case 'x': case 'X': {
377c572ef7fSdanielk1977       if( z[1]=='\'' || z[1]=='"' ){
3783fd0a736Sdanielk1977         int delim = z[1];
3793fd0a736Sdanielk1977         *tokenType = TK_BLOB;
3803fd0a736Sdanielk1977         for(i=2; z[i]; i++){
381c572ef7fSdanielk1977           if( z[i]==delim ){
3823fd0a736Sdanielk1977             if( i%2 ) *tokenType = TK_ILLEGAL;
383c572ef7fSdanielk1977             break;
384c572ef7fSdanielk1977           }
3853fd0a736Sdanielk1977           if( !isxdigit(z[i]) ){
3863fd0a736Sdanielk1977             *tokenType = TK_ILLEGAL;
3873fd0a736Sdanielk1977             return i;
3883fd0a736Sdanielk1977           }
389c572ef7fSdanielk1977         }
390c572ef7fSdanielk1977         if( z[i] ) i++;
391c572ef7fSdanielk1977         return i;
392c572ef7fSdanielk1977       }
393c572ef7fSdanielk1977       /* Otherwise fall through to the next case */
394c572ef7fSdanielk1977     }
39598808babSdrh     default: {
396ba21256fSdrh       if( (*z&0x80)==0 && !isIdChar[*z] ){
39798808babSdrh         break;
39898808babSdrh       }
399ba21256fSdrh       for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
4004adee20fSdanielk1977       *tokenType = sqlite3KeywordCode((char*)z, i);
40175897234Sdrh       return i;
40275897234Sdrh     }
40375897234Sdrh   }
40475897234Sdrh   *tokenType = TK_ILLEGAL;
40575897234Sdrh   return 1;
40675897234Sdrh }
40775897234Sdrh 
40875897234Sdrh /*
40975897234Sdrh ** Run the parser on the given SQL string.  The parser structure is
410b19a2bc6Sdrh ** passed in.  An SQLITE_ status code is returned.  If an error occurs
411b19a2bc6Sdrh ** and pzErrMsg!=NULL then an error message might be written into
412b19a2bc6Sdrh ** memory obtained from malloc() and *pzErrMsg made to point to that
413b19a2bc6Sdrh ** error message.  Or maybe not.
41475897234Sdrh */
4154adee20fSdanielk1977 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
41675897234Sdrh   int nErr = 0;
41775897234Sdrh   int i;
41875897234Sdrh   void *pEngine;
419b86ccfb2Sdrh   int tokenType;
420b86ccfb2Sdrh   int lastTokenParsed = -1;
4216d4abfbeSdrh   sqlite *db = pParse->db;
4224adee20fSdanielk1977   extern void *sqlite3ParserAlloc(void*(*)(int));
4234adee20fSdanielk1977   extern void sqlite3ParserFree(void*, void(*)(void*));
4244adee20fSdanielk1977   extern int sqlite3Parser(void*, int, Token, Parse*);
42575897234Sdrh 
4266d4abfbeSdrh   db->flags &= ~SQLITE_Interrupt;
4274c504391Sdrh   pParse->rc = SQLITE_OK;
42875897234Sdrh   i = 0;
4294adee20fSdanielk1977   pEngine = sqlite3ParserAlloc((void*(*)(int))malloc);
43075897234Sdrh   if( pEngine==0 ){
4314adee20fSdanielk1977     sqlite3SetString(pzErrMsg, "out of memory", (char*)0);
43275897234Sdrh     return 1;
43375897234Sdrh   }
43432eb7b47Sdrh   pParse->sLastToken.dyn = 0;
435326dce74Sdrh   pParse->zTail = zSql;
4366f8a503dSdanielk1977   while( sqlite3_malloc_failed==0 && zSql[i]!=0 ){
43732eb7b47Sdrh     assert( i>=0 );
43875897234Sdrh     pParse->sLastToken.z = &zSql[i];
43932eb7b47Sdrh     assert( pParse->sLastToken.dyn==0 );
44098808babSdrh     pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType);
44175897234Sdrh     i += pParse->sLastToken.n;
44275897234Sdrh     switch( tokenType ){
44375897234Sdrh       case TK_SPACE:
44475897234Sdrh       case TK_COMMENT: {
44532eb7b47Sdrh         if( (db->flags & SQLITE_Interrupt)!=0 ){
44632eb7b47Sdrh           pParse->rc = SQLITE_INTERRUPT;
4474adee20fSdanielk1977           sqlite3SetString(pzErrMsg, "interrupt", (char*)0);
44832eb7b47Sdrh           goto abort_parse;
44932eb7b47Sdrh         }
45075897234Sdrh         break;
45175897234Sdrh       }
45232eb7b47Sdrh       case TK_ILLEGAL: {
4534adee20fSdanielk1977         sqlite3SetNString(pzErrMsg, "unrecognized token: \"", -1,
454c837e709Sdrh            pParse->sLastToken.z, pParse->sLastToken.n, "\"", 1, 0);
45575897234Sdrh         nErr++;
456caec2f12Sdrh         goto abort_parse;
45732eb7b47Sdrh       }
458326dce74Sdrh       case TK_SEMI: {
459326dce74Sdrh         pParse->zTail = &zSql[i];
460326dce74Sdrh         /* Fall thru into the default case */
461326dce74Sdrh       }
46232eb7b47Sdrh       default: {
4634adee20fSdanielk1977         sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
464b86ccfb2Sdrh         lastTokenParsed = tokenType;
465b86ccfb2Sdrh         if( pParse->rc!=SQLITE_OK ){
46632eb7b47Sdrh           goto abort_parse;
46775897234Sdrh         }
46875897234Sdrh         break;
46975897234Sdrh       }
47075897234Sdrh     }
47132eb7b47Sdrh   }
47232eb7b47Sdrh abort_parse:
473b86ccfb2Sdrh   if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){
474b86ccfb2Sdrh     if( lastTokenParsed!=TK_SEMI ){
4754adee20fSdanielk1977       sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
476326dce74Sdrh       pParse->zTail = &zSql[i];
47775897234Sdrh     }
4784adee20fSdanielk1977     sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
47975897234Sdrh   }
4804adee20fSdanielk1977   sqlite3ParserFree(pEngine, free);
481b86ccfb2Sdrh   if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
482f20b21c8Sdanielk1977     sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc),
48341743984Sdrh                     (char*)0);
484b86ccfb2Sdrh   }
48575897234Sdrh   if( pParse->zErrMsg ){
486b86ccfb2Sdrh     if( pzErrMsg && *pzErrMsg==0 ){
48775897234Sdrh       *pzErrMsg = pParse->zErrMsg;
48875897234Sdrh     }else{
48975897234Sdrh       sqliteFree(pParse->zErrMsg);
49075897234Sdrh     }
491b86ccfb2Sdrh     pParse->zErrMsg = 0;
49275897234Sdrh     if( !nErr ) nErr++;
49375897234Sdrh   }
494826fb5a3Sdrh   if( pParse->pVdbe && pParse->nErr>0 ){
4954adee20fSdanielk1977     sqlite3VdbeDelete(pParse->pVdbe);
49675897234Sdrh     pParse->pVdbe = 0;
49775897234Sdrh   }
49875897234Sdrh   if( pParse->pNewTable ){
4994adee20fSdanielk1977     sqlite3DeleteTable(pParse->db, pParse->pNewTable);
50075897234Sdrh     pParse->pNewTable = 0;
50175897234Sdrh   }
502f0f258b1Sdrh   if( pParse->pNewTrigger ){
5034adee20fSdanielk1977     sqlite3DeleteTrigger(pParse->pNewTrigger);
504f0f258b1Sdrh     pParse->pNewTrigger = 0;
505f0f258b1Sdrh   }
506b86ccfb2Sdrh   if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){
5074c504391Sdrh     pParse->rc = SQLITE_ERROR;
5084c504391Sdrh   }
50975897234Sdrh   return nErr;
51075897234Sdrh }
5117ad4334aSdrh 
5127ad4334aSdrh /*
5136f8a503dSdanielk1977 ** Token types used by the sqlite3_complete() routine.  See the header
5147ad4334aSdrh ** comments on that procedure for additional information.
5157ad4334aSdrh */
5167ad4334aSdrh #define tkEXPLAIN 0
5177ad4334aSdrh #define tkCREATE  1
5187ad4334aSdrh #define tkTEMP    2
5197ad4334aSdrh #define tkTRIGGER 3
5207ad4334aSdrh #define tkEND     4
5217ad4334aSdrh #define tkSEMI    5
5227ad4334aSdrh #define tkWS      6
5237ad4334aSdrh #define tkOTHER   7
5247ad4334aSdrh 
5257ad4334aSdrh /*
5267ad4334aSdrh ** Return TRUE if the given SQL string ends in a semicolon.
5277ad4334aSdrh **
5287ad4334aSdrh ** Special handling is require for CREATE TRIGGER statements.
5297ad4334aSdrh ** Whenever the CREATE TRIGGER keywords are seen, the statement
5307ad4334aSdrh ** must end with ";END;".
5317ad4334aSdrh **
5327ad4334aSdrh ** This implementation uses a state machine with 7 states:
5337ad4334aSdrh **
5347ad4334aSdrh **   (0) START     At the beginning or end of an SQL statement.  This routine
5357ad4334aSdrh **                 returns 1 if it ends in the START state and 0 if it ends
5367ad4334aSdrh **                 in any other state.
5377ad4334aSdrh **
5387ad4334aSdrh **   (1) EXPLAIN   The keyword EXPLAIN has been seen at the beginning of
5397ad4334aSdrh **                 a statement.
5407ad4334aSdrh **
5417ad4334aSdrh **   (2) CREATE    The keyword CREATE has been seen at the beginning of a
5427ad4334aSdrh **                 statement, possibly preceeded by EXPLAIN and/or followed by
5437ad4334aSdrh **                 TEMP or TEMPORARY
5447ad4334aSdrh **
5457ad4334aSdrh **   (3) NORMAL    We are in the middle of statement which ends with a single
5467ad4334aSdrh **                 semicolon.
5477ad4334aSdrh **
5487ad4334aSdrh **   (4) TRIGGER   We are in the middle of a trigger definition that must be
5497ad4334aSdrh **                 ended by a semicolon, the keyword END, and another semicolon.
5507ad4334aSdrh **
5517ad4334aSdrh **   (5) SEMI      We've seen the first semicolon in the ";END;" that occurs at
5527ad4334aSdrh **                 the end of a trigger definition.
5537ad4334aSdrh **
5547ad4334aSdrh **   (6) END       We've seen the ";END" of the ";END;" that occurs at the end
5557ad4334aSdrh **                 of a trigger difinition.
5567ad4334aSdrh **
5577ad4334aSdrh ** Transitions between states above are determined by tokens extracted
5587ad4334aSdrh ** from the input.  The following tokens are significant:
5597ad4334aSdrh **
5607ad4334aSdrh **   (0) tkEXPLAIN   The "explain" keyword.
5617ad4334aSdrh **   (1) tkCREATE    The "create" keyword.
5627ad4334aSdrh **   (2) tkTEMP      The "temp" or "temporary" keyword.
5637ad4334aSdrh **   (3) tkTRIGGER   The "trigger" keyword.
5647ad4334aSdrh **   (4) tkEND       The "end" keyword.
5657ad4334aSdrh **   (5) tkSEMI      A semicolon.
5667ad4334aSdrh **   (6) tkWS        Whitespace
5677ad4334aSdrh **   (7) tkOTHER     Any other SQL token.
5687ad4334aSdrh **
5697ad4334aSdrh ** Whitespace never causes a state transition and is always ignored.
5707ad4334aSdrh */
5716f8a503dSdanielk1977 int sqlite3_complete(const char *zSql){
5727ad4334aSdrh   u8 state = 0;   /* Current state, using numbers defined in header comment */
5737ad4334aSdrh   u8 token;       /* Value of the next token */
5747ad4334aSdrh 
5757ad4334aSdrh   /* The following matrix defines the transition from one state to another
5767ad4334aSdrh   ** according to what token is seen.  trans[state][token] returns the
5777ad4334aSdrh   ** next state.
5787ad4334aSdrh   */
5797ad4334aSdrh   static const u8 trans[7][8] = {
580e1e38c4dSdrh                      /* Token:                                                */
5817ad4334aSdrh      /* State:       **  EXPLAIN  CREATE  TEMP  TRIGGER  END  SEMI  WS  OTHER */
5827ad4334aSdrh      /* 0   START: */ {       1,      2,    3,       3,   3,    0,  0,     3, },
5837ad4334aSdrh      /* 1 EXPLAIN: */ {       3,      2,    3,       3,   3,    0,  1,     3, },
5847ad4334aSdrh      /* 2  CREATE: */ {       3,      3,    2,       4,   3,    0,  2,     3, },
5857ad4334aSdrh      /* 3  NORMAL: */ {       3,      3,    3,       3,   3,    0,  3,     3, },
5867ad4334aSdrh      /* 4 TRIGGER: */ {       4,      4,    4,       4,   4,    5,  4,     4, },
5877ad4334aSdrh      /* 5    SEMI: */ {       4,      4,    4,       4,   6,    5,  5,     4, },
5887ad4334aSdrh      /* 6     END: */ {       4,      4,    4,       4,   4,    0,  6,     4, },
5897ad4334aSdrh   };
5907ad4334aSdrh 
5917ad4334aSdrh   while( *zSql ){
5927ad4334aSdrh     switch( *zSql ){
5937ad4334aSdrh       case ';': {  /* A semicolon */
5947ad4334aSdrh         token = tkSEMI;
5957ad4334aSdrh         break;
5967ad4334aSdrh       }
5977ad4334aSdrh       case ' ':
5987ad4334aSdrh       case '\r':
5997ad4334aSdrh       case '\t':
6007ad4334aSdrh       case '\n':
6017ad4334aSdrh       case '\f': {  /* White space is ignored */
6027ad4334aSdrh         token = tkWS;
6037ad4334aSdrh         break;
6047ad4334aSdrh       }
6057ad4334aSdrh       case '/': {   /* C-style comments */
6067ad4334aSdrh         if( zSql[1]!='*' ){
6077ad4334aSdrh           token = tkOTHER;
6087ad4334aSdrh           break;
6097ad4334aSdrh         }
6107ad4334aSdrh         zSql += 2;
6117ad4334aSdrh         while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; }
6127ad4334aSdrh         if( zSql[0]==0 ) return 0;
6137ad4334aSdrh         zSql++;
6147ad4334aSdrh         token = tkWS;
6157ad4334aSdrh         break;
6167ad4334aSdrh       }
6177ad4334aSdrh       case '-': {   /* SQL-style comments from "--" to end of line */
6187ad4334aSdrh         if( zSql[1]!='-' ){
6197ad4334aSdrh           token = tkOTHER;
6207ad4334aSdrh           break;
6217ad4334aSdrh         }
6227ad4334aSdrh         while( *zSql && *zSql!='\n' ){ zSql++; }
6237ad4334aSdrh         if( *zSql==0 ) return state==0;
6247ad4334aSdrh         token = tkWS;
6257ad4334aSdrh         break;
6267ad4334aSdrh       }
6277ad4334aSdrh       case '[': {   /* Microsoft-style identifiers in [...] */
6287ad4334aSdrh         zSql++;
6297ad4334aSdrh         while( *zSql && *zSql!=']' ){ zSql++; }
6307ad4334aSdrh         if( *zSql==0 ) return 0;
6317ad4334aSdrh         token = tkOTHER;
6327ad4334aSdrh         break;
6337ad4334aSdrh       }
6347ad4334aSdrh       case '"':     /* single- and double-quoted strings */
6357ad4334aSdrh       case '\'': {
6367ad4334aSdrh         int c = *zSql;
6377ad4334aSdrh         zSql++;
6387ad4334aSdrh         while( *zSql && *zSql!=c ){ zSql++; }
6397ad4334aSdrh         if( *zSql==0 ) return 0;
6407ad4334aSdrh         token = tkOTHER;
6417ad4334aSdrh         break;
6427ad4334aSdrh       }
6437ad4334aSdrh       default: {
644e1e38c4dSdrh         if( isIdChar[(u8)*zSql] ){
6457ad4334aSdrh           /* Keywords and unquoted identifiers */
6467ad4334aSdrh           int nId;
647e1e38c4dSdrh           for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){}
6487ad4334aSdrh           switch( *zSql ){
6497ad4334aSdrh             case 'c': case 'C': {
6504adee20fSdanielk1977               if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){
6517ad4334aSdrh                 token = tkCREATE;
6527ad4334aSdrh               }else{
6537ad4334aSdrh                 token = tkOTHER;
6547ad4334aSdrh               }
6557ad4334aSdrh               break;
6567ad4334aSdrh             }
6577ad4334aSdrh             case 't': case 'T': {
6584adee20fSdanielk1977               if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){
6597ad4334aSdrh                 token = tkTRIGGER;
6604adee20fSdanielk1977               }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){
6617ad4334aSdrh                 token = tkTEMP;
6624adee20fSdanielk1977               }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){
6637ad4334aSdrh                 token = tkTEMP;
6647ad4334aSdrh               }else{
6657ad4334aSdrh                 token = tkOTHER;
6667ad4334aSdrh               }
6677ad4334aSdrh               break;
6687ad4334aSdrh             }
6697ad4334aSdrh             case 'e':  case 'E': {
6704adee20fSdanielk1977               if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){
6717ad4334aSdrh                 token = tkEND;
6724adee20fSdanielk1977               }else if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){
6737ad4334aSdrh                 token = tkEXPLAIN;
6747ad4334aSdrh               }else{
6757ad4334aSdrh                 token = tkOTHER;
6767ad4334aSdrh               }
6777ad4334aSdrh               break;
6787ad4334aSdrh             }
6797ad4334aSdrh             default: {
6807ad4334aSdrh               token = tkOTHER;
6817ad4334aSdrh               break;
6827ad4334aSdrh             }
6837ad4334aSdrh           }
6847ad4334aSdrh           zSql += nId-1;
6857ad4334aSdrh         }else{
6867ad4334aSdrh           /* Operators and special symbols */
6877ad4334aSdrh           token = tkOTHER;
6887ad4334aSdrh         }
6897ad4334aSdrh         break;
6907ad4334aSdrh       }
6917ad4334aSdrh     }
6927ad4334aSdrh     state = trans[state][token];
6937ad4334aSdrh     zSql++;
6947ad4334aSdrh   }
6957ad4334aSdrh   return state==0;
6967ad4334aSdrh }
69761de0d1bSdanielk1977 
69861de0d1bSdanielk1977 /*
69961de0d1bSdanielk1977 ** This routine is the same as the sqlite3_complete() routine described
70061de0d1bSdanielk1977 ** above, except that the parameter is required to be UTF-16 encoded, not
70161de0d1bSdanielk1977 ** UTF-8.
70261de0d1bSdanielk1977 */
70361de0d1bSdanielk1977 int sqlite3_complete16(const void *zSql){
704bfd6cce5Sdanielk1977   sqlite3_value *pVal;
705*5314c4d7Sdanielk1977   char const *zSql8;
706bfd6cce5Sdanielk1977   int rc = 0;
707bfd6cce5Sdanielk1977 
708bfd6cce5Sdanielk1977   pVal = sqlite3ValueNew();
709bfd6cce5Sdanielk1977   sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC);
710bfd6cce5Sdanielk1977   zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8);
711bfd6cce5Sdanielk1977   if( zSql8 ){
71261de0d1bSdanielk1977     rc = sqlite3_complete(zSql8);
713bfd6cce5Sdanielk1977   }
714bfd6cce5Sdanielk1977   sqlite3ValueFree(pVal);
71561de0d1bSdanielk1977   return rc;
71661de0d1bSdanielk1977 }
717bfd6cce5Sdanielk1977 
718