175897234Sdrh /* 2b19a2bc6Sdrh ** 2001 September 15 375897234Sdrh ** 4b19a2bc6Sdrh ** The author disclaims copyright to this source code. In place of 5b19a2bc6Sdrh ** a legal notice, here is a blessing: 675897234Sdrh ** 7b19a2bc6Sdrh ** May you do good and not evil. 8b19a2bc6Sdrh ** May you find forgiveness for yourself and forgive others. 9b19a2bc6Sdrh ** May you share freely, never taking more than you give. 1075897234Sdrh ** 1175897234Sdrh ************************************************************************* 1275897234Sdrh ** An tokenizer for SQL 1375897234Sdrh ** 1475897234Sdrh ** This file contains C code that splits an SQL input string up into 1575897234Sdrh ** individual tokens and sends those tokens one-by-one over to the 1675897234Sdrh ** parser for analysis. 1775897234Sdrh ** 18*9c054830Sdrh ** $Id: tokenize.c,v 1.75 2004/05/31 18:51:58 drh Exp $ 1975897234Sdrh */ 2075897234Sdrh #include "sqliteInt.h" 21ad75e987Sdrh #include "os.h" 2275897234Sdrh #include <ctype.h> 23dcc581ccSdrh #include <stdlib.h> 2475897234Sdrh 2575897234Sdrh /* 2675897234Sdrh ** All the keywords of the SQL language are stored as in a hash 2775897234Sdrh ** table composed of instances of the following structure. 2875897234Sdrh */ 2975897234Sdrh typedef struct Keyword Keyword; 3075897234Sdrh struct Keyword { 3175897234Sdrh char *zName; /* The keyword name */ 32ba21256fSdrh u8 tokenType; /* Token value for this keyword */ 33ba21256fSdrh u8 len; /* Length of this keyword */ 34ba21256fSdrh u8 iNext; /* Index in aKeywordTable[] of next with same hash */ 3575897234Sdrh }; 3675897234Sdrh 3775897234Sdrh /* 3875897234Sdrh ** These are the keywords 3975897234Sdrh */ 4075897234Sdrh static Keyword aKeywordTable[] = { 41ba21256fSdrh { "ABORT", TK_ABORT, }, 42ba21256fSdrh { "AFTER", TK_AFTER, }, 43ba21256fSdrh { "ALL", TK_ALL, }, 44ba21256fSdrh { "AND", TK_AND, }, 45ba21256fSdrh { "AS", TK_AS, }, 46ba21256fSdrh { "ASC", TK_ASC, }, 47ba21256fSdrh { "ATTACH", TK_ATTACH, }, 48ba21256fSdrh { "BEFORE", TK_BEFORE, }, 49ba21256fSdrh { "BEGIN", TK_BEGIN, }, 50ba21256fSdrh { "BETWEEN", TK_BETWEEN, }, 51ba21256fSdrh { "BY", TK_BY, }, 52ba21256fSdrh { "CASCADE", TK_CASCADE, }, 53ba21256fSdrh { "CASE", TK_CASE, }, 54ba21256fSdrh { "CHECK", TK_CHECK, }, 55ba21256fSdrh { "CLUSTER", TK_CLUSTER, }, 56ba21256fSdrh { "COLLATE", TK_COLLATE, }, 57ba21256fSdrh { "COMMIT", TK_COMMIT, }, 58ba21256fSdrh { "CONFLICT", TK_CONFLICT, }, 59ba21256fSdrh { "CONSTRAINT", TK_CONSTRAINT, }, 60ba21256fSdrh { "CREATE", TK_CREATE, }, 61ba21256fSdrh { "CROSS", TK_JOIN_KW, }, 62ba21256fSdrh { "DATABASE", TK_DATABASE, }, 63ba21256fSdrh { "DEFAULT", TK_DEFAULT, }, 64ba21256fSdrh { "DEFERRED", TK_DEFERRED, }, 65ba21256fSdrh { "DEFERRABLE", TK_DEFERRABLE, }, 66ba21256fSdrh { "DELETE", TK_DELETE, }, 67ba21256fSdrh { "DESC", TK_DESC, }, 68ba21256fSdrh { "DETACH", TK_DETACH, }, 69ba21256fSdrh { "DISTINCT", TK_DISTINCT, }, 70ba21256fSdrh { "DROP", TK_DROP, }, 71ba21256fSdrh { "END", TK_END, }, 72ba21256fSdrh { "EACH", TK_EACH, }, 73ba21256fSdrh { "ELSE", TK_ELSE, }, 74ba21256fSdrh { "EXCEPT", TK_EXCEPT, }, 75ba21256fSdrh { "EXPLAIN", TK_EXPLAIN, }, 76ba21256fSdrh { "FAIL", TK_FAIL, }, 77ba21256fSdrh { "FOR", TK_FOR, }, 78ba21256fSdrh { "FOREIGN", TK_FOREIGN, }, 79ba21256fSdrh { "FROM", TK_FROM, }, 80ba21256fSdrh { "FULL", TK_JOIN_KW, }, 81ba21256fSdrh { "GLOB", TK_GLOB, }, 82ba21256fSdrh { "GROUP", TK_GROUP, }, 83ba21256fSdrh { "HAVING", TK_HAVING, }, 84ba21256fSdrh { "IGNORE", TK_IGNORE, }, 85ba21256fSdrh { "IMMEDIATE", TK_IMMEDIATE, }, 86ba21256fSdrh { "IN", TK_IN, }, 87ba21256fSdrh { "INDEX", TK_INDEX, }, 88ba21256fSdrh { "INITIALLY", TK_INITIALLY, }, 89ba21256fSdrh { "INNER", TK_JOIN_KW, }, 90ba21256fSdrh { "INSERT", TK_INSERT, }, 91ba21256fSdrh { "INSTEAD", TK_INSTEAD, }, 92ba21256fSdrh { "INTERSECT", TK_INTERSECT, }, 93ba21256fSdrh { "INTO", TK_INTO, }, 94ba21256fSdrh { "IS", TK_IS, }, 95ba21256fSdrh { "ISNULL", TK_ISNULL, }, 96ba21256fSdrh { "JOIN", TK_JOIN, }, 97ba21256fSdrh { "KEY", TK_KEY, }, 98ba21256fSdrh { "LEFT", TK_JOIN_KW, }, 99ba21256fSdrh { "LIKE", TK_LIKE, }, 100ba21256fSdrh { "LIMIT", TK_LIMIT, }, 101ba21256fSdrh { "MATCH", TK_MATCH, }, 102ba21256fSdrh { "NATURAL", TK_JOIN_KW, }, 103ba21256fSdrh { "NOT", TK_NOT, }, 104ba21256fSdrh { "NOTNULL", TK_NOTNULL, }, 105ba21256fSdrh { "NULL", TK_NULL, }, 106ba21256fSdrh { "OF", TK_OF, }, 107ba21256fSdrh { "OFFSET", TK_OFFSET, }, 108ba21256fSdrh { "ON", TK_ON, }, 109ba21256fSdrh { "OR", TK_OR, }, 110ba21256fSdrh { "ORDER", TK_ORDER, }, 111ba21256fSdrh { "OUTER", TK_JOIN_KW, }, 112ba21256fSdrh { "PRAGMA", TK_PRAGMA, }, 113ba21256fSdrh { "PRIMARY", TK_PRIMARY, }, 114ba21256fSdrh { "RAISE", TK_RAISE, }, 115ba21256fSdrh { "REFERENCES", TK_REFERENCES, }, 116ba21256fSdrh { "REPLACE", TK_REPLACE, }, 117ba21256fSdrh { "RESTRICT", TK_RESTRICT, }, 118ba21256fSdrh { "RIGHT", TK_JOIN_KW, }, 119ba21256fSdrh { "ROLLBACK", TK_ROLLBACK, }, 120ba21256fSdrh { "ROW", TK_ROW, }, 121ba21256fSdrh { "SELECT", TK_SELECT, }, 122ba21256fSdrh { "SET", TK_SET, }, 123ba21256fSdrh { "STATEMENT", TK_STATEMENT, }, 124ba21256fSdrh { "TABLE", TK_TABLE, }, 125ba21256fSdrh { "TEMP", TK_TEMP, }, 126ba21256fSdrh { "TEMPORARY", TK_TEMP, }, 127ba21256fSdrh { "THEN", TK_THEN, }, 128ba21256fSdrh { "TRANSACTION", TK_TRANSACTION, }, 129ba21256fSdrh { "TRIGGER", TK_TRIGGER, }, 130ba21256fSdrh { "UNION", TK_UNION, }, 131ba21256fSdrh { "UNIQUE", TK_UNIQUE, }, 132ba21256fSdrh { "UPDATE", TK_UPDATE, }, 133ba21256fSdrh { "USING", TK_USING, }, 134ba21256fSdrh { "VACUUM", TK_VACUUM, }, 135ba21256fSdrh { "VALUES", TK_VALUES, }, 136ba21256fSdrh { "VIEW", TK_VIEW, }, 137ba21256fSdrh { "WHEN", TK_WHEN, }, 138ba21256fSdrh { "WHERE", TK_WHERE, }, 13975897234Sdrh }; 14075897234Sdrh 14175897234Sdrh /* 14275897234Sdrh ** This is the hash table 14375897234Sdrh */ 144ba21256fSdrh #define KEY_HASH_SIZE 101 145ba21256fSdrh static u8 aiHashTable[KEY_HASH_SIZE]; 14675897234Sdrh 14775897234Sdrh 14875897234Sdrh /* 14975897234Sdrh ** This function looks up an identifier to determine if it is a 15075897234Sdrh ** keyword. If it is a keyword, the token code of that keyword is 15175897234Sdrh ** returned. If the input is not a keyword, TK_ID is returned. 15275897234Sdrh */ 1534adee20fSdanielk1977 int sqlite3KeywordCode(const char *z, int n){ 154ba21256fSdrh int h, i; 15575897234Sdrh Keyword *p; 15693a5c6bdSdrh static char needInit = 1; 15793a5c6bdSdrh if( needInit ){ 15875897234Sdrh /* Initialize the keyword hash table */ 1594adee20fSdanielk1977 sqlite3OsEnterMutex(); 16093a5c6bdSdrh if( needInit ){ 161ba21256fSdrh int nk; 162ba21256fSdrh nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]); 163ba21256fSdrh for(i=0; i<nk; i++){ 16475897234Sdrh aKeywordTable[i].len = strlen(aKeywordTable[i].zName); 1654adee20fSdanielk1977 h = sqlite3HashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len); 16675897234Sdrh h %= KEY_HASH_SIZE; 167ba21256fSdrh aKeywordTable[i].iNext = aiHashTable[h]; 168ba21256fSdrh aiHashTable[h] = i+1; 16975897234Sdrh } 17093a5c6bdSdrh needInit = 0; 17175897234Sdrh } 1724adee20fSdanielk1977 sqlite3OsLeaveMutex(); 173ad75e987Sdrh } 1744adee20fSdanielk1977 h = sqlite3HashNoCase(z, n) % KEY_HASH_SIZE; 175ba21256fSdrh for(i=aiHashTable[h]; i; i=p->iNext){ 176ba21256fSdrh p = &aKeywordTable[i-1]; 1774adee20fSdanielk1977 if( p->len==n && sqlite3StrNICmp(p->zName, z, n)==0 ){ 17875897234Sdrh return p->tokenType; 17975897234Sdrh } 18075897234Sdrh } 18175897234Sdrh return TK_ID; 18275897234Sdrh } 18375897234Sdrh 18498808babSdrh 18598808babSdrh /* 186ba21256fSdrh ** If X is a character that can be used in an identifier and 187ba21256fSdrh ** X&0x80==0 then isIdChar[X] will be 1. If X&0x80==0x80 then 188ba21256fSdrh ** X is always an identifier character. (Hence all UTF-8 189ba21256fSdrh ** characters can be part of an identifier). isIdChar[X] will 190ba21256fSdrh ** be 0 for every character in the lower 128 ASCII characters 191ba21256fSdrh ** that cannot be used as part of an identifier. 19298808babSdrh ** 19398808babSdrh ** In this implementation, an identifier can be a string of 19498808babSdrh ** alphabetic characters, digits, and "_" plus any character 19598808babSdrh ** with the high-order bit set. The latter rule means that 19698808babSdrh ** any sequence of UTF-8 characters or characters taken from 19798808babSdrh ** an extended ISO8859 character set can form an identifier. 19898808babSdrh */ 19998808babSdrh static const char isIdChar[] = { 20098808babSdrh /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 20198808babSdrh 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 20298808babSdrh 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 20398808babSdrh 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 20498808babSdrh 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 20598808babSdrh 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 20698808babSdrh 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 20798808babSdrh 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 20898808babSdrh 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ 20998808babSdrh }; 21098808babSdrh 21198808babSdrh 21275897234Sdrh /* 21361b487d0Sdrh ** Return the length of the token that begins at z[0]. 21461b487d0Sdrh ** Store the token type in *tokenType before returning. 21575897234Sdrh */ 21698808babSdrh static int sqliteGetToken(const unsigned char *z, int *tokenType){ 21775897234Sdrh int i; 21875897234Sdrh switch( *z ){ 21930cab809Sdrh case ' ': case '\t': case '\n': case '\f': case '\r': { 22032eb7b47Sdrh for(i=1; isspace(z[i]); i++){} 22175897234Sdrh *tokenType = TK_SPACE; 22275897234Sdrh return i; 22375897234Sdrh } 22475897234Sdrh case '-': { 22575897234Sdrh if( z[1]=='-' ){ 22675897234Sdrh for(i=2; z[i] && z[i]!='\n'; i++){} 22775897234Sdrh *tokenType = TK_COMMENT; 22875897234Sdrh return i; 22975897234Sdrh } 23075897234Sdrh *tokenType = TK_MINUS; 23175897234Sdrh return 1; 23275897234Sdrh } 23375897234Sdrh case '(': { 23475897234Sdrh *tokenType = TK_LP; 23575897234Sdrh return 1; 23675897234Sdrh } 23775897234Sdrh case ')': { 23875897234Sdrh *tokenType = TK_RP; 23975897234Sdrh return 1; 24075897234Sdrh } 24175897234Sdrh case ';': { 24275897234Sdrh *tokenType = TK_SEMI; 24375897234Sdrh return 1; 24475897234Sdrh } 24575897234Sdrh case '+': { 24675897234Sdrh *tokenType = TK_PLUS; 24775897234Sdrh return 1; 24875897234Sdrh } 24975897234Sdrh case '*': { 25075897234Sdrh *tokenType = TK_STAR; 25175897234Sdrh return 1; 25275897234Sdrh } 25375897234Sdrh case '/': { 25466105a8eSdrh if( z[1]!='*' || z[2]==0 ){ 25575897234Sdrh *tokenType = TK_SLASH; 25675897234Sdrh return 1; 25775897234Sdrh } 25866105a8eSdrh for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){} 25966105a8eSdrh if( z[i] ) i++; 26066105a8eSdrh *tokenType = TK_COMMENT; 26166105a8eSdrh return i; 26266105a8eSdrh } 263bf4133cbSdrh case '%': { 264bf4133cbSdrh *tokenType = TK_REM; 265bf4133cbSdrh return 1; 266bf4133cbSdrh } 26775897234Sdrh case '=': { 26875897234Sdrh *tokenType = TK_EQ; 26975897234Sdrh return 1 + (z[1]=='='); 27075897234Sdrh } 27175897234Sdrh case '<': { 27275897234Sdrh if( z[1]=='=' ){ 27375897234Sdrh *tokenType = TK_LE; 27475897234Sdrh return 2; 27575897234Sdrh }else if( z[1]=='>' ){ 27675897234Sdrh *tokenType = TK_NE; 27775897234Sdrh return 2; 278bf4133cbSdrh }else if( z[1]=='<' ){ 279bf4133cbSdrh *tokenType = TK_LSHIFT; 280bf4133cbSdrh return 2; 28175897234Sdrh }else{ 28275897234Sdrh *tokenType = TK_LT; 28375897234Sdrh return 1; 28475897234Sdrh } 28575897234Sdrh } 28675897234Sdrh case '>': { 28775897234Sdrh if( z[1]=='=' ){ 28875897234Sdrh *tokenType = TK_GE; 28975897234Sdrh return 2; 290bf4133cbSdrh }else if( z[1]=='>' ){ 291bf4133cbSdrh *tokenType = TK_RSHIFT; 292bf4133cbSdrh return 2; 29375897234Sdrh }else{ 29475897234Sdrh *tokenType = TK_GT; 29575897234Sdrh return 1; 29675897234Sdrh } 29775897234Sdrh } 29875897234Sdrh case '!': { 29975897234Sdrh if( z[1]!='=' ){ 30075897234Sdrh *tokenType = TK_ILLEGAL; 301c837e709Sdrh return 2; 30275897234Sdrh }else{ 30375897234Sdrh *tokenType = TK_NE; 30475897234Sdrh return 2; 30575897234Sdrh } 30675897234Sdrh } 3070040077dSdrh case '|': { 3080040077dSdrh if( z[1]!='|' ){ 309bf4133cbSdrh *tokenType = TK_BITOR; 3100040077dSdrh return 1; 3110040077dSdrh }else{ 3120040077dSdrh *tokenType = TK_CONCAT; 3130040077dSdrh return 2; 3140040077dSdrh } 3150040077dSdrh } 31675897234Sdrh case ',': { 31775897234Sdrh *tokenType = TK_COMMA; 31875897234Sdrh return 1; 31975897234Sdrh } 320bf4133cbSdrh case '&': { 321bf4133cbSdrh *tokenType = TK_BITAND; 322bf4133cbSdrh return 1; 323bf4133cbSdrh } 324bf4133cbSdrh case '~': { 325bf4133cbSdrh *tokenType = TK_BITNOT; 326bf4133cbSdrh return 1; 327bf4133cbSdrh } 32875897234Sdrh case '\'': case '"': { 32975897234Sdrh int delim = z[0]; 33075897234Sdrh for(i=1; z[i]; i++){ 33175897234Sdrh if( z[i]==delim ){ 33275897234Sdrh if( z[i+1]==delim ){ 33375897234Sdrh i++; 33475897234Sdrh }else{ 33575897234Sdrh break; 33675897234Sdrh } 33775897234Sdrh } 33875897234Sdrh } 33975897234Sdrh if( z[i] ) i++; 34075897234Sdrh *tokenType = TK_STRING; 34175897234Sdrh return i; 34275897234Sdrh } 34375897234Sdrh case '.': { 34475897234Sdrh *tokenType = TK_DOT; 34575897234Sdrh return 1; 34675897234Sdrh } 34775897234Sdrh case '0': case '1': case '2': case '3': case '4': 34875897234Sdrh case '5': case '6': case '7': case '8': case '9': { 349c837e709Sdrh *tokenType = TK_INTEGER; 35032eb7b47Sdrh for(i=1; isdigit(z[i]); i++){} 351bb07e9a3Sdrh if( z[i]=='.' && isdigit(z[i+1]) ){ 352bb07e9a3Sdrh i += 2; 35332eb7b47Sdrh while( isdigit(z[i]) ){ i++; } 354c837e709Sdrh *tokenType = TK_FLOAT; 355c837e709Sdrh } 35675897234Sdrh if( (z[i]=='e' || z[i]=='E') && 35775897234Sdrh ( isdigit(z[i+1]) 35875897234Sdrh || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2])) 35975897234Sdrh ) 36075897234Sdrh ){ 36175897234Sdrh i += 2; 36232eb7b47Sdrh while( isdigit(z[i]) ){ i++; } 36375897234Sdrh *tokenType = TK_FLOAT; 36475897234Sdrh } 36575897234Sdrh return i; 36675897234Sdrh } 3672f4392ffSdrh case '[': { 3682f4392ffSdrh for(i=1; z[i] && z[i-1]!=']'; i++){} 3692f4392ffSdrh *tokenType = TK_ID; 3702f4392ffSdrh return i; 3712f4392ffSdrh } 3727c972decSdrh case '?': { 37350457896Sdrh *tokenType = TK_VARIABLE; 3747c972decSdrh return 1; 37550457896Sdrh } 376c572ef7fSdanielk1977 case 'x': case 'X': { 377c572ef7fSdanielk1977 if( z[1]=='\'' || z[1]=='"' ){ 3783fd0a736Sdanielk1977 int delim = z[1]; 3793fd0a736Sdanielk1977 *tokenType = TK_BLOB; 3803fd0a736Sdanielk1977 for(i=2; z[i]; i++){ 381c572ef7fSdanielk1977 if( z[i]==delim ){ 3823fd0a736Sdanielk1977 if( i%2 ) *tokenType = TK_ILLEGAL; 383c572ef7fSdanielk1977 break; 384c572ef7fSdanielk1977 } 3853fd0a736Sdanielk1977 if( !isxdigit(z[i]) ){ 3863fd0a736Sdanielk1977 *tokenType = TK_ILLEGAL; 3873fd0a736Sdanielk1977 return i; 3883fd0a736Sdanielk1977 } 389c572ef7fSdanielk1977 } 390c572ef7fSdanielk1977 if( z[i] ) i++; 391c572ef7fSdanielk1977 return i; 392c572ef7fSdanielk1977 } 393c572ef7fSdanielk1977 /* Otherwise fall through to the next case */ 394c572ef7fSdanielk1977 } 39598808babSdrh default: { 396ba21256fSdrh if( (*z&0x80)==0 && !isIdChar[*z] ){ 39798808babSdrh break; 39898808babSdrh } 399ba21256fSdrh for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){} 4004adee20fSdanielk1977 *tokenType = sqlite3KeywordCode((char*)z, i); 40175897234Sdrh return i; 40275897234Sdrh } 40375897234Sdrh } 40475897234Sdrh *tokenType = TK_ILLEGAL; 40575897234Sdrh return 1; 40675897234Sdrh } 40775897234Sdrh 40875897234Sdrh /* 40975897234Sdrh ** Run the parser on the given SQL string. The parser structure is 410b19a2bc6Sdrh ** passed in. An SQLITE_ status code is returned. If an error occurs 411b19a2bc6Sdrh ** and pzErrMsg!=NULL then an error message might be written into 412b19a2bc6Sdrh ** memory obtained from malloc() and *pzErrMsg made to point to that 413b19a2bc6Sdrh ** error message. Or maybe not. 41475897234Sdrh */ 4154adee20fSdanielk1977 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ 41675897234Sdrh int nErr = 0; 41775897234Sdrh int i; 41875897234Sdrh void *pEngine; 419b86ccfb2Sdrh int tokenType; 420b86ccfb2Sdrh int lastTokenParsed = -1; 4216d4abfbeSdrh sqlite *db = pParse->db; 4224adee20fSdanielk1977 extern void *sqlite3ParserAlloc(void*(*)(int)); 4234adee20fSdanielk1977 extern void sqlite3ParserFree(void*, void(*)(void*)); 4244adee20fSdanielk1977 extern int sqlite3Parser(void*, int, Token, Parse*); 42575897234Sdrh 4266d4abfbeSdrh db->flags &= ~SQLITE_Interrupt; 4274c504391Sdrh pParse->rc = SQLITE_OK; 42875897234Sdrh i = 0; 4294adee20fSdanielk1977 pEngine = sqlite3ParserAlloc((void*(*)(int))malloc); 43075897234Sdrh if( pEngine==0 ){ 4314adee20fSdanielk1977 sqlite3SetString(pzErrMsg, "out of memory", (char*)0); 43275897234Sdrh return 1; 43375897234Sdrh } 43432eb7b47Sdrh pParse->sLastToken.dyn = 0; 435326dce74Sdrh pParse->zTail = zSql; 4366f8a503dSdanielk1977 while( sqlite3_malloc_failed==0 && zSql[i]!=0 ){ 43732eb7b47Sdrh assert( i>=0 ); 43875897234Sdrh pParse->sLastToken.z = &zSql[i]; 43932eb7b47Sdrh assert( pParse->sLastToken.dyn==0 ); 44098808babSdrh pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType); 44175897234Sdrh i += pParse->sLastToken.n; 44275897234Sdrh switch( tokenType ){ 44375897234Sdrh case TK_SPACE: 44475897234Sdrh case TK_COMMENT: { 44532eb7b47Sdrh if( (db->flags & SQLITE_Interrupt)!=0 ){ 44632eb7b47Sdrh pParse->rc = SQLITE_INTERRUPT; 4474adee20fSdanielk1977 sqlite3SetString(pzErrMsg, "interrupt", (char*)0); 44832eb7b47Sdrh goto abort_parse; 44932eb7b47Sdrh } 45075897234Sdrh break; 45175897234Sdrh } 45232eb7b47Sdrh case TK_ILLEGAL: { 4534adee20fSdanielk1977 sqlite3SetNString(pzErrMsg, "unrecognized token: \"", -1, 454c837e709Sdrh pParse->sLastToken.z, pParse->sLastToken.n, "\"", 1, 0); 45575897234Sdrh nErr++; 456caec2f12Sdrh goto abort_parse; 45732eb7b47Sdrh } 458326dce74Sdrh case TK_SEMI: { 459326dce74Sdrh pParse->zTail = &zSql[i]; 460326dce74Sdrh /* Fall thru into the default case */ 461326dce74Sdrh } 46232eb7b47Sdrh default: { 4634adee20fSdanielk1977 sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); 464b86ccfb2Sdrh lastTokenParsed = tokenType; 465b86ccfb2Sdrh if( pParse->rc!=SQLITE_OK ){ 46632eb7b47Sdrh goto abort_parse; 46775897234Sdrh } 46875897234Sdrh break; 46975897234Sdrh } 47075897234Sdrh } 47132eb7b47Sdrh } 47232eb7b47Sdrh abort_parse: 473b86ccfb2Sdrh if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ 474b86ccfb2Sdrh if( lastTokenParsed!=TK_SEMI ){ 4754adee20fSdanielk1977 sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); 476326dce74Sdrh pParse->zTail = &zSql[i]; 47775897234Sdrh } 4784adee20fSdanielk1977 sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); 47975897234Sdrh } 4804adee20fSdanielk1977 sqlite3ParserFree(pEngine, free); 481b86ccfb2Sdrh if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ 4826f8a503dSdanielk1977 sqlite3SetString(&pParse->zErrMsg, sqlite3_error_string(pParse->rc), 48341743984Sdrh (char*)0); 484b86ccfb2Sdrh } 48575897234Sdrh if( pParse->zErrMsg ){ 486b86ccfb2Sdrh if( pzErrMsg && *pzErrMsg==0 ){ 48775897234Sdrh *pzErrMsg = pParse->zErrMsg; 48875897234Sdrh }else{ 48975897234Sdrh sqliteFree(pParse->zErrMsg); 49075897234Sdrh } 491b86ccfb2Sdrh pParse->zErrMsg = 0; 49275897234Sdrh if( !nErr ) nErr++; 49375897234Sdrh } 494826fb5a3Sdrh if( pParse->pVdbe && pParse->nErr>0 ){ 4954adee20fSdanielk1977 sqlite3VdbeDelete(pParse->pVdbe); 49675897234Sdrh pParse->pVdbe = 0; 49775897234Sdrh } 49875897234Sdrh if( pParse->pNewTable ){ 4994adee20fSdanielk1977 sqlite3DeleteTable(pParse->db, pParse->pNewTable); 50075897234Sdrh pParse->pNewTable = 0; 50175897234Sdrh } 502f0f258b1Sdrh if( pParse->pNewTrigger ){ 5034adee20fSdanielk1977 sqlite3DeleteTrigger(pParse->pNewTrigger); 504f0f258b1Sdrh pParse->pNewTrigger = 0; 505f0f258b1Sdrh } 506b86ccfb2Sdrh if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){ 5074c504391Sdrh pParse->rc = SQLITE_ERROR; 5084c504391Sdrh } 50975897234Sdrh return nErr; 51075897234Sdrh } 5117ad4334aSdrh 5127ad4334aSdrh /* 5136f8a503dSdanielk1977 ** Token types used by the sqlite3_complete() routine. See the header 5147ad4334aSdrh ** comments on that procedure for additional information. 5157ad4334aSdrh */ 5167ad4334aSdrh #define tkEXPLAIN 0 5177ad4334aSdrh #define tkCREATE 1 5187ad4334aSdrh #define tkTEMP 2 5197ad4334aSdrh #define tkTRIGGER 3 5207ad4334aSdrh #define tkEND 4 5217ad4334aSdrh #define tkSEMI 5 5227ad4334aSdrh #define tkWS 6 5237ad4334aSdrh #define tkOTHER 7 5247ad4334aSdrh 5257ad4334aSdrh /* 5267ad4334aSdrh ** Return TRUE if the given SQL string ends in a semicolon. 5277ad4334aSdrh ** 5287ad4334aSdrh ** Special handling is require for CREATE TRIGGER statements. 5297ad4334aSdrh ** Whenever the CREATE TRIGGER keywords are seen, the statement 5307ad4334aSdrh ** must end with ";END;". 5317ad4334aSdrh ** 5327ad4334aSdrh ** This implementation uses a state machine with 7 states: 5337ad4334aSdrh ** 5347ad4334aSdrh ** (0) START At the beginning or end of an SQL statement. This routine 5357ad4334aSdrh ** returns 1 if it ends in the START state and 0 if it ends 5367ad4334aSdrh ** in any other state. 5377ad4334aSdrh ** 5387ad4334aSdrh ** (1) EXPLAIN The keyword EXPLAIN has been seen at the beginning of 5397ad4334aSdrh ** a statement. 5407ad4334aSdrh ** 5417ad4334aSdrh ** (2) CREATE The keyword CREATE has been seen at the beginning of a 5427ad4334aSdrh ** statement, possibly preceeded by EXPLAIN and/or followed by 5437ad4334aSdrh ** TEMP or TEMPORARY 5447ad4334aSdrh ** 5457ad4334aSdrh ** (3) NORMAL We are in the middle of statement which ends with a single 5467ad4334aSdrh ** semicolon. 5477ad4334aSdrh ** 5487ad4334aSdrh ** (4) TRIGGER We are in the middle of a trigger definition that must be 5497ad4334aSdrh ** ended by a semicolon, the keyword END, and another semicolon. 5507ad4334aSdrh ** 5517ad4334aSdrh ** (5) SEMI We've seen the first semicolon in the ";END;" that occurs at 5527ad4334aSdrh ** the end of a trigger definition. 5537ad4334aSdrh ** 5547ad4334aSdrh ** (6) END We've seen the ";END" of the ";END;" that occurs at the end 5557ad4334aSdrh ** of a trigger difinition. 5567ad4334aSdrh ** 5577ad4334aSdrh ** Transitions between states above are determined by tokens extracted 5587ad4334aSdrh ** from the input. The following tokens are significant: 5597ad4334aSdrh ** 5607ad4334aSdrh ** (0) tkEXPLAIN The "explain" keyword. 5617ad4334aSdrh ** (1) tkCREATE The "create" keyword. 5627ad4334aSdrh ** (2) tkTEMP The "temp" or "temporary" keyword. 5637ad4334aSdrh ** (3) tkTRIGGER The "trigger" keyword. 5647ad4334aSdrh ** (4) tkEND The "end" keyword. 5657ad4334aSdrh ** (5) tkSEMI A semicolon. 5667ad4334aSdrh ** (6) tkWS Whitespace 5677ad4334aSdrh ** (7) tkOTHER Any other SQL token. 5687ad4334aSdrh ** 5697ad4334aSdrh ** Whitespace never causes a state transition and is always ignored. 5707ad4334aSdrh */ 5716f8a503dSdanielk1977 int sqlite3_complete(const char *zSql){ 5727ad4334aSdrh u8 state = 0; /* Current state, using numbers defined in header comment */ 5737ad4334aSdrh u8 token; /* Value of the next token */ 5747ad4334aSdrh 5757ad4334aSdrh /* The following matrix defines the transition from one state to another 5767ad4334aSdrh ** according to what token is seen. trans[state][token] returns the 5777ad4334aSdrh ** next state. 5787ad4334aSdrh */ 5797ad4334aSdrh static const u8 trans[7][8] = { 580e1e38c4dSdrh /* Token: */ 5817ad4334aSdrh /* State: ** EXPLAIN CREATE TEMP TRIGGER END SEMI WS OTHER */ 5827ad4334aSdrh /* 0 START: */ { 1, 2, 3, 3, 3, 0, 0, 3, }, 5837ad4334aSdrh /* 1 EXPLAIN: */ { 3, 2, 3, 3, 3, 0, 1, 3, }, 5847ad4334aSdrh /* 2 CREATE: */ { 3, 3, 2, 4, 3, 0, 2, 3, }, 5857ad4334aSdrh /* 3 NORMAL: */ { 3, 3, 3, 3, 3, 0, 3, 3, }, 5867ad4334aSdrh /* 4 TRIGGER: */ { 4, 4, 4, 4, 4, 5, 4, 4, }, 5877ad4334aSdrh /* 5 SEMI: */ { 4, 4, 4, 4, 6, 5, 5, 4, }, 5887ad4334aSdrh /* 6 END: */ { 4, 4, 4, 4, 4, 0, 6, 4, }, 5897ad4334aSdrh }; 5907ad4334aSdrh 5917ad4334aSdrh while( *zSql ){ 5927ad4334aSdrh switch( *zSql ){ 5937ad4334aSdrh case ';': { /* A semicolon */ 5947ad4334aSdrh token = tkSEMI; 5957ad4334aSdrh break; 5967ad4334aSdrh } 5977ad4334aSdrh case ' ': 5987ad4334aSdrh case '\r': 5997ad4334aSdrh case '\t': 6007ad4334aSdrh case '\n': 6017ad4334aSdrh case '\f': { /* White space is ignored */ 6027ad4334aSdrh token = tkWS; 6037ad4334aSdrh break; 6047ad4334aSdrh } 6057ad4334aSdrh case '/': { /* C-style comments */ 6067ad4334aSdrh if( zSql[1]!='*' ){ 6077ad4334aSdrh token = tkOTHER; 6087ad4334aSdrh break; 6097ad4334aSdrh } 6107ad4334aSdrh zSql += 2; 6117ad4334aSdrh while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } 6127ad4334aSdrh if( zSql[0]==0 ) return 0; 6137ad4334aSdrh zSql++; 6147ad4334aSdrh token = tkWS; 6157ad4334aSdrh break; 6167ad4334aSdrh } 6177ad4334aSdrh case '-': { /* SQL-style comments from "--" to end of line */ 6187ad4334aSdrh if( zSql[1]!='-' ){ 6197ad4334aSdrh token = tkOTHER; 6207ad4334aSdrh break; 6217ad4334aSdrh } 6227ad4334aSdrh while( *zSql && *zSql!='\n' ){ zSql++; } 6237ad4334aSdrh if( *zSql==0 ) return state==0; 6247ad4334aSdrh token = tkWS; 6257ad4334aSdrh break; 6267ad4334aSdrh } 6277ad4334aSdrh case '[': { /* Microsoft-style identifiers in [...] */ 6287ad4334aSdrh zSql++; 6297ad4334aSdrh while( *zSql && *zSql!=']' ){ zSql++; } 6307ad4334aSdrh if( *zSql==0 ) return 0; 6317ad4334aSdrh token = tkOTHER; 6327ad4334aSdrh break; 6337ad4334aSdrh } 6347ad4334aSdrh case '"': /* single- and double-quoted strings */ 6357ad4334aSdrh case '\'': { 6367ad4334aSdrh int c = *zSql; 6377ad4334aSdrh zSql++; 6387ad4334aSdrh while( *zSql && *zSql!=c ){ zSql++; } 6397ad4334aSdrh if( *zSql==0 ) return 0; 6407ad4334aSdrh token = tkOTHER; 6417ad4334aSdrh break; 6427ad4334aSdrh } 6437ad4334aSdrh default: { 644e1e38c4dSdrh if( isIdChar[(u8)*zSql] ){ 6457ad4334aSdrh /* Keywords and unquoted identifiers */ 6467ad4334aSdrh int nId; 647e1e38c4dSdrh for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){} 6487ad4334aSdrh switch( *zSql ){ 6497ad4334aSdrh case 'c': case 'C': { 6504adee20fSdanielk1977 if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ 6517ad4334aSdrh token = tkCREATE; 6527ad4334aSdrh }else{ 6537ad4334aSdrh token = tkOTHER; 6547ad4334aSdrh } 6557ad4334aSdrh break; 6567ad4334aSdrh } 6577ad4334aSdrh case 't': case 'T': { 6584adee20fSdanielk1977 if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ 6597ad4334aSdrh token = tkTRIGGER; 6604adee20fSdanielk1977 }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ 6617ad4334aSdrh token = tkTEMP; 6624adee20fSdanielk1977 }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ 6637ad4334aSdrh token = tkTEMP; 6647ad4334aSdrh }else{ 6657ad4334aSdrh token = tkOTHER; 6667ad4334aSdrh } 6677ad4334aSdrh break; 6687ad4334aSdrh } 6697ad4334aSdrh case 'e': case 'E': { 6704adee20fSdanielk1977 if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ 6717ad4334aSdrh token = tkEND; 6724adee20fSdanielk1977 }else if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ 6737ad4334aSdrh token = tkEXPLAIN; 6747ad4334aSdrh }else{ 6757ad4334aSdrh token = tkOTHER; 6767ad4334aSdrh } 6777ad4334aSdrh break; 6787ad4334aSdrh } 6797ad4334aSdrh default: { 6807ad4334aSdrh token = tkOTHER; 6817ad4334aSdrh break; 6827ad4334aSdrh } 6837ad4334aSdrh } 6847ad4334aSdrh zSql += nId-1; 6857ad4334aSdrh }else{ 6867ad4334aSdrh /* Operators and special symbols */ 6877ad4334aSdrh token = tkOTHER; 6887ad4334aSdrh } 6897ad4334aSdrh break; 6907ad4334aSdrh } 6917ad4334aSdrh } 6927ad4334aSdrh state = trans[state][token]; 6937ad4334aSdrh zSql++; 6947ad4334aSdrh } 6957ad4334aSdrh return state==0; 6967ad4334aSdrh } 69761de0d1bSdanielk1977 69861de0d1bSdanielk1977 /* 69961de0d1bSdanielk1977 ** This routine is the same as the sqlite3_complete() routine described 70061de0d1bSdanielk1977 ** above, except that the parameter is required to be UTF-16 encoded, not 70161de0d1bSdanielk1977 ** UTF-8. 70261de0d1bSdanielk1977 */ 70361de0d1bSdanielk1977 int sqlite3_complete16(const void *zSql){ 70461de0d1bSdanielk1977 int rc; 705*9c054830Sdrh char *zSql8 = sqlite3utf16to8(zSql, -1, SQLITE_BIGENDIAN); 70661de0d1bSdanielk1977 if( !zSql8 ) return 0; 70761de0d1bSdanielk1977 rc = sqlite3_complete(zSql8); 70861de0d1bSdanielk1977 sqliteFree(zSql8); 70961de0d1bSdanielk1977 return rc; 71061de0d1bSdanielk1977 } 711