1 /* 2 ** 2001 September 15 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ************************************************************************* 12 ** An tokenizer for SQL 13 ** 14 ** This file contains C code that splits an SQL input string up into 15 ** individual tokens and sends those tokens one-by-one over to the 16 ** parser for analysis. 17 ** 18 ** $Id: tokenize.c,v 1.152 2008/09/01 15:52:11 drh Exp $ 19 */ 20 #include "sqliteInt.h" 21 #include <ctype.h> 22 #include <stdlib.h> 23 24 /* 25 ** The charMap() macro maps alphabetic characters into their 26 ** lower-case ASCII equivalent. On ASCII machines, this is just 27 ** an upper-to-lower case map. On EBCDIC machines we also need 28 ** to adjust the encoding. Only alphabetic characters and underscores 29 ** need to be translated. 30 */ 31 #ifdef SQLITE_ASCII 32 # define charMap(X) sqlite3UpperToLower[(unsigned char)X] 33 #endif 34 #ifdef SQLITE_EBCDIC 35 # define charMap(X) ebcdicToAscii[(unsigned char)X] 36 const unsigned char ebcdicToAscii[] = { 37 /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ 38 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 39 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 40 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 41 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ 42 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ 43 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ 44 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ 45 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ 46 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ 47 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ 48 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ 49 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ 50 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ 51 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ 52 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ 53 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ 54 }; 55 #endif 56 57 /* 58 ** The sqlite3KeywordCode function looks up an identifier to determine if 59 ** it is a keyword. If it is a keyword, the token code of that keyword is 60 ** returned. If the input is not a keyword, TK_ID is returned. 61 ** 62 ** The implementation of this routine was generated by a program, 63 ** mkkeywordhash.h, located in the tool subdirectory of the distribution. 64 ** The output of the mkkeywordhash.c program is written into a file 65 ** named keywordhash.h and then included into this source file by 66 ** the #include below. 67 */ 68 #include "keywordhash.h" 69 70 71 /* 72 ** If X is a character that can be used in an identifier then 73 ** IdChar(X) will be true. Otherwise it is false. 74 ** 75 ** For ASCII, any character with the high-order bit set is 76 ** allowed in an identifier. For 7-bit characters, 77 ** sqlite3IsIdChar[X] must be 1. 78 ** 79 ** For EBCDIC, the rules are more complex but have the same 80 ** end result. 81 ** 82 ** Ticket #1066. the SQL standard does not allow '$' in the 83 ** middle of identfiers. But many SQL implementations do. 84 ** SQLite will allow '$' in identifiers for compatibility. 85 ** But the feature is undocumented. 86 */ 87 #ifdef SQLITE_ASCII 88 const char sqlite3IsAsciiIdChar[] = { 89 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 90 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 91 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 92 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 93 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 94 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 95 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ 96 }; 97 #define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20])) 98 #endif 99 #ifdef SQLITE_EBCDIC 100 const char sqlite3IsEbcdicIdChar[] = { 101 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 102 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ 103 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ 104 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ 105 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ 106 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ 107 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ 108 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ 109 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ 110 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ 111 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ 112 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ 113 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ 114 }; 115 #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) 116 #endif 117 118 119 /* 120 ** Return the length of the token that begins at z[0]. 121 ** Store the token type in *tokenType before returning. 122 */ 123 int sqlite3GetToken(const unsigned char *z, int *tokenType){ 124 int i, c; 125 switch( *z ){ 126 case ' ': case '\t': case '\n': case '\f': case '\r': { 127 for(i=1; isspace(z[i]); i++){} 128 *tokenType = TK_SPACE; 129 return i; 130 } 131 case '-': { 132 if( z[1]=='-' ){ 133 for(i=2; (c=z[i])!=0 && c!='\n'; i++){} 134 *tokenType = TK_SPACE; 135 return i; 136 } 137 *tokenType = TK_MINUS; 138 return 1; 139 } 140 case '(': { 141 *tokenType = TK_LP; 142 return 1; 143 } 144 case ')': { 145 *tokenType = TK_RP; 146 return 1; 147 } 148 case ';': { 149 *tokenType = TK_SEMI; 150 return 1; 151 } 152 case '+': { 153 *tokenType = TK_PLUS; 154 return 1; 155 } 156 case '*': { 157 *tokenType = TK_STAR; 158 return 1; 159 } 160 case '/': { 161 if( z[1]!='*' || z[2]==0 ){ 162 *tokenType = TK_SLASH; 163 return 1; 164 } 165 for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} 166 if( c ) i++; 167 *tokenType = TK_SPACE; 168 return i; 169 } 170 case '%': { 171 *tokenType = TK_REM; 172 return 1; 173 } 174 case '=': { 175 *tokenType = TK_EQ; 176 return 1 + (z[1]=='='); 177 } 178 case '<': { 179 if( (c=z[1])=='=' ){ 180 *tokenType = TK_LE; 181 return 2; 182 }else if( c=='>' ){ 183 *tokenType = TK_NE; 184 return 2; 185 }else if( c=='<' ){ 186 *tokenType = TK_LSHIFT; 187 return 2; 188 }else{ 189 *tokenType = TK_LT; 190 return 1; 191 } 192 } 193 case '>': { 194 if( (c=z[1])=='=' ){ 195 *tokenType = TK_GE; 196 return 2; 197 }else if( c=='>' ){ 198 *tokenType = TK_RSHIFT; 199 return 2; 200 }else{ 201 *tokenType = TK_GT; 202 return 1; 203 } 204 } 205 case '!': { 206 if( z[1]!='=' ){ 207 *tokenType = TK_ILLEGAL; 208 return 2; 209 }else{ 210 *tokenType = TK_NE; 211 return 2; 212 } 213 } 214 case '|': { 215 if( z[1]!='|' ){ 216 *tokenType = TK_BITOR; 217 return 1; 218 }else{ 219 *tokenType = TK_CONCAT; 220 return 2; 221 } 222 } 223 case ',': { 224 *tokenType = TK_COMMA; 225 return 1; 226 } 227 case '&': { 228 *tokenType = TK_BITAND; 229 return 1; 230 } 231 case '~': { 232 *tokenType = TK_BITNOT; 233 return 1; 234 } 235 case '`': 236 case '\'': 237 case '"': { 238 int delim = z[0]; 239 for(i=1; (c=z[i])!=0; i++){ 240 if( c==delim ){ 241 if( z[i+1]==delim ){ 242 i++; 243 }else{ 244 break; 245 } 246 } 247 } 248 if( c=='\'' ){ 249 *tokenType = TK_STRING; 250 return i+1; 251 }else if( c!=0 ){ 252 *tokenType = TK_ID; 253 return i+1; 254 }else{ 255 *tokenType = TK_ILLEGAL; 256 return i; 257 } 258 } 259 case '.': { 260 #ifndef SQLITE_OMIT_FLOATING_POINT 261 if( !isdigit(z[1]) ) 262 #endif 263 { 264 *tokenType = TK_DOT; 265 return 1; 266 } 267 /* If the next character is a digit, this is a floating point 268 ** number that begins with ".". Fall thru into the next case */ 269 } 270 case '0': case '1': case '2': case '3': case '4': 271 case '5': case '6': case '7': case '8': case '9': { 272 *tokenType = TK_INTEGER; 273 for(i=0; isdigit(z[i]); i++){} 274 #ifndef SQLITE_OMIT_FLOATING_POINT 275 if( z[i]=='.' ){ 276 i++; 277 while( isdigit(z[i]) ){ i++; } 278 *tokenType = TK_FLOAT; 279 } 280 if( (z[i]=='e' || z[i]=='E') && 281 ( isdigit(z[i+1]) 282 || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2])) 283 ) 284 ){ 285 i += 2; 286 while( isdigit(z[i]) ){ i++; } 287 *tokenType = TK_FLOAT; 288 } 289 #endif 290 while( IdChar(z[i]) ){ 291 *tokenType = TK_ILLEGAL; 292 i++; 293 } 294 return i; 295 } 296 case '[': { 297 for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} 298 *tokenType = c==']' ? TK_ID : TK_ILLEGAL; 299 return i; 300 } 301 case '?': { 302 *tokenType = TK_VARIABLE; 303 for(i=1; isdigit(z[i]); i++){} 304 return i; 305 } 306 case '#': { 307 for(i=1; isdigit(z[i]); i++){} 308 if( i>1 ){ 309 /* Parameters of the form #NNN (where NNN is a number) are used 310 ** internally by sqlite3NestedParse. */ 311 *tokenType = TK_REGISTER; 312 return i; 313 } 314 /* Fall through into the next case if the '#' is not followed by 315 ** a digit. Try to match #AAAA where AAAA is a parameter name. */ 316 } 317 #ifndef SQLITE_OMIT_TCL_VARIABLE 318 case '$': 319 #endif 320 case '@': /* For compatibility with MS SQL Server */ 321 case ':': { 322 int n = 0; 323 *tokenType = TK_VARIABLE; 324 for(i=1; (c=z[i])!=0; i++){ 325 if( IdChar(c) ){ 326 n++; 327 #ifndef SQLITE_OMIT_TCL_VARIABLE 328 }else if( c=='(' && n>0 ){ 329 do{ 330 i++; 331 }while( (c=z[i])!=0 && !isspace(c) && c!=')' ); 332 if( c==')' ){ 333 i++; 334 }else{ 335 *tokenType = TK_ILLEGAL; 336 } 337 break; 338 }else if( c==':' && z[i+1]==':' ){ 339 i++; 340 #endif 341 }else{ 342 break; 343 } 344 } 345 if( n==0 ) *tokenType = TK_ILLEGAL; 346 return i; 347 } 348 #ifndef SQLITE_OMIT_BLOB_LITERAL 349 case 'x': case 'X': { 350 if( z[1]=='\'' ){ 351 *tokenType = TK_BLOB; 352 for(i=2; (c=z[i])!=0 && c!='\''; i++){ 353 if( !isxdigit(c) ){ 354 *tokenType = TK_ILLEGAL; 355 } 356 } 357 if( i%2 || !c ) *tokenType = TK_ILLEGAL; 358 if( c ) i++; 359 return i; 360 } 361 /* Otherwise fall through to the next case */ 362 } 363 #endif 364 default: { 365 if( !IdChar(*z) ){ 366 break; 367 } 368 for(i=1; IdChar(z[i]); i++){} 369 *tokenType = keywordCode((char*)z, i); 370 return i; 371 } 372 } 373 *tokenType = TK_ILLEGAL; 374 return 1; 375 } 376 377 /* 378 ** Run the parser on the given SQL string. The parser structure is 379 ** passed in. An SQLITE_ status code is returned. If an error occurs 380 ** then an and attempt is made to write an error message into 381 ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that 382 ** error message. 383 */ 384 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ 385 int nErr = 0; 386 int i; 387 void *pEngine; 388 int tokenType; 389 int lastTokenParsed = -1; 390 sqlite3 *db = pParse->db; 391 int mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; 392 393 if( db->activeVdbeCnt==0 ){ 394 db->u1.isInterrupted = 0; 395 } 396 pParse->rc = SQLITE_OK; 397 pParse->zTail = pParse->zSql = zSql; 398 i = 0; 399 assert( pzErrMsg!=0 ); 400 pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); 401 if( pEngine==0 ){ 402 db->mallocFailed = 1; 403 return SQLITE_NOMEM; 404 } 405 assert( pParse->sLastToken.dyn==0 ); 406 assert( pParse->pNewTable==0 ); 407 assert( pParse->pNewTrigger==0 ); 408 assert( pParse->nVar==0 ); 409 assert( pParse->nVarExpr==0 ); 410 assert( pParse->nVarExprAlloc==0 ); 411 assert( pParse->apVarExpr==0 ); 412 while( !db->mallocFailed && zSql[i]!=0 ){ 413 assert( i>=0 ); 414 pParse->sLastToken.z = (u8*)&zSql[i]; 415 assert( pParse->sLastToken.dyn==0 ); 416 pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); 417 i += pParse->sLastToken.n; 418 if( i>mxSqlLen ){ 419 pParse->rc = SQLITE_TOOBIG; 420 break; 421 } 422 switch( tokenType ){ 423 case TK_SPACE: { 424 if( db->u1.isInterrupted ){ 425 pParse->rc = SQLITE_INTERRUPT; 426 sqlite3SetString(pzErrMsg, db, "interrupt"); 427 goto abort_parse; 428 } 429 break; 430 } 431 case TK_ILLEGAL: { 432 sqlite3DbFree(db, *pzErrMsg); 433 *pzErrMsg = sqlite3MPrintf(db, "unrecognized token: \"%T\"", 434 &pParse->sLastToken); 435 nErr++; 436 goto abort_parse; 437 } 438 case TK_SEMI: { 439 pParse->zTail = &zSql[i]; 440 /* Fall thru into the default case */ 441 } 442 default: { 443 sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); 444 lastTokenParsed = tokenType; 445 if( pParse->rc!=SQLITE_OK ){ 446 goto abort_parse; 447 } 448 break; 449 } 450 } 451 } 452 abort_parse: 453 if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){ 454 if( lastTokenParsed!=TK_SEMI ){ 455 sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); 456 pParse->zTail = &zSql[i]; 457 } 458 sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); 459 } 460 #ifdef YYTRACKMAXSTACKDEPTH 461 sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, 462 sqlite3ParserStackPeak(pEngine) 463 ); 464 #endif /* YYDEBUG */ 465 sqlite3ParserFree(pEngine, sqlite3_free); 466 if( db->mallocFailed ){ 467 pParse->rc = SQLITE_NOMEM; 468 } 469 if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ 470 sqlite3SetString(&pParse->zErrMsg, db, "%s", sqlite3ErrStr(pParse->rc)); 471 } 472 if( pParse->zErrMsg ){ 473 if( *pzErrMsg==0 ){ 474 *pzErrMsg = pParse->zErrMsg; 475 }else{ 476 sqlite3DbFree(db, pParse->zErrMsg); 477 } 478 pParse->zErrMsg = 0; 479 nErr++; 480 } 481 if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ 482 sqlite3VdbeDelete(pParse->pVdbe); 483 pParse->pVdbe = 0; 484 } 485 #ifndef SQLITE_OMIT_SHARED_CACHE 486 if( pParse->nested==0 ){ 487 sqlite3DbFree(db, pParse->aTableLock); 488 pParse->aTableLock = 0; 489 pParse->nTableLock = 0; 490 } 491 #endif 492 #ifndef SQLITE_OMIT_VIRTUALTABLE 493 sqlite3DbFree(db, pParse->apVtabLock); 494 #endif 495 496 if( !IN_DECLARE_VTAB ){ 497 /* If the pParse->declareVtab flag is set, do not delete any table 498 ** structure built up in pParse->pNewTable. The calling code (see vtab.c) 499 ** will take responsibility for freeing the Table structure. 500 */ 501 sqlite3DeleteTable(pParse->pNewTable); 502 } 503 504 sqlite3DeleteTrigger(db, pParse->pNewTrigger); 505 sqlite3DbFree(db, pParse->apVarExpr); 506 sqlite3DbFree(db, pParse->aAlias); 507 while( pParse->pZombieTab ){ 508 Table *p = pParse->pZombieTab; 509 pParse->pZombieTab = p->pNextZombie; 510 sqlite3DeleteTable(p); 511 } 512 if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){ 513 pParse->rc = SQLITE_ERROR; 514 } 515 return nErr; 516 } 517