1 /* 2 ** 2012 April 10 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ************************************************************************* 12 ** 13 ** This module implements the spellfix1 VIRTUAL TABLE that can be used 14 ** to search a large vocabulary for close matches. See separate 15 ** documentation (http://www.sqlite.org/spellfix1.html) for details. 16 */ 17 #include "sqlite3ext.h" 18 SQLITE_EXTENSION_INIT1 19 20 #ifndef SQLITE_AMALGAMATION 21 # if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 22 # define NDEBUG 1 23 # endif 24 # if defined(NDEBUG) && defined(SQLITE_DEBUG) 25 # undef NDEBUG 26 # endif 27 # include <string.h> 28 # include <stdio.h> 29 # include <stdlib.h> 30 # include <assert.h> 31 # define ALWAYS(X) 1 32 # define NEVER(X) 0 33 typedef unsigned char u8; 34 typedef unsigned short u16; 35 #endif 36 #include <ctype.h> 37 38 #ifndef SQLITE_OMIT_VIRTUALTABLE 39 40 /* 41 ** Character classes for ASCII characters: 42 ** 43 ** 0 '' Silent letters: H W 44 ** 1 'A' Any vowel: A E I O U (Y) 45 ** 2 'B' A bilabeal stop or fricative: B F P V W 46 ** 3 'C' Other fricatives or back stops: C G J K Q S X Z 47 ** 4 'D' Alveolar stops: D T 48 ** 5 'H' Letter H at the beginning of a word 49 ** 6 'L' Glide: L 50 ** 7 'R' Semivowel: R 51 ** 8 'M' Nasals: M N 52 ** 9 'Y' Letter Y at the beginning of a word. 53 ** 10 '9' Digits: 0 1 2 3 4 5 6 7 8 9 54 ** 11 ' ' White space 55 ** 12 '?' Other. 56 */ 57 #define CCLASS_SILENT 0 58 #define CCLASS_VOWEL 1 59 #define CCLASS_B 2 60 #define CCLASS_C 3 61 #define CCLASS_D 4 62 #define CCLASS_H 5 63 #define CCLASS_L 6 64 #define CCLASS_R 7 65 #define CCLASS_M 8 66 #define CCLASS_Y 9 67 #define CCLASS_DIGIT 10 68 #define CCLASS_SPACE 11 69 #define CCLASS_OTHER 12 70 71 /* 72 ** The following table gives the character class for non-initial ASCII 73 ** characters. 74 */ 75 static const unsigned char midClass[] = { 76 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 77 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 78 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 79 /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 80 /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, 81 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 82 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 83 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 84 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 85 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 86 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, 87 /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, 88 /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, 89 /* ' */ CCLASS_SILENT, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, 90 /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, 91 /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, 92 /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, 93 /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, 94 /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, 95 /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, 96 /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, 97 /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, 98 /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, 99 /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, 100 /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, 101 /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, 102 /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, 103 /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, 104 /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, 105 /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_VOWEL, 106 /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, 107 /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, 108 /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, 109 /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, 110 /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, 111 /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, 112 /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, 113 /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, 114 /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, 115 /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, 116 /* x */ CCLASS_C, /* y */ CCLASS_VOWEL, /* z */ CCLASS_C, 117 /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, 118 /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, 119 }; 120 /* 121 ** This tables gives the character class for ASCII characters that form the 122 ** initial character of a word. The only difference from midClass is with 123 ** the letters H, W, and Y. 124 */ 125 static const unsigned char initClass[] = { 126 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 127 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 128 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 129 /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 130 /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, 131 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 132 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 133 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 134 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 135 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 136 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, 137 /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, 138 /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, 139 /* ' */ CCLASS_OTHER, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, 140 /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, 141 /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, 142 /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, 143 /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, 144 /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, 145 /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, 146 /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, 147 /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, 148 /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, 149 /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, 150 /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, 151 /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, 152 /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, 153 /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, 154 /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, 155 /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_Y, 156 /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, 157 /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, 158 /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, 159 /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, 160 /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, 161 /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, 162 /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, 163 /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, 164 /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, 165 /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, 166 /* x */ CCLASS_C, /* y */ CCLASS_Y, /* z */ CCLASS_C, 167 /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, 168 /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, 169 }; 170 171 /* 172 ** Mapping from the character class number (0-13) to a symbol for each 173 ** character class. Note that initClass[] can be used to map the class 174 ** symbol back into the class number. 175 */ 176 static const unsigned char className[] = ".ABCDHLRMY9 ?"; 177 178 /* 179 ** Generate a "phonetic hash" from a string of ASCII characters 180 ** in zIn[0..nIn-1]. 181 ** 182 ** * Map characters by character class as defined above. 183 ** * Omit double-letters 184 ** * Omit vowels beside R and L 185 ** * Omit T when followed by CH 186 ** * Omit W when followed by R 187 ** * Omit D when followed by J or G 188 ** * Omit K in KN or G in GN at the beginning of a word 189 ** 190 ** Space to hold the result is obtained from sqlite3_malloc() 191 ** 192 ** Return NULL if memory allocation fails. 193 */ 194 static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){ 195 unsigned char *zOut = sqlite3_malloc64( nIn + 1 ); 196 int i; 197 int nOut = 0; 198 char cPrev = 0x77; 199 char cPrevX = 0x77; 200 const unsigned char *aClass = initClass; 201 202 if( zOut==0 ) return 0; 203 if( nIn>2 ){ 204 switch( zIn[0] ){ 205 case 'g': 206 case 'k': { 207 if( zIn[1]=='n' ){ zIn++; nIn--; } 208 break; 209 } 210 } 211 } 212 for(i=0; i<nIn; i++){ 213 unsigned char c = zIn[i]; 214 if( i+1<nIn ){ 215 if( c=='w' && zIn[i+1]=='r' ) continue; 216 if( c=='d' && (zIn[i+1]=='j' || zIn[i+1]=='g') ) continue; 217 if( i+2<nIn ){ 218 if( c=='t' && zIn[i+1]=='c' && zIn[i+2]=='h' ) continue; 219 } 220 } 221 c = aClass[c&0x7f]; 222 if( c==CCLASS_SPACE ) continue; 223 if( c==CCLASS_OTHER && cPrev!=CCLASS_DIGIT ) continue; 224 aClass = midClass; 225 if( c==CCLASS_VOWEL && (cPrevX==CCLASS_R || cPrevX==CCLASS_L) ){ 226 continue; /* No vowels beside L or R */ 227 } 228 if( (c==CCLASS_R || c==CCLASS_L) && cPrevX==CCLASS_VOWEL ){ 229 nOut--; /* No vowels beside L or R */ 230 } 231 cPrev = c; 232 if( c==CCLASS_SILENT ) continue; 233 cPrevX = c; 234 c = className[c]; 235 assert( nOut>=0 ); 236 if( nOut==0 || c!=zOut[nOut-1] ) zOut[nOut++] = c; 237 } 238 zOut[nOut] = 0; 239 return zOut; 240 } 241 242 /* 243 ** This is an SQL function wrapper around phoneticHash(). See 244 ** the description of phoneticHash() for additional information. 245 */ 246 static void phoneticHashSqlFunc( 247 sqlite3_context *context, 248 int argc, 249 sqlite3_value **argv 250 ){ 251 const unsigned char *zIn; 252 unsigned char *zOut; 253 254 zIn = sqlite3_value_text(argv[0]); 255 if( zIn==0 ) return; 256 zOut = phoneticHash(zIn, sqlite3_value_bytes(argv[0])); 257 if( zOut==0 ){ 258 sqlite3_result_error_nomem(context); 259 }else{ 260 sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); 261 } 262 } 263 264 /* 265 ** Return the character class number for a character given its 266 ** context. 267 */ 268 static char characterClass(char cPrev, char c){ 269 return cPrev==0 ? initClass[c&0x7f] : midClass[c&0x7f]; 270 } 271 272 /* 273 ** Return the cost of inserting or deleting character c immediately 274 ** following character cPrev. If cPrev==0, that means c is the first 275 ** character of the word. 276 */ 277 static int insertOrDeleteCost(char cPrev, char c, char cNext){ 278 char classC = characterClass(cPrev, c); 279 char classCprev; 280 281 if( classC==CCLASS_SILENT ){ 282 /* Insert or delete "silent" characters such as H or W */ 283 return 1; 284 } 285 if( cPrev==c ){ 286 /* Repeated characters, or miss a repeat */ 287 return 10; 288 } 289 if( classC==CCLASS_VOWEL && (cPrev=='r' || cNext=='r') ){ 290 return 20; /* Insert a vowel before or after 'r' */ 291 } 292 classCprev = characterClass(cPrev, cPrev); 293 if( classC==classCprev ){ 294 if( classC==CCLASS_VOWEL ){ 295 /* Remove or add a new vowel to a vowel cluster */ 296 return 15; 297 }else{ 298 /* Remove or add a consonant not in the same class */ 299 return 50; 300 } 301 } 302 303 /* any other character insertion or deletion */ 304 return 100; 305 } 306 307 /* 308 ** Divide the insertion cost by this factor when appending to the 309 ** end of the word. 310 */ 311 #define FINAL_INS_COST_DIV 4 312 313 /* 314 ** Return the cost of substituting cTo in place of cFrom assuming 315 ** the previous character is cPrev. If cPrev==0 then cTo is the first 316 ** character of the word. 317 */ 318 static int substituteCost(char cPrev, char cFrom, char cTo){ 319 char classFrom, classTo; 320 if( cFrom==cTo ){ 321 /* Exact match */ 322 return 0; 323 } 324 if( cFrom==(cTo^0x20) && ((cTo>='A' && cTo<='Z') || (cTo>='a' && cTo<='z')) ){ 325 /* differ only in case */ 326 return 0; 327 } 328 classFrom = characterClass(cPrev, cFrom); 329 classTo = characterClass(cPrev, cTo); 330 if( classFrom==classTo ){ 331 /* Same character class */ 332 return 40; 333 } 334 if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y 335 && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ 336 /* Convert from one consonant to another, but in a different class */ 337 return 75; 338 } 339 /* Any other subsitution */ 340 return 100; 341 } 342 343 /* 344 ** Given two strings zA and zB which are pure ASCII, return the cost 345 ** of transforming zA into zB. If zA ends with '*' assume that it is 346 ** a prefix of zB and give only minimal penalty for extra characters 347 ** on the end of zB. 348 ** 349 ** Smaller numbers mean a closer match. 350 ** 351 ** Negative values indicate an error: 352 ** -1 One of the inputs is NULL 353 ** -2 Non-ASCII characters on input 354 ** -3 Unable to allocate memory 355 ** 356 ** If pnMatch is not NULL, then *pnMatch is set to the number of bytes 357 ** of zB that matched the pattern in zA. If zA does not end with a '*', 358 ** then this value is always the number of bytes in zB (i.e. strlen(zB)). 359 ** If zA does end in a '*', then it is the number of bytes in the prefix 360 ** of zB that was deemed to match zA. 361 */ 362 static int editdist1(const char *zA, const char *zB, int *pnMatch){ 363 int nA, nB; /* Number of characters in zA[] and zB[] */ 364 int xA, xB; /* Loop counters for zA[] and zB[] */ 365 char cA = 0, cB; /* Current character of zA and zB */ 366 char cAprev, cBprev; /* Previous character of zA and zB */ 367 char cAnext, cBnext; /* Next character in zA and zB */ 368 int d; /* North-west cost value */ 369 int dc = 0; /* North-west character value */ 370 int res; /* Final result */ 371 int *m; /* The cost matrix */ 372 char *cx; /* Corresponding character values */ 373 int *toFree = 0; /* Malloced space */ 374 int nMatch = 0; 375 int mStack[60+15]; /* Stack space to use if not too much is needed */ 376 377 /* Early out if either input is NULL */ 378 if( zA==0 || zB==0 ) return -1; 379 380 /* Skip any common prefix */ 381 while( zA[0] && zA[0]==zB[0] ){ dc = zA[0]; zA++; zB++; nMatch++; } 382 if( pnMatch ) *pnMatch = nMatch; 383 if( zA[0]==0 && zB[0]==0 ) return 0; 384 385 #if 0 386 printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' '); 387 #endif 388 389 /* Verify input strings and measure their lengths */ 390 for(nA=0; zA[nA]; nA++){ 391 if( zA[nA]&0x80 ) return -2; 392 } 393 for(nB=0; zB[nB]; nB++){ 394 if( zB[nB]&0x80 ) return -2; 395 } 396 397 /* Special processing if either string is empty */ 398 if( nA==0 ){ 399 cBprev = (char)dc; 400 for(xB=res=0; (cB = zB[xB])!=0; xB++){ 401 res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV; 402 cBprev = cB; 403 } 404 return res; 405 } 406 if( nB==0 ){ 407 cAprev = (char)dc; 408 for(xA=res=0; (cA = zA[xA])!=0; xA++){ 409 res += insertOrDeleteCost(cAprev, cA, zA[xA+1]); 410 cAprev = cA; 411 } 412 return res; 413 } 414 415 /* A is a prefix of B */ 416 if( zA[0]=='*' && zA[1]==0 ) return 0; 417 418 /* Allocate and initialize the Wagner matrix */ 419 if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ 420 m = mStack; 421 }else{ 422 m = toFree = sqlite3_malloc64( (nB+1)*5*sizeof(m[0])/4 ); 423 if( m==0 ) return -3; 424 } 425 cx = (char*)&m[nB+1]; 426 427 /* Compute the Wagner edit distance */ 428 m[0] = 0; 429 cx[0] = (char)dc; 430 cBprev = (char)dc; 431 for(xB=1; xB<=nB; xB++){ 432 cBnext = zB[xB]; 433 cB = zB[xB-1]; 434 cx[xB] = cB; 435 m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext); 436 cBprev = cB; 437 } 438 cAprev = (char)dc; 439 for(xA=1; xA<=nA; xA++){ 440 int lastA = (xA==nA); 441 cA = zA[xA-1]; 442 cAnext = zA[xA]; 443 if( cA=='*' && lastA ) break; 444 d = m[0]; 445 dc = cx[0]; 446 m[0] = d + insertOrDeleteCost(cAprev, cA, cAnext); 447 cBprev = 0; 448 for(xB=1; xB<=nB; xB++){ 449 int totalCost, insCost, delCost, subCost, ncx; 450 cB = zB[xB-1]; 451 cBnext = zB[xB]; 452 453 /* Cost to insert cB */ 454 insCost = insertOrDeleteCost(cx[xB-1], cB, cBnext); 455 if( lastA ) insCost /= FINAL_INS_COST_DIV; 456 457 /* Cost to delete cA */ 458 delCost = insertOrDeleteCost(cx[xB], cA, cBnext); 459 460 /* Cost to substitute cA->cB */ 461 subCost = substituteCost(cx[xB-1], cA, cB); 462 463 /* Best cost */ 464 totalCost = insCost + m[xB-1]; 465 ncx = cB; 466 if( (delCost + m[xB])<totalCost ){ 467 totalCost = delCost + m[xB]; 468 ncx = cA; 469 } 470 if( (subCost + d)<totalCost ){ 471 totalCost = subCost + d; 472 } 473 474 #if 0 475 printf("%d,%d d=%4d u=%4d r=%4d dc=%c cA=%c cB=%c" 476 " ins=%4d del=%4d sub=%4d t=%4d ncx=%c\n", 477 xA, xB, d, m[xB], m[xB-1], dc?dc:' ', cA, cB, 478 insCost, delCost, subCost, totalCost, ncx?ncx:' '); 479 #endif 480 481 /* Update the matrix */ 482 d = m[xB]; 483 dc = cx[xB]; 484 m[xB] = totalCost; 485 cx[xB] = (char)ncx; 486 cBprev = cB; 487 } 488 cAprev = cA; 489 } 490 491 /* Free the wagner matrix and return the result */ 492 if( cA=='*' ){ 493 res = m[1]; 494 for(xB=1; xB<=nB; xB++){ 495 if( m[xB]<res ){ 496 res = m[xB]; 497 if( pnMatch ) *pnMatch = xB+nMatch; 498 } 499 } 500 }else{ 501 res = m[nB]; 502 /* In the current implementation, pnMatch is always NULL if zA does 503 ** not end in "*" */ 504 assert( pnMatch==0 ); 505 } 506 sqlite3_free(toFree); 507 return res; 508 } 509 510 /* 511 ** Function: editdist(A,B) 512 ** 513 ** Return the cost of transforming string A into string B. Both strings 514 ** must be pure ASCII text. If A ends with '*' then it is assumed to be 515 ** a prefix of B and extra characters on the end of B have minimal additional 516 ** cost. 517 */ 518 static void editdistSqlFunc( 519 sqlite3_context *context, 520 int argc, 521 sqlite3_value **argv 522 ){ 523 int res = editdist1( 524 (const char*)sqlite3_value_text(argv[0]), 525 (const char*)sqlite3_value_text(argv[1]), 526 0); 527 if( res<0 ){ 528 if( res==(-3) ){ 529 sqlite3_result_error_nomem(context); 530 }else if( res==(-2) ){ 531 sqlite3_result_error(context, "non-ASCII input to editdist()", -1); 532 }else{ 533 sqlite3_result_error(context, "NULL input to editdist()", -1); 534 } 535 }else{ 536 sqlite3_result_int(context, res); 537 } 538 } 539 540 /* End of the fixed-cost edit distance implementation 541 ****************************************************************************** 542 ***************************************************************************** 543 ** Begin: Configurable cost unicode edit distance routines 544 */ 545 /* Forward declaration of structures */ 546 typedef struct EditDist3Cost EditDist3Cost; 547 typedef struct EditDist3Config EditDist3Config; 548 typedef struct EditDist3Point EditDist3Point; 549 typedef struct EditDist3From EditDist3From; 550 typedef struct EditDist3FromString EditDist3FromString; 551 typedef struct EditDist3To EditDist3To; 552 typedef struct EditDist3ToString EditDist3ToString; 553 typedef struct EditDist3Lang EditDist3Lang; 554 555 556 /* 557 ** An entry in the edit cost table 558 */ 559 struct EditDist3Cost { 560 EditDist3Cost *pNext; /* Next cost element */ 561 u8 nFrom; /* Number of bytes in aFrom */ 562 u8 nTo; /* Number of bytes in aTo */ 563 u16 iCost; /* Cost of this transformation */ 564 char a[4] ; /* FROM string followed by TO string */ 565 /* Additional TO and FROM string bytes appended as necessary */ 566 }; 567 568 /* 569 ** Edit costs for a particular language ID 570 */ 571 struct EditDist3Lang { 572 int iLang; /* Language ID */ 573 int iInsCost; /* Default insertion cost */ 574 int iDelCost; /* Default deletion cost */ 575 int iSubCost; /* Default substitution cost */ 576 EditDist3Cost *pCost; /* Costs */ 577 }; 578 579 580 /* 581 ** The default EditDist3Lang object, with default costs. 582 */ 583 static const EditDist3Lang editDist3Lang = { 0, 100, 100, 150, 0 }; 584 585 /* 586 ** Complete configuration 587 */ 588 struct EditDist3Config { 589 int nLang; /* Number of language IDs. Size of a[] */ 590 EditDist3Lang *a; /* One for each distinct language ID */ 591 }; 592 593 /* 594 ** Extra information about each character in the FROM string. 595 */ 596 struct EditDist3From { 597 int nSubst; /* Number of substitution cost entries */ 598 int nDel; /* Number of deletion cost entries */ 599 int nByte; /* Number of bytes in this character */ 600 EditDist3Cost **apSubst; /* Array of substitution costs for this element */ 601 EditDist3Cost **apDel; /* Array of deletion cost entries */ 602 }; 603 604 /* 605 ** A precompiled FROM string. 606 * 607 ** In the common case we expect the FROM string to be reused multiple times. 608 ** In other words, the common case will be to measure the edit distance 609 ** from a single origin string to multiple target strings. 610 */ 611 struct EditDist3FromString { 612 char *z; /* The complete text of the FROM string */ 613 int n; /* Number of characters in the FROM string */ 614 int isPrefix; /* True if ends with '*' character */ 615 EditDist3From *a; /* Extra info about each char of the FROM string */ 616 }; 617 618 /* 619 ** Extra information about each character in the TO string. 620 */ 621 struct EditDist3To { 622 int nIns; /* Number of insertion cost entries */ 623 int nByte; /* Number of bytes in this character */ 624 EditDist3Cost **apIns; /* Array of deletion cost entries */ 625 }; 626 627 /* 628 ** A precompiled FROM string 629 */ 630 struct EditDist3ToString { 631 char *z; /* The complete text of the TO string */ 632 int n; /* Number of characters in the TO string */ 633 EditDist3To *a; /* Extra info about each char of the TO string */ 634 }; 635 636 /* 637 ** Clear or delete an instance of the object that records all edit-distance 638 ** weights. 639 */ 640 static void editDist3ConfigClear(EditDist3Config *p){ 641 int i; 642 if( p==0 ) return; 643 for(i=0; i<p->nLang; i++){ 644 EditDist3Cost *pCost, *pNext; 645 pCost = p->a[i].pCost; 646 while( pCost ){ 647 pNext = pCost->pNext; 648 sqlite3_free(pCost); 649 pCost = pNext; 650 } 651 } 652 sqlite3_free(p->a); 653 memset(p, 0, sizeof(*p)); 654 } 655 static void editDist3ConfigDelete(void *pIn){ 656 EditDist3Config *p = (EditDist3Config*)pIn; 657 editDist3ConfigClear(p); 658 sqlite3_free(p); 659 } 660 661 /* 662 ** Load all edit-distance weights from a table. 663 */ 664 static int editDist3ConfigLoad( 665 EditDist3Config *p, /* The edit distance configuration to load */ 666 sqlite3 *db, /* Load from this database */ 667 const char *zTable /* Name of the table from which to load */ 668 ){ 669 sqlite3_stmt *pStmt; 670 int rc, rc2; 671 char *zSql; 672 int iLangPrev = -9999; 673 EditDist3Lang *pLang = 0; 674 675 zSql = sqlite3_mprintf("SELECT iLang, cFrom, cTo, iCost" 676 " FROM \"%w\" WHERE iLang>=0 ORDER BY iLang", zTable); 677 if( zSql==0 ) return SQLITE_NOMEM; 678 rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); 679 sqlite3_free(zSql); 680 if( rc ) return rc; 681 editDist3ConfigClear(p); 682 while( sqlite3_step(pStmt)==SQLITE_ROW ){ 683 int iLang = sqlite3_column_int(pStmt, 0); 684 const char *zFrom = (const char*)sqlite3_column_text(pStmt, 1); 685 int nFrom = zFrom ? sqlite3_column_bytes(pStmt, 1) : 0; 686 const char *zTo = (const char*)sqlite3_column_text(pStmt, 2); 687 int nTo = zTo ? sqlite3_column_bytes(pStmt, 2) : 0; 688 int iCost = sqlite3_column_int(pStmt, 3); 689 690 assert( zFrom!=0 || nFrom==0 ); 691 assert( zTo!=0 || nTo==0 ); 692 if( nFrom>100 || nTo>100 ) continue; 693 if( iCost<0 ) continue; 694 if( iCost>10000 ) continue; /* Costs above 10K are considered infinite */ 695 if( pLang==0 || iLang!=iLangPrev ){ 696 EditDist3Lang *pNew; 697 pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0])); 698 if( pNew==0 ){ rc = SQLITE_NOMEM; break; } 699 p->a = pNew; 700 pLang = &p->a[p->nLang]; 701 p->nLang++; 702 pLang->iLang = iLang; 703 pLang->iInsCost = 100; 704 pLang->iDelCost = 100; 705 pLang->iSubCost = 150; 706 pLang->pCost = 0; 707 iLangPrev = iLang; 708 } 709 if( nFrom==1 && zFrom[0]=='?' && nTo==0 ){ 710 pLang->iDelCost = iCost; 711 }else if( nFrom==0 && nTo==1 && zTo[0]=='?' ){ 712 pLang->iInsCost = iCost; 713 }else if( nFrom==1 && nTo==1 && zFrom[0]=='?' && zTo[0]=='?' ){ 714 pLang->iSubCost = iCost; 715 }else{ 716 EditDist3Cost *pCost; 717 int nExtra = nFrom + nTo - 4; 718 if( nExtra<0 ) nExtra = 0; 719 pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra ); 720 if( pCost==0 ){ rc = SQLITE_NOMEM; break; } 721 pCost->nFrom = (u8)nFrom; 722 pCost->nTo = (u8)nTo; 723 pCost->iCost = (u16)iCost; 724 memcpy(pCost->a, zFrom, nFrom); 725 memcpy(pCost->a + nFrom, zTo, nTo); 726 pCost->pNext = pLang->pCost; 727 pLang->pCost = pCost; 728 } 729 } 730 rc2 = sqlite3_finalize(pStmt); 731 if( rc==SQLITE_OK ) rc = rc2; 732 return rc; 733 } 734 735 /* 736 ** Return the length (in bytes) of a utf-8 character. Or return a maximum 737 ** of N. 738 */ 739 static int utf8Len(unsigned char c, int N){ 740 int len = 1; 741 if( c>0x7f ){ 742 if( (c&0xe0)==0xc0 ){ 743 len = 2; 744 }else if( (c&0xf0)==0xe0 ){ 745 len = 3; 746 }else{ 747 len = 4; 748 } 749 } 750 if( len>N ) len = N; 751 return len; 752 } 753 754 /* 755 ** Return TRUE (non-zero) if the To side of the given cost matches 756 ** the given string. 757 */ 758 static int matchTo(EditDist3Cost *p, const char *z, int n){ 759 if( p->nTo>n ) return 0; 760 if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0; 761 return 1; 762 } 763 764 /* 765 ** Return TRUE (non-zero) if the From side of the given cost matches 766 ** the given string. 767 */ 768 static int matchFrom(EditDist3Cost *p, const char *z, int n){ 769 assert( p->nFrom<=n ); 770 if( strncmp(p->a, z, p->nFrom)!=0 ) return 0; 771 return 1; 772 } 773 774 /* 775 ** Return TRUE (non-zero) of the next FROM character and the next TO 776 ** character are the same. 777 */ 778 static int matchFromTo( 779 EditDist3FromString *pStr, /* Left hand string */ 780 int n1, /* Index of comparison character on the left */ 781 const char *z2, /* Right-handl comparison character */ 782 int n2 /* Bytes remaining in z2[] */ 783 ){ 784 int b1 = pStr->a[n1].nByte; 785 if( b1>n2 ) return 0; 786 if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0; 787 return 1; 788 } 789 790 /* 791 ** Delete an EditDist3FromString objecct 792 */ 793 static void editDist3FromStringDelete(EditDist3FromString *p){ 794 int i; 795 if( p ){ 796 for(i=0; i<p->n; i++){ 797 sqlite3_free(p->a[i].apDel); 798 sqlite3_free(p->a[i].apSubst); 799 } 800 sqlite3_free(p); 801 } 802 } 803 804 /* 805 ** Create a EditDist3FromString object. 806 */ 807 static EditDist3FromString *editDist3FromStringNew( 808 const EditDist3Lang *pLang, 809 const char *z, 810 int n 811 ){ 812 EditDist3FromString *pStr; 813 EditDist3Cost *p; 814 int i; 815 816 if( z==0 ) return 0; 817 if( n<0 ) n = (int)strlen(z); 818 pStr = sqlite3_malloc64( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); 819 if( pStr==0 ) return 0; 820 pStr->a = (EditDist3From*)&pStr[1]; 821 memset(pStr->a, 0, sizeof(pStr->a[0])*n); 822 pStr->n = n; 823 pStr->z = (char*)&pStr->a[n]; 824 memcpy(pStr->z, z, n+1); 825 if( n && z[n-1]=='*' ){ 826 pStr->isPrefix = 1; 827 n--; 828 pStr->n--; 829 pStr->z[n] = 0; 830 }else{ 831 pStr->isPrefix = 0; 832 } 833 834 for(i=0; i<n; i++){ 835 EditDist3From *pFrom = &pStr->a[i]; 836 memset(pFrom, 0, sizeof(*pFrom)); 837 pFrom->nByte = utf8Len((unsigned char)z[i], n-i); 838 for(p=pLang->pCost; p; p=p->pNext){ 839 EditDist3Cost **apNew; 840 if( i+p->nFrom>n ) continue; 841 if( matchFrom(p, z+i, n-i)==0 ) continue; 842 if( p->nTo==0 ){ 843 apNew = sqlite3_realloc64(pFrom->apDel, 844 sizeof(*apNew)*(pFrom->nDel+1)); 845 if( apNew==0 ) break; 846 pFrom->apDel = apNew; 847 apNew[pFrom->nDel++] = p; 848 }else{ 849 apNew = sqlite3_realloc64(pFrom->apSubst, 850 sizeof(*apNew)*(pFrom->nSubst+1)); 851 if( apNew==0 ) break; 852 pFrom->apSubst = apNew; 853 apNew[pFrom->nSubst++] = p; 854 } 855 } 856 if( p ){ 857 editDist3FromStringDelete(pStr); 858 pStr = 0; 859 break; 860 } 861 } 862 return pStr; 863 } 864 865 /* 866 ** Update entry m[i] such that it is the minimum of its current value 867 ** and m[j]+iCost. 868 */ 869 static void updateCost( 870 unsigned int *m, 871 int i, 872 int j, 873 int iCost 874 ){ 875 unsigned int b; 876 assert( iCost>=0 ); 877 assert( iCost<10000 ); 878 b = m[j] + iCost; 879 if( b<m[i] ) m[i] = b; 880 } 881 882 /* 883 ** How much stack space (int bytes) to use for Wagner matrix in 884 ** editDist3Core(). If more space than this is required, the entire 885 ** matrix is taken from the heap. To reduce the load on the memory 886 ** allocator, make this value as large as practical for the 887 ** architecture in use. 888 */ 889 #ifndef SQLITE_SPELLFIX_STACKALLOC_SZ 890 # define SQLITE_SPELLFIX_STACKALLOC_SZ (1024) 891 #endif 892 893 /* Compute the edit distance between two strings. 894 ** 895 ** If an error occurs, return a negative number which is the error code. 896 ** 897 ** If pnMatch is not NULL, then *pnMatch is set to the number of characters 898 ** (not bytes) in z2 that matched the search pattern in *pFrom. If pFrom does 899 ** not contain the pattern for a prefix-search, then this is always the number 900 ** of characters in z2. If pFrom does contain a prefix search pattern, then 901 ** it is the number of characters in the prefix of z2 that was deemed to 902 ** match pFrom. 903 */ 904 static int editDist3Core( 905 EditDist3FromString *pFrom, /* The FROM string */ 906 const char *z2, /* The TO string */ 907 int n2, /* Length of the TO string */ 908 const EditDist3Lang *pLang, /* Edit weights for a particular language ID */ 909 int *pnMatch /* OUT: Characters in matched prefix */ 910 ){ 911 int k, n; 912 int i1, b1; 913 int i2, b2; 914 EditDist3FromString f = *pFrom; 915 EditDist3To *a2; 916 unsigned int *m; 917 unsigned int *pToFree; 918 int szRow; 919 EditDist3Cost *p; 920 int res; 921 sqlite3_uint64 nByte; 922 unsigned int stackSpace[SQLITE_SPELLFIX_STACKALLOC_SZ/sizeof(unsigned int)]; 923 924 /* allocate the Wagner matrix and the aTo[] array for the TO string */ 925 n = (f.n+1)*(n2+1); 926 n = (n+1)&~1; 927 nByte = n*sizeof(m[0]) + sizeof(a2[0])*n2; 928 if( nByte<=sizeof(stackSpace) ){ 929 m = stackSpace; 930 pToFree = 0; 931 }else{ 932 m = pToFree = sqlite3_malloc64( nByte ); 933 if( m==0 ) return -1; /* Out of memory */ 934 } 935 a2 = (EditDist3To*)&m[n]; 936 memset(a2, 0, sizeof(a2[0])*n2); 937 938 /* Fill in the a1[] matrix for all characters of the TO string */ 939 for(i2=0; i2<n2; i2++){ 940 a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2); 941 for(p=pLang->pCost; p; p=p->pNext){ 942 EditDist3Cost **apNew; 943 if( p->nFrom>0 ) continue; 944 if( i2+p->nTo>n2 ) continue; 945 if( matchTo(p, z2+i2, n2-i2)==0 ) continue; 946 a2[i2].nIns++; 947 apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); 948 if( apNew==0 ){ 949 res = -1; /* Out of memory */ 950 goto editDist3Abort; 951 } 952 a2[i2].apIns = apNew; 953 a2[i2].apIns[a2[i2].nIns-1] = p; 954 } 955 } 956 957 /* Prepare to compute the minimum edit distance */ 958 szRow = f.n+1; 959 memset(m, 0x01, (n2+1)*szRow*sizeof(m[0])); 960 m[0] = 0; 961 962 /* First fill in the top-row of the matrix with FROM deletion costs */ 963 for(i1=0; i1<f.n; i1 += b1){ 964 b1 = f.a[i1].nByte; 965 updateCost(m, i1+b1, i1, pLang->iDelCost); 966 for(k=0; k<f.a[i1].nDel; k++){ 967 p = f.a[i1].apDel[k]; 968 updateCost(m, i1+p->nFrom, i1, p->iCost); 969 } 970 } 971 972 /* Fill in all subsequent rows, top-to-bottom, left-to-right */ 973 for(i2=0; i2<n2; i2 += b2){ 974 int rx; /* Starting index for current row */ 975 int rxp; /* Starting index for previous row */ 976 b2 = a2[i2].nByte; 977 rx = szRow*(i2+b2); 978 rxp = szRow*i2; 979 updateCost(m, rx, rxp, pLang->iInsCost); 980 for(k=0; k<a2[i2].nIns; k++){ 981 p = a2[i2].apIns[k]; 982 updateCost(m, szRow*(i2+p->nTo), rxp, p->iCost); 983 } 984 for(i1=0; i1<f.n; i1+=b1){ 985 int cx; /* Index of current cell */ 986 int cxp; /* Index of cell immediately to the left */ 987 int cxd; /* Index of cell to the left and one row above */ 988 int cxu; /* Index of cell immediately above */ 989 b1 = f.a[i1].nByte; 990 cxp = rx + i1; 991 cx = cxp + b1; 992 cxd = rxp + i1; 993 cxu = cxd + b1; 994 updateCost(m, cx, cxp, pLang->iDelCost); 995 for(k=0; k<f.a[i1].nDel; k++){ 996 p = f.a[i1].apDel[k]; 997 updateCost(m, cxp+p->nFrom, cxp, p->iCost); 998 } 999 updateCost(m, cx, cxu, pLang->iInsCost); 1000 if( matchFromTo(&f, i1, z2+i2, n2-i2) ){ 1001 updateCost(m, cx, cxd, 0); 1002 } 1003 updateCost(m, cx, cxd, pLang->iSubCost); 1004 for(k=0; k<f.a[i1].nSubst; k++){ 1005 p = f.a[i1].apSubst[k]; 1006 if( matchTo(p, z2+i2, n2-i2) ){ 1007 updateCost(m, cxd+p->nFrom+szRow*p->nTo, cxd, p->iCost); 1008 } 1009 } 1010 } 1011 } 1012 1013 #if 0 /* Enable for debugging */ 1014 printf(" ^"); 1015 for(i1=0; i1<f.n; i1++) printf(" %c-%2x", f.z[i1], f.z[i1]&0xff); 1016 printf("\n ^:"); 1017 for(i1=0; i1<szRow; i1++){ 1018 int v = m[i1]; 1019 if( v>9999 ) printf(" ****"); 1020 else printf(" %4d", v); 1021 } 1022 printf("\n"); 1023 for(i2=0; i2<n2; i2++){ 1024 printf("%c-%02x:", z2[i2], z2[i2]&0xff); 1025 for(i1=0; i1<szRow; i1++){ 1026 int v = m[(i2+1)*szRow+i1]; 1027 if( v>9999 ) printf(" ****"); 1028 else printf(" %4d", v); 1029 } 1030 printf("\n"); 1031 } 1032 #endif 1033 1034 /* Free memory allocations and return the result */ 1035 res = (int)m[szRow*(n2+1)-1]; 1036 n = n2; 1037 if( f.isPrefix ){ 1038 for(i2=1; i2<=n2; i2++){ 1039 int b = m[szRow*i2-1]; 1040 if( b<=res ){ 1041 res = b; 1042 n = i2 - 1; 1043 } 1044 } 1045 } 1046 if( pnMatch ){ 1047 int nExtra = 0; 1048 for(k=0; k<n; k++){ 1049 if( (z2[k] & 0xc0)==0x80 ) nExtra++; 1050 } 1051 *pnMatch = n - nExtra; 1052 } 1053 1054 editDist3Abort: 1055 for(i2=0; i2<n2; i2++) sqlite3_free(a2[i2].apIns); 1056 sqlite3_free(pToFree); 1057 return res; 1058 } 1059 1060 /* 1061 ** Get an appropriate EditDist3Lang object. 1062 */ 1063 static const EditDist3Lang *editDist3FindLang( 1064 EditDist3Config *pConfig, 1065 int iLang 1066 ){ 1067 int i; 1068 for(i=0; i<pConfig->nLang; i++){ 1069 if( pConfig->a[i].iLang==iLang ) return &pConfig->a[i]; 1070 } 1071 return &editDist3Lang; 1072 } 1073 1074 /* 1075 ** Function: editdist3(A,B,iLang) 1076 ** editdist3(tablename) 1077 ** 1078 ** Return the cost of transforming string A into string B using edit 1079 ** weights for iLang. 1080 ** 1081 ** The second form loads edit weights into memory from a table. 1082 */ 1083 static void editDist3SqlFunc( 1084 sqlite3_context *context, 1085 int argc, 1086 sqlite3_value **argv 1087 ){ 1088 EditDist3Config *pConfig = (EditDist3Config*)sqlite3_user_data(context); 1089 sqlite3 *db = sqlite3_context_db_handle(context); 1090 int rc; 1091 if( argc==1 ){ 1092 const char *zTable = (const char*)sqlite3_value_text(argv[0]); 1093 rc = editDist3ConfigLoad(pConfig, db, zTable); 1094 if( rc ) sqlite3_result_error_code(context, rc); 1095 }else{ 1096 const char *zA = (const char*)sqlite3_value_text(argv[0]); 1097 const char *zB = (const char*)sqlite3_value_text(argv[1]); 1098 int nA = sqlite3_value_bytes(argv[0]); 1099 int nB = sqlite3_value_bytes(argv[1]); 1100 int iLang = argc==3 ? sqlite3_value_int(argv[2]) : 0; 1101 const EditDist3Lang *pLang = editDist3FindLang(pConfig, iLang); 1102 EditDist3FromString *pFrom; 1103 int dist; 1104 1105 pFrom = editDist3FromStringNew(pLang, zA, nA); 1106 if( pFrom==0 ){ 1107 sqlite3_result_error_nomem(context); 1108 return; 1109 } 1110 dist = editDist3Core(pFrom, zB, nB, pLang, 0); 1111 editDist3FromStringDelete(pFrom); 1112 if( dist==(-1) ){ 1113 sqlite3_result_error_nomem(context); 1114 }else{ 1115 sqlite3_result_int(context, dist); 1116 } 1117 } 1118 } 1119 1120 /* 1121 ** Register the editDist3 function with SQLite 1122 */ 1123 static int editDist3Install(sqlite3 *db){ 1124 int rc; 1125 EditDist3Config *pConfig = sqlite3_malloc64( sizeof(*pConfig) ); 1126 if( pConfig==0 ) return SQLITE_NOMEM; 1127 memset(pConfig, 0, sizeof(*pConfig)); 1128 rc = sqlite3_create_function_v2(db, "editdist3", 1129 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1130 editDist3SqlFunc, 0, 0, 0); 1131 if( rc==SQLITE_OK ){ 1132 rc = sqlite3_create_function_v2(db, "editdist3", 1133 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1134 editDist3SqlFunc, 0, 0, 0); 1135 } 1136 if( rc==SQLITE_OK ){ 1137 rc = sqlite3_create_function_v2(db, "editdist3", 1138 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1139 editDist3SqlFunc, 0, 0, editDist3ConfigDelete); 1140 }else{ 1141 sqlite3_free(pConfig); 1142 } 1143 return rc; 1144 } 1145 /* End configurable cost unicode edit distance routines 1146 ****************************************************************************** 1147 ****************************************************************************** 1148 ** Begin transliterate unicode-to-ascii implementation 1149 */ 1150 1151 #if !SQLITE_AMALGAMATION 1152 /* 1153 ** This lookup table is used to help decode the first byte of 1154 ** a multi-byte UTF8 character. 1155 */ 1156 static const unsigned char sqlite3Utf8Trans1[] = { 1157 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1158 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 1159 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 1160 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 1161 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1162 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 1163 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1164 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, 1165 }; 1166 #endif 1167 1168 /* 1169 ** Return the value of the first UTF-8 character in the string. 1170 */ 1171 static int utf8Read(const unsigned char *z, int n, int *pSize){ 1172 int c, i; 1173 1174 /* All callers to this routine (in the current implementation) 1175 ** always have n>0. */ 1176 if( NEVER(n==0) ){ 1177 c = i = 0; 1178 }else{ 1179 c = z[0]; 1180 i = 1; 1181 if( c>=0xc0 ){ 1182 c = sqlite3Utf8Trans1[c-0xc0]; 1183 while( i<n && (z[i] & 0xc0)==0x80 ){ 1184 c = (c<<6) + (0x3f & z[i++]); 1185 } 1186 } 1187 } 1188 *pSize = i; 1189 return c; 1190 } 1191 1192 /* 1193 ** Return the number of characters in the utf-8 string in the nIn byte 1194 ** buffer pointed to by zIn. 1195 */ 1196 static int utf8Charlen(const char *zIn, int nIn){ 1197 int i; 1198 int nChar = 0; 1199 for(i=0; i<nIn; nChar++){ 1200 int sz; 1201 utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); 1202 i += sz; 1203 } 1204 return nChar; 1205 } 1206 1207 typedef struct Transliteration Transliteration; 1208 struct Transliteration { 1209 unsigned short int cFrom; 1210 unsigned char cTo0, cTo1, cTo2, cTo3; 1211 }; 1212 1213 /* 1214 ** Table of translations from unicode characters into ASCII. 1215 */ 1216 static const Transliteration translit[] = { 1217 { 0x00A0, 0x20, 0x00, 0x00, 0x00 }, /* to */ 1218 { 0x00B5, 0x75, 0x00, 0x00, 0x00 }, /* µ to u */ 1219 { 0x00C0, 0x41, 0x00, 0x00, 0x00 }, /* À to A */ 1220 { 0x00C1, 0x41, 0x00, 0x00, 0x00 }, /* Á to A */ 1221 { 0x00C2, 0x41, 0x00, 0x00, 0x00 }, /*  to A */ 1222 { 0x00C3, 0x41, 0x00, 0x00, 0x00 }, /* à to A */ 1223 { 0x00C4, 0x41, 0x65, 0x00, 0x00 }, /* Ä to Ae */ 1224 { 0x00C5, 0x41, 0x61, 0x00, 0x00 }, /* Å to Aa */ 1225 { 0x00C6, 0x41, 0x45, 0x00, 0x00 }, /* Æ to AE */ 1226 { 0x00C7, 0x43, 0x00, 0x00, 0x00 }, /* Ç to C */ 1227 { 0x00C8, 0x45, 0x00, 0x00, 0x00 }, /* È to E */ 1228 { 0x00C9, 0x45, 0x00, 0x00, 0x00 }, /* É to E */ 1229 { 0x00CA, 0x45, 0x00, 0x00, 0x00 }, /* Ê to E */ 1230 { 0x00CB, 0x45, 0x00, 0x00, 0x00 }, /* Ë to E */ 1231 { 0x00CC, 0x49, 0x00, 0x00, 0x00 }, /* Ì to I */ 1232 { 0x00CD, 0x49, 0x00, 0x00, 0x00 }, /* Í to I */ 1233 { 0x00CE, 0x49, 0x00, 0x00, 0x00 }, /* Î to I */ 1234 { 0x00CF, 0x49, 0x00, 0x00, 0x00 }, /* Ï to I */ 1235 { 0x00D0, 0x44, 0x00, 0x00, 0x00 }, /* Ð to D */ 1236 { 0x00D1, 0x4E, 0x00, 0x00, 0x00 }, /* Ñ to N */ 1237 { 0x00D2, 0x4F, 0x00, 0x00, 0x00 }, /* Ò to O */ 1238 { 0x00D3, 0x4F, 0x00, 0x00, 0x00 }, /* Ó to O */ 1239 { 0x00D4, 0x4F, 0x00, 0x00, 0x00 }, /* Ô to O */ 1240 { 0x00D5, 0x4F, 0x00, 0x00, 0x00 }, /* Õ to O */ 1241 { 0x00D6, 0x4F, 0x65, 0x00, 0x00 }, /* Ö to Oe */ 1242 { 0x00D7, 0x78, 0x00, 0x00, 0x00 }, /* × to x */ 1243 { 0x00D8, 0x4F, 0x00, 0x00, 0x00 }, /* Ø to O */ 1244 { 0x00D9, 0x55, 0x00, 0x00, 0x00 }, /* Ù to U */ 1245 { 0x00DA, 0x55, 0x00, 0x00, 0x00 }, /* Ú to U */ 1246 { 0x00DB, 0x55, 0x00, 0x00, 0x00 }, /* Û to U */ 1247 { 0x00DC, 0x55, 0x65, 0x00, 0x00 }, /* Ü to Ue */ 1248 { 0x00DD, 0x59, 0x00, 0x00, 0x00 }, /* Ý to Y */ 1249 { 0x00DE, 0x54, 0x68, 0x00, 0x00 }, /* Þ to Th */ 1250 { 0x00DF, 0x73, 0x73, 0x00, 0x00 }, /* ß to ss */ 1251 { 0x00E0, 0x61, 0x00, 0x00, 0x00 }, /* à to a */ 1252 { 0x00E1, 0x61, 0x00, 0x00, 0x00 }, /* á to a */ 1253 { 0x00E2, 0x61, 0x00, 0x00, 0x00 }, /* â to a */ 1254 { 0x00E3, 0x61, 0x00, 0x00, 0x00 }, /* ã to a */ 1255 { 0x00E4, 0x61, 0x65, 0x00, 0x00 }, /* ä to ae */ 1256 { 0x00E5, 0x61, 0x61, 0x00, 0x00 }, /* å to aa */ 1257 { 0x00E6, 0x61, 0x65, 0x00, 0x00 }, /* æ to ae */ 1258 { 0x00E7, 0x63, 0x00, 0x00, 0x00 }, /* ç to c */ 1259 { 0x00E8, 0x65, 0x00, 0x00, 0x00 }, /* è to e */ 1260 { 0x00E9, 0x65, 0x00, 0x00, 0x00 }, /* é to e */ 1261 { 0x00EA, 0x65, 0x00, 0x00, 0x00 }, /* ê to e */ 1262 { 0x00EB, 0x65, 0x00, 0x00, 0x00 }, /* ë to e */ 1263 { 0x00EC, 0x69, 0x00, 0x00, 0x00 }, /* ì to i */ 1264 { 0x00ED, 0x69, 0x00, 0x00, 0x00 }, /* í to i */ 1265 { 0x00EE, 0x69, 0x00, 0x00, 0x00 }, /* î to i */ 1266 { 0x00EF, 0x69, 0x00, 0x00, 0x00 }, /* ï to i */ 1267 { 0x00F0, 0x64, 0x00, 0x00, 0x00 }, /* ð to d */ 1268 { 0x00F1, 0x6E, 0x00, 0x00, 0x00 }, /* ñ to n */ 1269 { 0x00F2, 0x6F, 0x00, 0x00, 0x00 }, /* ò to o */ 1270 { 0x00F3, 0x6F, 0x00, 0x00, 0x00 }, /* ó to o */ 1271 { 0x00F4, 0x6F, 0x00, 0x00, 0x00 }, /* ô to o */ 1272 { 0x00F5, 0x6F, 0x00, 0x00, 0x00 }, /* õ to o */ 1273 { 0x00F6, 0x6F, 0x65, 0x00, 0x00 }, /* ö to oe */ 1274 { 0x00F7, 0x3A, 0x00, 0x00, 0x00 }, /* ÷ to : */ 1275 { 0x00F8, 0x6F, 0x00, 0x00, 0x00 }, /* ø to o */ 1276 { 0x00F9, 0x75, 0x00, 0x00, 0x00 }, /* ù to u */ 1277 { 0x00FA, 0x75, 0x00, 0x00, 0x00 }, /* ú to u */ 1278 { 0x00FB, 0x75, 0x00, 0x00, 0x00 }, /* û to u */ 1279 { 0x00FC, 0x75, 0x65, 0x00, 0x00 }, /* ü to ue */ 1280 { 0x00FD, 0x79, 0x00, 0x00, 0x00 }, /* ý to y */ 1281 { 0x00FE, 0x74, 0x68, 0x00, 0x00 }, /* þ to th */ 1282 { 0x00FF, 0x79, 0x00, 0x00, 0x00 }, /* ÿ to y */ 1283 { 0x0100, 0x41, 0x00, 0x00, 0x00 }, /* Ā to A */ 1284 { 0x0101, 0x61, 0x00, 0x00, 0x00 }, /* ā to a */ 1285 { 0x0102, 0x41, 0x00, 0x00, 0x00 }, /* Ă to A */ 1286 { 0x0103, 0x61, 0x00, 0x00, 0x00 }, /* ă to a */ 1287 { 0x0104, 0x41, 0x00, 0x00, 0x00 }, /* Ą to A */ 1288 { 0x0105, 0x61, 0x00, 0x00, 0x00 }, /* ą to a */ 1289 { 0x0106, 0x43, 0x00, 0x00, 0x00 }, /* Ć to C */ 1290 { 0x0107, 0x63, 0x00, 0x00, 0x00 }, /* ć to c */ 1291 { 0x0108, 0x43, 0x68, 0x00, 0x00 }, /* Ĉ to Ch */ 1292 { 0x0109, 0x63, 0x68, 0x00, 0x00 }, /* ĉ to ch */ 1293 { 0x010A, 0x43, 0x00, 0x00, 0x00 }, /* Ċ to C */ 1294 { 0x010B, 0x63, 0x00, 0x00, 0x00 }, /* ċ to c */ 1295 { 0x010C, 0x43, 0x00, 0x00, 0x00 }, /* Č to C */ 1296 { 0x010D, 0x63, 0x00, 0x00, 0x00 }, /* č to c */ 1297 { 0x010E, 0x44, 0x00, 0x00, 0x00 }, /* Ď to D */ 1298 { 0x010F, 0x64, 0x00, 0x00, 0x00 }, /* ď to d */ 1299 { 0x0110, 0x44, 0x00, 0x00, 0x00 }, /* Đ to D */ 1300 { 0x0111, 0x64, 0x00, 0x00, 0x00 }, /* đ to d */ 1301 { 0x0112, 0x45, 0x00, 0x00, 0x00 }, /* Ē to E */ 1302 { 0x0113, 0x65, 0x00, 0x00, 0x00 }, /* ē to e */ 1303 { 0x0114, 0x45, 0x00, 0x00, 0x00 }, /* Ĕ to E */ 1304 { 0x0115, 0x65, 0x00, 0x00, 0x00 }, /* ĕ to e */ 1305 { 0x0116, 0x45, 0x00, 0x00, 0x00 }, /* Ė to E */ 1306 { 0x0117, 0x65, 0x00, 0x00, 0x00 }, /* ė to e */ 1307 { 0x0118, 0x45, 0x00, 0x00, 0x00 }, /* Ę to E */ 1308 { 0x0119, 0x65, 0x00, 0x00, 0x00 }, /* ę to e */ 1309 { 0x011A, 0x45, 0x00, 0x00, 0x00 }, /* Ě to E */ 1310 { 0x011B, 0x65, 0x00, 0x00, 0x00 }, /* ě to e */ 1311 { 0x011C, 0x47, 0x68, 0x00, 0x00 }, /* Ĝ to Gh */ 1312 { 0x011D, 0x67, 0x68, 0x00, 0x00 }, /* ĝ to gh */ 1313 { 0x011E, 0x47, 0x00, 0x00, 0x00 }, /* Ğ to G */ 1314 { 0x011F, 0x67, 0x00, 0x00, 0x00 }, /* ğ to g */ 1315 { 0x0120, 0x47, 0x00, 0x00, 0x00 }, /* Ġ to G */ 1316 { 0x0121, 0x67, 0x00, 0x00, 0x00 }, /* ġ to g */ 1317 { 0x0122, 0x47, 0x00, 0x00, 0x00 }, /* Ģ to G */ 1318 { 0x0123, 0x67, 0x00, 0x00, 0x00 }, /* ģ to g */ 1319 { 0x0124, 0x48, 0x68, 0x00, 0x00 }, /* Ĥ to Hh */ 1320 { 0x0125, 0x68, 0x68, 0x00, 0x00 }, /* ĥ to hh */ 1321 { 0x0126, 0x48, 0x00, 0x00, 0x00 }, /* Ħ to H */ 1322 { 0x0127, 0x68, 0x00, 0x00, 0x00 }, /* ħ to h */ 1323 { 0x0128, 0x49, 0x00, 0x00, 0x00 }, /* Ĩ to I */ 1324 { 0x0129, 0x69, 0x00, 0x00, 0x00 }, /* ĩ to i */ 1325 { 0x012A, 0x49, 0x00, 0x00, 0x00 }, /* Ī to I */ 1326 { 0x012B, 0x69, 0x00, 0x00, 0x00 }, /* ī to i */ 1327 { 0x012C, 0x49, 0x00, 0x00, 0x00 }, /* Ĭ to I */ 1328 { 0x012D, 0x69, 0x00, 0x00, 0x00 }, /* ĭ to i */ 1329 { 0x012E, 0x49, 0x00, 0x00, 0x00 }, /* Į to I */ 1330 { 0x012F, 0x69, 0x00, 0x00, 0x00 }, /* į to i */ 1331 { 0x0130, 0x49, 0x00, 0x00, 0x00 }, /* İ to I */ 1332 { 0x0131, 0x69, 0x00, 0x00, 0x00 }, /* ı to i */ 1333 { 0x0132, 0x49, 0x4A, 0x00, 0x00 }, /* IJ to IJ */ 1334 { 0x0133, 0x69, 0x6A, 0x00, 0x00 }, /* ij to ij */ 1335 { 0x0134, 0x4A, 0x68, 0x00, 0x00 }, /* Ĵ to Jh */ 1336 { 0x0135, 0x6A, 0x68, 0x00, 0x00 }, /* ĵ to jh */ 1337 { 0x0136, 0x4B, 0x00, 0x00, 0x00 }, /* Ķ to K */ 1338 { 0x0137, 0x6B, 0x00, 0x00, 0x00 }, /* ķ to k */ 1339 { 0x0138, 0x6B, 0x00, 0x00, 0x00 }, /* ĸ to k */ 1340 { 0x0139, 0x4C, 0x00, 0x00, 0x00 }, /* Ĺ to L */ 1341 { 0x013A, 0x6C, 0x00, 0x00, 0x00 }, /* ĺ to l */ 1342 { 0x013B, 0x4C, 0x00, 0x00, 0x00 }, /* Ļ to L */ 1343 { 0x013C, 0x6C, 0x00, 0x00, 0x00 }, /* ļ to l */ 1344 { 0x013D, 0x4C, 0x00, 0x00, 0x00 }, /* Ľ to L */ 1345 { 0x013E, 0x6C, 0x00, 0x00, 0x00 }, /* ľ to l */ 1346 { 0x013F, 0x4C, 0x2E, 0x00, 0x00 }, /* Ŀ to L. */ 1347 { 0x0140, 0x6C, 0x2E, 0x00, 0x00 }, /* ŀ to l. */ 1348 { 0x0141, 0x4C, 0x00, 0x00, 0x00 }, /* Ł to L */ 1349 { 0x0142, 0x6C, 0x00, 0x00, 0x00 }, /* ł to l */ 1350 { 0x0143, 0x4E, 0x00, 0x00, 0x00 }, /* Ń to N */ 1351 { 0x0144, 0x6E, 0x00, 0x00, 0x00 }, /* ń to n */ 1352 { 0x0145, 0x4E, 0x00, 0x00, 0x00 }, /* Ņ to N */ 1353 { 0x0146, 0x6E, 0x00, 0x00, 0x00 }, /* ņ to n */ 1354 { 0x0147, 0x4E, 0x00, 0x00, 0x00 }, /* Ň to N */ 1355 { 0x0148, 0x6E, 0x00, 0x00, 0x00 }, /* ň to n */ 1356 { 0x0149, 0x27, 0x6E, 0x00, 0x00 }, /* ʼn to 'n */ 1357 { 0x014A, 0x4E, 0x47, 0x00, 0x00 }, /* Ŋ to NG */ 1358 { 0x014B, 0x6E, 0x67, 0x00, 0x00 }, /* ŋ to ng */ 1359 { 0x014C, 0x4F, 0x00, 0x00, 0x00 }, /* Ō to O */ 1360 { 0x014D, 0x6F, 0x00, 0x00, 0x00 }, /* ō to o */ 1361 { 0x014E, 0x4F, 0x00, 0x00, 0x00 }, /* Ŏ to O */ 1362 { 0x014F, 0x6F, 0x00, 0x00, 0x00 }, /* ŏ to o */ 1363 { 0x0150, 0x4F, 0x00, 0x00, 0x00 }, /* Ő to O */ 1364 { 0x0151, 0x6F, 0x00, 0x00, 0x00 }, /* ő to o */ 1365 { 0x0152, 0x4F, 0x45, 0x00, 0x00 }, /* Œ to OE */ 1366 { 0x0153, 0x6F, 0x65, 0x00, 0x00 }, /* œ to oe */ 1367 { 0x0154, 0x52, 0x00, 0x00, 0x00 }, /* Ŕ to R */ 1368 { 0x0155, 0x72, 0x00, 0x00, 0x00 }, /* ŕ to r */ 1369 { 0x0156, 0x52, 0x00, 0x00, 0x00 }, /* Ŗ to R */ 1370 { 0x0157, 0x72, 0x00, 0x00, 0x00 }, /* ŗ to r */ 1371 { 0x0158, 0x52, 0x00, 0x00, 0x00 }, /* Ř to R */ 1372 { 0x0159, 0x72, 0x00, 0x00, 0x00 }, /* ř to r */ 1373 { 0x015A, 0x53, 0x00, 0x00, 0x00 }, /* Ś to S */ 1374 { 0x015B, 0x73, 0x00, 0x00, 0x00 }, /* ś to s */ 1375 { 0x015C, 0x53, 0x68, 0x00, 0x00 }, /* Ŝ to Sh */ 1376 { 0x015D, 0x73, 0x68, 0x00, 0x00 }, /* ŝ to sh */ 1377 { 0x015E, 0x53, 0x00, 0x00, 0x00 }, /* Ş to S */ 1378 { 0x015F, 0x73, 0x00, 0x00, 0x00 }, /* ş to s */ 1379 { 0x0160, 0x53, 0x00, 0x00, 0x00 }, /* Š to S */ 1380 { 0x0161, 0x73, 0x00, 0x00, 0x00 }, /* š to s */ 1381 { 0x0162, 0x54, 0x00, 0x00, 0x00 }, /* Ţ to T */ 1382 { 0x0163, 0x74, 0x00, 0x00, 0x00 }, /* ţ to t */ 1383 { 0x0164, 0x54, 0x00, 0x00, 0x00 }, /* Ť to T */ 1384 { 0x0165, 0x74, 0x00, 0x00, 0x00 }, /* ť to t */ 1385 { 0x0166, 0x54, 0x00, 0x00, 0x00 }, /* Ŧ to T */ 1386 { 0x0167, 0x74, 0x00, 0x00, 0x00 }, /* ŧ to t */ 1387 { 0x0168, 0x55, 0x00, 0x00, 0x00 }, /* Ũ to U */ 1388 { 0x0169, 0x75, 0x00, 0x00, 0x00 }, /* ũ to u */ 1389 { 0x016A, 0x55, 0x00, 0x00, 0x00 }, /* Ū to U */ 1390 { 0x016B, 0x75, 0x00, 0x00, 0x00 }, /* ū to u */ 1391 { 0x016C, 0x55, 0x00, 0x00, 0x00 }, /* Ŭ to U */ 1392 { 0x016D, 0x75, 0x00, 0x00, 0x00 }, /* ŭ to u */ 1393 { 0x016E, 0x55, 0x00, 0x00, 0x00 }, /* Ů to U */ 1394 { 0x016F, 0x75, 0x00, 0x00, 0x00 }, /* ů to u */ 1395 { 0x0170, 0x55, 0x00, 0x00, 0x00 }, /* Ű to U */ 1396 { 0x0171, 0x75, 0x00, 0x00, 0x00 }, /* ű to u */ 1397 { 0x0172, 0x55, 0x00, 0x00, 0x00 }, /* Ų to U */ 1398 { 0x0173, 0x75, 0x00, 0x00, 0x00 }, /* ų to u */ 1399 { 0x0174, 0x57, 0x00, 0x00, 0x00 }, /* Ŵ to W */ 1400 { 0x0175, 0x77, 0x00, 0x00, 0x00 }, /* ŵ to w */ 1401 { 0x0176, 0x59, 0x00, 0x00, 0x00 }, /* Ŷ to Y */ 1402 { 0x0177, 0x79, 0x00, 0x00, 0x00 }, /* ŷ to y */ 1403 { 0x0178, 0x59, 0x00, 0x00, 0x00 }, /* Ÿ to Y */ 1404 { 0x0179, 0x5A, 0x00, 0x00, 0x00 }, /* Ź to Z */ 1405 { 0x017A, 0x7A, 0x00, 0x00, 0x00 }, /* ź to z */ 1406 { 0x017B, 0x5A, 0x00, 0x00, 0x00 }, /* Ż to Z */ 1407 { 0x017C, 0x7A, 0x00, 0x00, 0x00 }, /* ż to z */ 1408 { 0x017D, 0x5A, 0x00, 0x00, 0x00 }, /* Ž to Z */ 1409 { 0x017E, 0x7A, 0x00, 0x00, 0x00 }, /* ž to z */ 1410 { 0x017F, 0x73, 0x00, 0x00, 0x00 }, /* ſ to s */ 1411 { 0x0192, 0x66, 0x00, 0x00, 0x00 }, /* ƒ to f */ 1412 { 0x0218, 0x53, 0x00, 0x00, 0x00 }, /* Ș to S */ 1413 { 0x0219, 0x73, 0x00, 0x00, 0x00 }, /* ș to s */ 1414 { 0x021A, 0x54, 0x00, 0x00, 0x00 }, /* Ț to T */ 1415 { 0x021B, 0x74, 0x00, 0x00, 0x00 }, /* ț to t */ 1416 { 0x0386, 0x41, 0x00, 0x00, 0x00 }, /* Ά to A */ 1417 { 0x0388, 0x45, 0x00, 0x00, 0x00 }, /* Έ to E */ 1418 { 0x0389, 0x49, 0x00, 0x00, 0x00 }, /* Ή to I */ 1419 { 0x038A, 0x49, 0x00, 0x00, 0x00 }, /* Ί to I */ 1420 { 0x038C, 0x4f, 0x00, 0x00, 0x00 }, /* Ό to O */ 1421 { 0x038E, 0x59, 0x00, 0x00, 0x00 }, /* Ύ to Y */ 1422 { 0x038F, 0x4f, 0x00, 0x00, 0x00 }, /* Ώ to O */ 1423 { 0x0390, 0x69, 0x00, 0x00, 0x00 }, /* ΐ to i */ 1424 { 0x0391, 0x41, 0x00, 0x00, 0x00 }, /* Α to A */ 1425 { 0x0392, 0x42, 0x00, 0x00, 0x00 }, /* Β to B */ 1426 { 0x0393, 0x47, 0x00, 0x00, 0x00 }, /* Γ to G */ 1427 { 0x0394, 0x44, 0x00, 0x00, 0x00 }, /* Δ to D */ 1428 { 0x0395, 0x45, 0x00, 0x00, 0x00 }, /* Ε to E */ 1429 { 0x0396, 0x5a, 0x00, 0x00, 0x00 }, /* Ζ to Z */ 1430 { 0x0397, 0x49, 0x00, 0x00, 0x00 }, /* Η to I */ 1431 { 0x0398, 0x54, 0x68, 0x00, 0x00 }, /* Θ to Th */ 1432 { 0x0399, 0x49, 0x00, 0x00, 0x00 }, /* Ι to I */ 1433 { 0x039A, 0x4b, 0x00, 0x00, 0x00 }, /* Κ to K */ 1434 { 0x039B, 0x4c, 0x00, 0x00, 0x00 }, /* Λ to L */ 1435 { 0x039C, 0x4d, 0x00, 0x00, 0x00 }, /* Μ to M */ 1436 { 0x039D, 0x4e, 0x00, 0x00, 0x00 }, /* Ν to N */ 1437 { 0x039E, 0x58, 0x00, 0x00, 0x00 }, /* Ξ to X */ 1438 { 0x039F, 0x4f, 0x00, 0x00, 0x00 }, /* Ο to O */ 1439 { 0x03A0, 0x50, 0x00, 0x00, 0x00 }, /* Π to P */ 1440 { 0x03A1, 0x52, 0x00, 0x00, 0x00 }, /* Ρ to R */ 1441 { 0x03A3, 0x53, 0x00, 0x00, 0x00 }, /* Σ to S */ 1442 { 0x03A4, 0x54, 0x00, 0x00, 0x00 }, /* Τ to T */ 1443 { 0x03A5, 0x59, 0x00, 0x00, 0x00 }, /* Υ to Y */ 1444 { 0x03A6, 0x46, 0x00, 0x00, 0x00 }, /* Φ to F */ 1445 { 0x03A7, 0x43, 0x68, 0x00, 0x00 }, /* Χ to Ch */ 1446 { 0x03A8, 0x50, 0x73, 0x00, 0x00 }, /* Ψ to Ps */ 1447 { 0x03A9, 0x4f, 0x00, 0x00, 0x00 }, /* Ω to O */ 1448 { 0x03AA, 0x49, 0x00, 0x00, 0x00 }, /* Ϊ to I */ 1449 { 0x03AB, 0x59, 0x00, 0x00, 0x00 }, /* Ϋ to Y */ 1450 { 0x03AC, 0x61, 0x00, 0x00, 0x00 }, /* ά to a */ 1451 { 0x03AD, 0x65, 0x00, 0x00, 0x00 }, /* έ to e */ 1452 { 0x03AE, 0x69, 0x00, 0x00, 0x00 }, /* ή to i */ 1453 { 0x03AF, 0x69, 0x00, 0x00, 0x00 }, /* ί to i */ 1454 { 0x03B1, 0x61, 0x00, 0x00, 0x00 }, /* α to a */ 1455 { 0x03B2, 0x62, 0x00, 0x00, 0x00 }, /* β to b */ 1456 { 0x03B3, 0x67, 0x00, 0x00, 0x00 }, /* γ to g */ 1457 { 0x03B4, 0x64, 0x00, 0x00, 0x00 }, /* δ to d */ 1458 { 0x03B5, 0x65, 0x00, 0x00, 0x00 }, /* ε to e */ 1459 { 0x03B6, 0x7a, 0x00, 0x00, 0x00 }, /* ζ to z */ 1460 { 0x03B7, 0x69, 0x00, 0x00, 0x00 }, /* η to i */ 1461 { 0x03B8, 0x74, 0x68, 0x00, 0x00 }, /* θ to th */ 1462 { 0x03B9, 0x69, 0x00, 0x00, 0x00 }, /* ι to i */ 1463 { 0x03BA, 0x6b, 0x00, 0x00, 0x00 }, /* κ to k */ 1464 { 0x03BB, 0x6c, 0x00, 0x00, 0x00 }, /* λ to l */ 1465 { 0x03BC, 0x6d, 0x00, 0x00, 0x00 }, /* μ to m */ 1466 { 0x03BD, 0x6e, 0x00, 0x00, 0x00 }, /* ν to n */ 1467 { 0x03BE, 0x78, 0x00, 0x00, 0x00 }, /* ξ to x */ 1468 { 0x03BF, 0x6f, 0x00, 0x00, 0x00 }, /* ο to o */ 1469 { 0x03C0, 0x70, 0x00, 0x00, 0x00 }, /* π to p */ 1470 { 0x03C1, 0x72, 0x00, 0x00, 0x00 }, /* ρ to r */ 1471 { 0x03C3, 0x73, 0x00, 0x00, 0x00 }, /* σ to s */ 1472 { 0x03C4, 0x74, 0x00, 0x00, 0x00 }, /* τ to t */ 1473 { 0x03C5, 0x79, 0x00, 0x00, 0x00 }, /* υ to y */ 1474 { 0x03C6, 0x66, 0x00, 0x00, 0x00 }, /* φ to f */ 1475 { 0x03C7, 0x63, 0x68, 0x00, 0x00 }, /* χ to ch */ 1476 { 0x03C8, 0x70, 0x73, 0x00, 0x00 }, /* ψ to ps */ 1477 { 0x03C9, 0x6f, 0x00, 0x00, 0x00 }, /* ω to o */ 1478 { 0x03CA, 0x69, 0x00, 0x00, 0x00 }, /* ϊ to i */ 1479 { 0x03CB, 0x79, 0x00, 0x00, 0x00 }, /* ϋ to y */ 1480 { 0x03CC, 0x6f, 0x00, 0x00, 0x00 }, /* ό to o */ 1481 { 0x03CD, 0x79, 0x00, 0x00, 0x00 }, /* ύ to y */ 1482 { 0x03CE, 0x69, 0x00, 0x00, 0x00 }, /* ώ to i */ 1483 { 0x0400, 0x45, 0x00, 0x00, 0x00 }, /* Ѐ to E */ 1484 { 0x0401, 0x45, 0x00, 0x00, 0x00 }, /* Ё to E */ 1485 { 0x0402, 0x44, 0x00, 0x00, 0x00 }, /* Ђ to D */ 1486 { 0x0403, 0x47, 0x00, 0x00, 0x00 }, /* Ѓ to G */ 1487 { 0x0404, 0x45, 0x00, 0x00, 0x00 }, /* Є to E */ 1488 { 0x0405, 0x5a, 0x00, 0x00, 0x00 }, /* Ѕ to Z */ 1489 { 0x0406, 0x49, 0x00, 0x00, 0x00 }, /* І to I */ 1490 { 0x0407, 0x49, 0x00, 0x00, 0x00 }, /* Ї to I */ 1491 { 0x0408, 0x4a, 0x00, 0x00, 0x00 }, /* Ј to J */ 1492 { 0x0409, 0x49, 0x00, 0x00, 0x00 }, /* Љ to I */ 1493 { 0x040A, 0x4e, 0x00, 0x00, 0x00 }, /* Њ to N */ 1494 { 0x040B, 0x44, 0x00, 0x00, 0x00 }, /* Ћ to D */ 1495 { 0x040C, 0x4b, 0x00, 0x00, 0x00 }, /* Ќ to K */ 1496 { 0x040D, 0x49, 0x00, 0x00, 0x00 }, /* Ѝ to I */ 1497 { 0x040E, 0x55, 0x00, 0x00, 0x00 }, /* Ў to U */ 1498 { 0x040F, 0x44, 0x00, 0x00, 0x00 }, /* Џ to D */ 1499 { 0x0410, 0x41, 0x00, 0x00, 0x00 }, /* А to A */ 1500 { 0x0411, 0x42, 0x00, 0x00, 0x00 }, /* Б to B */ 1501 { 0x0412, 0x56, 0x00, 0x00, 0x00 }, /* В to V */ 1502 { 0x0413, 0x47, 0x00, 0x00, 0x00 }, /* Г to G */ 1503 { 0x0414, 0x44, 0x00, 0x00, 0x00 }, /* Д to D */ 1504 { 0x0415, 0x45, 0x00, 0x00, 0x00 }, /* Е to E */ 1505 { 0x0416, 0x5a, 0x68, 0x00, 0x00 }, /* Ж to Zh */ 1506 { 0x0417, 0x5a, 0x00, 0x00, 0x00 }, /* З to Z */ 1507 { 0x0418, 0x49, 0x00, 0x00, 0x00 }, /* И to I */ 1508 { 0x0419, 0x49, 0x00, 0x00, 0x00 }, /* Й to I */ 1509 { 0x041A, 0x4b, 0x00, 0x00, 0x00 }, /* К to K */ 1510 { 0x041B, 0x4c, 0x00, 0x00, 0x00 }, /* Л to L */ 1511 { 0x041C, 0x4d, 0x00, 0x00, 0x00 }, /* М to M */ 1512 { 0x041D, 0x4e, 0x00, 0x00, 0x00 }, /* Н to N */ 1513 { 0x041E, 0x4f, 0x00, 0x00, 0x00 }, /* О to O */ 1514 { 0x041F, 0x50, 0x00, 0x00, 0x00 }, /* П to P */ 1515 { 0x0420, 0x52, 0x00, 0x00, 0x00 }, /* Р to R */ 1516 { 0x0421, 0x53, 0x00, 0x00, 0x00 }, /* С to S */ 1517 { 0x0422, 0x54, 0x00, 0x00, 0x00 }, /* Т to T */ 1518 { 0x0423, 0x55, 0x00, 0x00, 0x00 }, /* У to U */ 1519 { 0x0424, 0x46, 0x00, 0x00, 0x00 }, /* Ф to F */ 1520 { 0x0425, 0x4b, 0x68, 0x00, 0x00 }, /* Х to Kh */ 1521 { 0x0426, 0x54, 0x63, 0x00, 0x00 }, /* Ц to Tc */ 1522 { 0x0427, 0x43, 0x68, 0x00, 0x00 }, /* Ч to Ch */ 1523 { 0x0428, 0x53, 0x68, 0x00, 0x00 }, /* Ш to Sh */ 1524 { 0x0429, 0x53, 0x68, 0x63, 0x68 }, /* Щ to Shch */ 1525 { 0x042A, 0x61, 0x00, 0x00, 0x00 }, /* to A */ 1526 { 0x042B, 0x59, 0x00, 0x00, 0x00 }, /* Ы to Y */ 1527 { 0x042C, 0x59, 0x00, 0x00, 0x00 }, /* to Y */ 1528 { 0x042D, 0x45, 0x00, 0x00, 0x00 }, /* Э to E */ 1529 { 0x042E, 0x49, 0x75, 0x00, 0x00 }, /* Ю to Iu */ 1530 { 0x042F, 0x49, 0x61, 0x00, 0x00 }, /* Я to Ia */ 1531 { 0x0430, 0x61, 0x00, 0x00, 0x00 }, /* а to a */ 1532 { 0x0431, 0x62, 0x00, 0x00, 0x00 }, /* б to b */ 1533 { 0x0432, 0x76, 0x00, 0x00, 0x00 }, /* в to v */ 1534 { 0x0433, 0x67, 0x00, 0x00, 0x00 }, /* г to g */ 1535 { 0x0434, 0x64, 0x00, 0x00, 0x00 }, /* д to d */ 1536 { 0x0435, 0x65, 0x00, 0x00, 0x00 }, /* е to e */ 1537 { 0x0436, 0x7a, 0x68, 0x00, 0x00 }, /* ж to zh */ 1538 { 0x0437, 0x7a, 0x00, 0x00, 0x00 }, /* з to z */ 1539 { 0x0438, 0x69, 0x00, 0x00, 0x00 }, /* и to i */ 1540 { 0x0439, 0x69, 0x00, 0x00, 0x00 }, /* й to i */ 1541 { 0x043A, 0x6b, 0x00, 0x00, 0x00 }, /* к to k */ 1542 { 0x043B, 0x6c, 0x00, 0x00, 0x00 }, /* л to l */ 1543 { 0x043C, 0x6d, 0x00, 0x00, 0x00 }, /* м to m */ 1544 { 0x043D, 0x6e, 0x00, 0x00, 0x00 }, /* н to n */ 1545 { 0x043E, 0x6f, 0x00, 0x00, 0x00 }, /* о to o */ 1546 { 0x043F, 0x70, 0x00, 0x00, 0x00 }, /* п to p */ 1547 { 0x0440, 0x72, 0x00, 0x00, 0x00 }, /* р to r */ 1548 { 0x0441, 0x73, 0x00, 0x00, 0x00 }, /* с to s */ 1549 { 0x0442, 0x74, 0x00, 0x00, 0x00 }, /* т to t */ 1550 { 0x0443, 0x75, 0x00, 0x00, 0x00 }, /* у to u */ 1551 { 0x0444, 0x66, 0x00, 0x00, 0x00 }, /* ф to f */ 1552 { 0x0445, 0x6b, 0x68, 0x00, 0x00 }, /* х to kh */ 1553 { 0x0446, 0x74, 0x63, 0x00, 0x00 }, /* ц to tc */ 1554 { 0x0447, 0x63, 0x68, 0x00, 0x00 }, /* ч to ch */ 1555 { 0x0448, 0x73, 0x68, 0x00, 0x00 }, /* ш to sh */ 1556 { 0x0449, 0x73, 0x68, 0x63, 0x68 }, /* щ to shch */ 1557 { 0x044A, 0x61, 0x00, 0x00, 0x00 }, /* to a */ 1558 { 0x044B, 0x79, 0x00, 0x00, 0x00 }, /* ы to y */ 1559 { 0x044C, 0x79, 0x00, 0x00, 0x00 }, /* to y */ 1560 { 0x044D, 0x65, 0x00, 0x00, 0x00 }, /* э to e */ 1561 { 0x044E, 0x69, 0x75, 0x00, 0x00 }, /* ю to iu */ 1562 { 0x044F, 0x69, 0x61, 0x00, 0x00 }, /* я to ia */ 1563 { 0x0450, 0x65, 0x00, 0x00, 0x00 }, /* ѐ to e */ 1564 { 0x0451, 0x65, 0x00, 0x00, 0x00 }, /* ё to e */ 1565 { 0x0452, 0x64, 0x00, 0x00, 0x00 }, /* ђ to d */ 1566 { 0x0453, 0x67, 0x00, 0x00, 0x00 }, /* ѓ to g */ 1567 { 0x0454, 0x65, 0x00, 0x00, 0x00 }, /* є to e */ 1568 { 0x0455, 0x7a, 0x00, 0x00, 0x00 }, /* ѕ to z */ 1569 { 0x0456, 0x69, 0x00, 0x00, 0x00 }, /* і to i */ 1570 { 0x0457, 0x69, 0x00, 0x00, 0x00 }, /* ї to i */ 1571 { 0x0458, 0x6a, 0x00, 0x00, 0x00 }, /* ј to j */ 1572 { 0x0459, 0x69, 0x00, 0x00, 0x00 }, /* љ to i */ 1573 { 0x045A, 0x6e, 0x00, 0x00, 0x00 }, /* њ to n */ 1574 { 0x045B, 0x64, 0x00, 0x00, 0x00 }, /* ћ to d */ 1575 { 0x045C, 0x6b, 0x00, 0x00, 0x00 }, /* ќ to k */ 1576 { 0x045D, 0x69, 0x00, 0x00, 0x00 }, /* ѝ to i */ 1577 { 0x045E, 0x75, 0x00, 0x00, 0x00 }, /* ў to u */ 1578 { 0x045F, 0x64, 0x00, 0x00, 0x00 }, /* џ to d */ 1579 { 0x1E02, 0x42, 0x00, 0x00, 0x00 }, /* Ḃ to B */ 1580 { 0x1E03, 0x62, 0x00, 0x00, 0x00 }, /* ḃ to b */ 1581 { 0x1E0A, 0x44, 0x00, 0x00, 0x00 }, /* Ḋ to D */ 1582 { 0x1E0B, 0x64, 0x00, 0x00, 0x00 }, /* ḋ to d */ 1583 { 0x1E1E, 0x46, 0x00, 0x00, 0x00 }, /* Ḟ to F */ 1584 { 0x1E1F, 0x66, 0x00, 0x00, 0x00 }, /* ḟ to f */ 1585 { 0x1E40, 0x4D, 0x00, 0x00, 0x00 }, /* Ṁ to M */ 1586 { 0x1E41, 0x6D, 0x00, 0x00, 0x00 }, /* ṁ to m */ 1587 { 0x1E56, 0x50, 0x00, 0x00, 0x00 }, /* Ṗ to P */ 1588 { 0x1E57, 0x70, 0x00, 0x00, 0x00 }, /* ṗ to p */ 1589 { 0x1E60, 0x53, 0x00, 0x00, 0x00 }, /* Ṡ to S */ 1590 { 0x1E61, 0x73, 0x00, 0x00, 0x00 }, /* ṡ to s */ 1591 { 0x1E6A, 0x54, 0x00, 0x00, 0x00 }, /* Ṫ to T */ 1592 { 0x1E6B, 0x74, 0x00, 0x00, 0x00 }, /* ṫ to t */ 1593 { 0x1E80, 0x57, 0x00, 0x00, 0x00 }, /* Ẁ to W */ 1594 { 0x1E81, 0x77, 0x00, 0x00, 0x00 }, /* ẁ to w */ 1595 { 0x1E82, 0x57, 0x00, 0x00, 0x00 }, /* Ẃ to W */ 1596 { 0x1E83, 0x77, 0x00, 0x00, 0x00 }, /* ẃ to w */ 1597 { 0x1E84, 0x57, 0x00, 0x00, 0x00 }, /* Ẅ to W */ 1598 { 0x1E85, 0x77, 0x00, 0x00, 0x00 }, /* ẅ to w */ 1599 { 0x1EF2, 0x59, 0x00, 0x00, 0x00 }, /* Ỳ to Y */ 1600 { 0x1EF3, 0x79, 0x00, 0x00, 0x00 }, /* ỳ to y */ 1601 { 0xFB00, 0x66, 0x66, 0x00, 0x00 }, /* ff to ff */ 1602 { 0xFB01, 0x66, 0x69, 0x00, 0x00 }, /* fi to fi */ 1603 { 0xFB02, 0x66, 0x6C, 0x00, 0x00 }, /* fl to fl */ 1604 { 0xFB05, 0x73, 0x74, 0x00, 0x00 }, /* ſt to st */ 1605 { 0xFB06, 0x73, 0x74, 0x00, 0x00 }, /* st to st */ 1606 }; 1607 1608 static const Transliteration *spellfixFindTranslit(int c, int *pxTop){ 1609 *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1; 1610 return translit; 1611 } 1612 1613 /* 1614 ** Convert the input string from UTF-8 into pure ASCII by converting 1615 ** all non-ASCII characters to some combination of characters in the 1616 ** ASCII subset. 1617 ** 1618 ** The returned string might contain more characters than the input. 1619 ** 1620 ** Space to hold the returned string comes from sqlite3_malloc() and 1621 ** should be freed by the caller. 1622 */ 1623 static unsigned char *transliterate(const unsigned char *zIn, int nIn){ 1624 unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 ); 1625 int c, sz, nOut; 1626 if( zOut==0 ) return 0; 1627 nOut = 0; 1628 while( nIn>0 ){ 1629 c = utf8Read(zIn, nIn, &sz); 1630 zIn += sz; 1631 nIn -= sz; 1632 if( c<=127 ){ 1633 zOut[nOut++] = (unsigned char)c; 1634 }else{ 1635 int xTop, xBtm, x; 1636 const Transliteration *tbl = spellfixFindTranslit(c, &xTop); 1637 xBtm = 0; 1638 while( xTop>=xBtm ){ 1639 x = (xTop + xBtm)/2; 1640 if( tbl[x].cFrom==c ){ 1641 zOut[nOut++] = tbl[x].cTo0; 1642 if( tbl[x].cTo1 ){ 1643 zOut[nOut++] = tbl[x].cTo1; 1644 if( tbl[x].cTo2 ){ 1645 zOut[nOut++] = tbl[x].cTo2; 1646 if( tbl[x].cTo3 ){ 1647 zOut[nOut++] = tbl[x].cTo3; 1648 } 1649 } 1650 } 1651 c = 0; 1652 break; 1653 }else if( tbl[x].cFrom>c ){ 1654 xTop = x-1; 1655 }else{ 1656 xBtm = x+1; 1657 } 1658 } 1659 if( c ) zOut[nOut++] = '?'; 1660 } 1661 } 1662 zOut[nOut] = 0; 1663 return zOut; 1664 } 1665 1666 /* 1667 ** Return the number of characters in the shortest prefix of the input 1668 ** string that transliterates to an ASCII string nTrans bytes or longer. 1669 ** Or, if the transliteration of the input string is less than nTrans 1670 ** bytes in size, return the number of characters in the input string. 1671 */ 1672 static int translen_to_charlen(const char *zIn, int nIn, int nTrans){ 1673 int i, c, sz, nOut; 1674 int nChar; 1675 1676 i = nOut = 0; 1677 for(nChar=0; i<nIn && nOut<nTrans; nChar++){ 1678 c = utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); 1679 i += sz; 1680 1681 nOut++; 1682 if( c>=128 ){ 1683 int xTop, xBtm, x; 1684 const Transliteration *tbl = spellfixFindTranslit(c, &xTop); 1685 xBtm = 0; 1686 while( xTop>=xBtm ){ 1687 x = (xTop + xBtm)/2; 1688 if( tbl[x].cFrom==c ){ 1689 if( tbl[x].cTo1 ){ 1690 nOut++; 1691 if( tbl[x].cTo2 ){ 1692 nOut++; 1693 if( tbl[x].cTo3 ){ 1694 nOut++; 1695 } 1696 } 1697 } 1698 break; 1699 }else if( tbl[x].cFrom>c ){ 1700 xTop = x-1; 1701 }else{ 1702 xBtm = x+1; 1703 } 1704 } 1705 } 1706 } 1707 1708 return nChar; 1709 } 1710 1711 1712 /* 1713 ** spellfix1_translit(X) 1714 ** 1715 ** Convert a string that contains non-ASCII Roman characters into 1716 ** pure ASCII. 1717 */ 1718 static void transliterateSqlFunc( 1719 sqlite3_context *context, 1720 int argc, 1721 sqlite3_value **argv 1722 ){ 1723 const unsigned char *zIn = sqlite3_value_text(argv[0]); 1724 int nIn = sqlite3_value_bytes(argv[0]); 1725 unsigned char *zOut = transliterate(zIn, nIn); 1726 if( zOut==0 ){ 1727 sqlite3_result_error_nomem(context); 1728 }else{ 1729 sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); 1730 } 1731 } 1732 1733 /* 1734 ** spellfix1_scriptcode(X) 1735 ** 1736 ** Try to determine the dominant script used by the word X and return 1737 ** its ISO 15924 numeric code. 1738 ** 1739 ** The current implementation only understands the following scripts: 1740 ** 1741 ** 215 (Latin) 1742 ** 220 (Cyrillic) 1743 ** 200 (Greek) 1744 ** 1745 ** This routine will return 998 if the input X contains characters from 1746 ** two or more of the above scripts or 999 if X contains no characters 1747 ** from any of the above scripts. 1748 */ 1749 static void scriptCodeSqlFunc( 1750 sqlite3_context *context, 1751 int argc, 1752 sqlite3_value **argv 1753 ){ 1754 const unsigned char *zIn = sqlite3_value_text(argv[0]); 1755 int nIn = sqlite3_value_bytes(argv[0]); 1756 int c, sz; 1757 int scriptMask = 0; 1758 int res; 1759 int seenDigit = 0; 1760 # define SCRIPT_LATIN 0x0001 1761 # define SCRIPT_CYRILLIC 0x0002 1762 # define SCRIPT_GREEK 0x0004 1763 # define SCRIPT_HEBREW 0x0008 1764 # define SCRIPT_ARABIC 0x0010 1765 1766 while( nIn>0 ){ 1767 c = utf8Read(zIn, nIn, &sz); 1768 zIn += sz; 1769 nIn -= sz; 1770 if( c<0x02af ){ 1771 if( c>=0x80 || midClass[c&0x7f]<CCLASS_DIGIT ){ 1772 scriptMask |= SCRIPT_LATIN; 1773 }else if( c>='0' && c<='9' ){ 1774 seenDigit = 1; 1775 } 1776 }else if( c>=0x0400 && c<=0x04ff ){ 1777 scriptMask |= SCRIPT_CYRILLIC; 1778 }else if( c>=0x0386 && c<=0x03ce ){ 1779 scriptMask |= SCRIPT_GREEK; 1780 }else if( c>=0x0590 && c<=0x05ff ){ 1781 scriptMask |= SCRIPT_HEBREW; 1782 }else if( c>=0x0600 && c<=0x06ff ){ 1783 scriptMask |= SCRIPT_ARABIC; 1784 } 1785 } 1786 if( scriptMask==0 && seenDigit ) scriptMask = SCRIPT_LATIN; 1787 switch( scriptMask ){ 1788 case 0: res = 999; break; 1789 case SCRIPT_LATIN: res = 215; break; 1790 case SCRIPT_CYRILLIC: res = 220; break; 1791 case SCRIPT_GREEK: res = 200; break; 1792 case SCRIPT_HEBREW: res = 125; break; 1793 case SCRIPT_ARABIC: res = 160; break; 1794 default: res = 998; break; 1795 } 1796 sqlite3_result_int(context, res); 1797 } 1798 1799 /* End transliterate 1800 ****************************************************************************** 1801 ****************************************************************************** 1802 ** Begin spellfix1 virtual table. 1803 */ 1804 1805 /* Maximum length of a phonehash used for querying the shadow table */ 1806 #define SPELLFIX_MX_HASH 32 1807 1808 /* Maximum number of hash strings to examine per query */ 1809 #define SPELLFIX_MX_RUN 1 1810 1811 typedef struct spellfix1_vtab spellfix1_vtab; 1812 typedef struct spellfix1_cursor spellfix1_cursor; 1813 1814 /* Fuzzy-search virtual table object */ 1815 struct spellfix1_vtab { 1816 sqlite3_vtab base; /* Base class - must be first */ 1817 sqlite3 *db; /* Database connection */ 1818 char *zDbName; /* Name of database holding this table */ 1819 char *zTableName; /* Name of the virtual table */ 1820 char *zCostTable; /* Table holding edit-distance cost numbers */ 1821 EditDist3Config *pConfig3; /* Parsed edit distance costs */ 1822 }; 1823 1824 /* Fuzzy-search cursor object */ 1825 struct spellfix1_cursor { 1826 sqlite3_vtab_cursor base; /* Base class - must be first */ 1827 spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ 1828 char *zPattern; /* rhs of MATCH clause */ 1829 int idxNum; /* idxNum value passed to xFilter() */ 1830 int nRow; /* Number of rows of content */ 1831 int nAlloc; /* Number of allocated rows */ 1832 int iRow; /* Current row of content */ 1833 int iLang; /* Value of the langid= constraint */ 1834 int iTop; /* Value of the top= constraint */ 1835 int iScope; /* Value of the scope= constraint */ 1836 int nSearch; /* Number of vocabulary items checked */ 1837 sqlite3_stmt *pFullScan; /* Shadow query for a full table scan */ 1838 struct spellfix1_row { /* For each row of content */ 1839 sqlite3_int64 iRowid; /* Rowid for this row */ 1840 char *zWord; /* Text for this row */ 1841 int iRank; /* Rank for this row */ 1842 int iDistance; /* Distance from pattern for this row */ 1843 int iScore; /* Score for sorting */ 1844 int iMatchlen; /* Value of matchlen column (or -1) */ 1845 char zHash[SPELLFIX_MX_HASH]; /* the phonehash used for this match */ 1846 } *a; 1847 }; 1848 1849 /* 1850 ** Construct one or more SQL statements from the format string given 1851 ** and then evaluate those statements. The success code is written 1852 ** into *pRc. 1853 ** 1854 ** If *pRc is initially non-zero then this routine is a no-op. 1855 */ 1856 static void spellfix1DbExec( 1857 int *pRc, /* Success code */ 1858 sqlite3 *db, /* Database in which to run SQL */ 1859 const char *zFormat, /* Format string for SQL */ 1860 ... /* Arguments to the format string */ 1861 ){ 1862 va_list ap; 1863 char *zSql; 1864 if( *pRc ) return; 1865 va_start(ap, zFormat); 1866 zSql = sqlite3_vmprintf(zFormat, ap); 1867 va_end(ap); 1868 if( zSql==0 ){ 1869 *pRc = SQLITE_NOMEM; 1870 }else{ 1871 *pRc = sqlite3_exec(db, zSql, 0, 0, 0); 1872 sqlite3_free(zSql); 1873 } 1874 } 1875 1876 /* 1877 ** xDisconnect/xDestroy method for the fuzzy-search module. 1878 */ 1879 static int spellfix1Uninit(int isDestroy, sqlite3_vtab *pVTab){ 1880 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 1881 int rc = SQLITE_OK; 1882 if( isDestroy ){ 1883 sqlite3 *db = p->db; 1884 spellfix1DbExec(&rc, db, "DROP TABLE IF EXISTS \"%w\".\"%w_vocab\"", 1885 p->zDbName, p->zTableName); 1886 } 1887 if( rc==SQLITE_OK ){ 1888 sqlite3_free(p->zTableName); 1889 editDist3ConfigDelete(p->pConfig3); 1890 sqlite3_free(p->zCostTable); 1891 sqlite3_free(p); 1892 } 1893 return rc; 1894 } 1895 static int spellfix1Disconnect(sqlite3_vtab *pVTab){ 1896 return spellfix1Uninit(0, pVTab); 1897 } 1898 static int spellfix1Destroy(sqlite3_vtab *pVTab){ 1899 return spellfix1Uninit(1, pVTab); 1900 } 1901 1902 /* 1903 ** Make a copy of a string. Remove leading and trailing whitespace 1904 ** and dequote it. 1905 */ 1906 static char *spellfix1Dequote(const char *zIn){ 1907 char *zOut; 1908 int i, j; 1909 char c; 1910 while( isspace((unsigned char)zIn[0]) ) zIn++; 1911 zOut = sqlite3_mprintf("%s", zIn); 1912 if( zOut==0 ) return 0; 1913 i = (int)strlen(zOut); 1914 #if 0 /* The parser will never leave spaces at the end */ 1915 while( i>0 && isspace(zOut[i-1]) ){ i--; } 1916 #endif 1917 zOut[i] = 0; 1918 c = zOut[0]; 1919 if( c=='\'' || c=='"' ){ 1920 for(i=1, j=0; ALWAYS(zOut[i]); i++){ 1921 zOut[j++] = zOut[i]; 1922 if( zOut[i]==c ){ 1923 if( zOut[i+1]==c ){ 1924 i++; 1925 }else{ 1926 zOut[j-1] = 0; 1927 break; 1928 } 1929 } 1930 } 1931 } 1932 return zOut; 1933 } 1934 1935 1936 /* 1937 ** xConnect/xCreate method for the spellfix1 module. Arguments are: 1938 ** 1939 ** argv[0] -> module name ("spellfix1") 1940 ** argv[1] -> database name 1941 ** argv[2] -> table name 1942 ** argv[3].. -> optional arguments (i.e. "edit_cost_table" parameter) 1943 */ 1944 static int spellfix1Init( 1945 int isCreate, 1946 sqlite3 *db, 1947 void *pAux, 1948 int argc, const char *const*argv, 1949 sqlite3_vtab **ppVTab, 1950 char **pzErr 1951 ){ 1952 spellfix1_vtab *pNew = 0; 1953 /* const char *zModule = argv[0]; // not used */ 1954 const char *zDbName = argv[1]; 1955 const char *zTableName = argv[2]; 1956 int nDbName; 1957 int rc = SQLITE_OK; 1958 int i; 1959 1960 nDbName = (int)strlen(zDbName); 1961 pNew = sqlite3_malloc64( sizeof(*pNew) + nDbName + 1); 1962 if( pNew==0 ){ 1963 rc = SQLITE_NOMEM; 1964 }else{ 1965 memset(pNew, 0, sizeof(*pNew)); 1966 pNew->zDbName = (char*)&pNew[1]; 1967 memcpy(pNew->zDbName, zDbName, nDbName+1); 1968 pNew->zTableName = sqlite3_mprintf("%s", zTableName); 1969 pNew->db = db; 1970 if( pNew->zTableName==0 ){ 1971 rc = SQLITE_NOMEM; 1972 }else{ 1973 rc = sqlite3_declare_vtab(db, 1974 "CREATE TABLE x(word,rank,distance,langid, " 1975 "score, matchlen, phonehash HIDDEN, " 1976 "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, " 1977 "soundslike HIDDEN, command HIDDEN)" 1978 ); 1979 #define SPELLFIX_COL_WORD 0 1980 #define SPELLFIX_COL_RANK 1 1981 #define SPELLFIX_COL_DISTANCE 2 1982 #define SPELLFIX_COL_LANGID 3 1983 #define SPELLFIX_COL_SCORE 4 1984 #define SPELLFIX_COL_MATCHLEN 5 1985 #define SPELLFIX_COL_PHONEHASH 6 1986 #define SPELLFIX_COL_TOP 7 1987 #define SPELLFIX_COL_SCOPE 8 1988 #define SPELLFIX_COL_SRCHCNT 9 1989 #define SPELLFIX_COL_SOUNDSLIKE 10 1990 #define SPELLFIX_COL_COMMAND 11 1991 } 1992 if( rc==SQLITE_OK && isCreate ){ 1993 spellfix1DbExec(&rc, db, 1994 "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" 1995 " id INTEGER PRIMARY KEY,\n" 1996 " rank INT,\n" 1997 " langid INT,\n" 1998 " word TEXT,\n" 1999 " k1 TEXT,\n" 2000 " k2 TEXT\n" 2001 ");\n", 2002 zDbName, zTableName 2003 ); 2004 spellfix1DbExec(&rc, db, 2005 "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" " 2006 "ON \"%w_vocab\"(langid,k2);", 2007 zDbName, zTableName, zTableName 2008 ); 2009 } 2010 for(i=3; rc==SQLITE_OK && i<argc; i++){ 2011 if( strncmp(argv[i],"edit_cost_table=",16)==0 && pNew->zCostTable==0 ){ 2012 pNew->zCostTable = spellfix1Dequote(&argv[i][16]); 2013 if( pNew->zCostTable==0 ) rc = SQLITE_NOMEM; 2014 continue; 2015 } 2016 *pzErr = sqlite3_mprintf("bad argument to spellfix1(): \"%s\"", argv[i]); 2017 rc = SQLITE_ERROR; 2018 } 2019 } 2020 2021 if( rc && pNew ){ 2022 *ppVTab = 0; 2023 spellfix1Uninit(0, &pNew->base); 2024 }else{ 2025 *ppVTab = (sqlite3_vtab *)pNew; 2026 } 2027 return rc; 2028 } 2029 2030 /* 2031 ** The xConnect and xCreate methods 2032 */ 2033 static int spellfix1Connect( 2034 sqlite3 *db, 2035 void *pAux, 2036 int argc, const char *const*argv, 2037 sqlite3_vtab **ppVTab, 2038 char **pzErr 2039 ){ 2040 return spellfix1Init(0, db, pAux, argc, argv, ppVTab, pzErr); 2041 } 2042 static int spellfix1Create( 2043 sqlite3 *db, 2044 void *pAux, 2045 int argc, const char *const*argv, 2046 sqlite3_vtab **ppVTab, 2047 char **pzErr 2048 ){ 2049 return spellfix1Init(1, db, pAux, argc, argv, ppVTab, pzErr); 2050 } 2051 2052 /* 2053 ** Clear all of the content from a cursor. 2054 */ 2055 static void spellfix1ResetCursor(spellfix1_cursor *pCur){ 2056 int i; 2057 for(i=0; i<pCur->nRow; i++){ 2058 sqlite3_free(pCur->a[i].zWord); 2059 } 2060 pCur->nRow = 0; 2061 pCur->iRow = 0; 2062 pCur->nSearch = 0; 2063 if( pCur->pFullScan ){ 2064 sqlite3_finalize(pCur->pFullScan); 2065 pCur->pFullScan = 0; 2066 } 2067 } 2068 2069 /* 2070 ** Resize the cursor to hold up to N rows of content 2071 */ 2072 static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){ 2073 struct spellfix1_row *aNew; 2074 assert( N>=pCur->nRow ); 2075 aNew = sqlite3_realloc64(pCur->a, sizeof(pCur->a[0])*N); 2076 if( aNew==0 && N>0 ){ 2077 spellfix1ResetCursor(pCur); 2078 sqlite3_free(pCur->a); 2079 pCur->nAlloc = 0; 2080 pCur->a = 0; 2081 }else{ 2082 pCur->nAlloc = N; 2083 pCur->a = aNew; 2084 } 2085 } 2086 2087 2088 /* 2089 ** Close a fuzzy-search cursor. 2090 */ 2091 static int spellfix1Close(sqlite3_vtab_cursor *cur){ 2092 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2093 spellfix1ResetCursor(pCur); 2094 spellfix1ResizeCursor(pCur, 0); 2095 sqlite3_free(pCur->zPattern); 2096 sqlite3_free(pCur); 2097 return SQLITE_OK; 2098 } 2099 2100 #define SPELLFIX_IDXNUM_MATCH 0x01 /* word MATCH $str */ 2101 #define SPELLFIX_IDXNUM_LANGID 0x02 /* langid == $langid */ 2102 #define SPELLFIX_IDXNUM_TOP 0x04 /* top = $top */ 2103 #define SPELLFIX_IDXNUM_SCOPE 0x08 /* scope = $scope */ 2104 #define SPELLFIX_IDXNUM_DISTLT 0x10 /* distance < $distance */ 2105 #define SPELLFIX_IDXNUM_DISTLE 0x20 /* distance <= $distance */ 2106 #define SPELLFIX_IDXNUM_ROWID 0x40 /* rowid = $rowid */ 2107 #define SPELLFIX_IDXNUM_DIST (0x10|0x20) /* DISTLT and DISTLE */ 2108 2109 /* 2110 ** 2111 ** The plan number is a bitmask of the SPELLFIX_IDXNUM_* values defined 2112 ** above. 2113 ** 2114 ** filter.argv[*] values contains $str, $langid, $top, $scope and $rowid 2115 ** if specified and in that order. 2116 */ 2117 static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ 2118 int iPlan = 0; 2119 int iLangTerm = -1; 2120 int iTopTerm = -1; 2121 int iScopeTerm = -1; 2122 int iDistTerm = -1; 2123 int iRowidTerm = -1; 2124 int i; 2125 const struct sqlite3_index_constraint *pConstraint; 2126 pConstraint = pIdxInfo->aConstraint; 2127 for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ 2128 if( pConstraint->usable==0 ) continue; 2129 2130 /* Terms of the form: word MATCH $str */ 2131 if( (iPlan & SPELLFIX_IDXNUM_MATCH)==0 2132 && pConstraint->iColumn==SPELLFIX_COL_WORD 2133 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH 2134 ){ 2135 iPlan |= SPELLFIX_IDXNUM_MATCH; 2136 pIdxInfo->aConstraintUsage[i].argvIndex = 1; 2137 pIdxInfo->aConstraintUsage[i].omit = 1; 2138 } 2139 2140 /* Terms of the form: langid = $langid */ 2141 if( (iPlan & SPELLFIX_IDXNUM_LANGID)==0 2142 && pConstraint->iColumn==SPELLFIX_COL_LANGID 2143 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2144 ){ 2145 iPlan |= SPELLFIX_IDXNUM_LANGID; 2146 iLangTerm = i; 2147 } 2148 2149 /* Terms of the form: top = $top */ 2150 if( (iPlan & SPELLFIX_IDXNUM_TOP)==0 2151 && pConstraint->iColumn==SPELLFIX_COL_TOP 2152 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2153 ){ 2154 iPlan |= SPELLFIX_IDXNUM_TOP; 2155 iTopTerm = i; 2156 } 2157 2158 /* Terms of the form: scope = $scope */ 2159 if( (iPlan & SPELLFIX_IDXNUM_SCOPE)==0 2160 && pConstraint->iColumn==SPELLFIX_COL_SCOPE 2161 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2162 ){ 2163 iPlan |= SPELLFIX_IDXNUM_SCOPE; 2164 iScopeTerm = i; 2165 } 2166 2167 /* Terms of the form: distance < $dist or distance <= $dist */ 2168 if( (iPlan & SPELLFIX_IDXNUM_DIST)==0 2169 && pConstraint->iColumn==SPELLFIX_COL_DISTANCE 2170 && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT 2171 || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) 2172 ){ 2173 if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ 2174 iPlan |= SPELLFIX_IDXNUM_DISTLT; 2175 }else{ 2176 iPlan |= SPELLFIX_IDXNUM_DISTLE; 2177 } 2178 iDistTerm = i; 2179 } 2180 2181 /* Terms of the form: distance < $dist or distance <= $dist */ 2182 if( (iPlan & SPELLFIX_IDXNUM_ROWID)==0 2183 && pConstraint->iColumn<0 2184 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2185 ){ 2186 iPlan |= SPELLFIX_IDXNUM_ROWID; 2187 iRowidTerm = i; 2188 } 2189 } 2190 if( iPlan&SPELLFIX_IDXNUM_MATCH ){ 2191 int idx = 2; 2192 pIdxInfo->idxNum = iPlan; 2193 if( pIdxInfo->nOrderBy==1 2194 && pIdxInfo->aOrderBy[0].iColumn==SPELLFIX_COL_SCORE 2195 && pIdxInfo->aOrderBy[0].desc==0 2196 ){ 2197 pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ 2198 } 2199 if( iPlan&SPELLFIX_IDXNUM_LANGID ){ 2200 pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; 2201 pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; 2202 } 2203 if( iPlan&SPELLFIX_IDXNUM_TOP ){ 2204 pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; 2205 pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; 2206 } 2207 if( iPlan&SPELLFIX_IDXNUM_SCOPE ){ 2208 pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; 2209 pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; 2210 } 2211 if( iPlan&SPELLFIX_IDXNUM_DIST ){ 2212 pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; 2213 pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; 2214 } 2215 pIdxInfo->estimatedCost = 1e5; 2216 }else if( (iPlan & SPELLFIX_IDXNUM_ROWID) ){ 2217 pIdxInfo->idxNum = SPELLFIX_IDXNUM_ROWID; 2218 pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; 2219 pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; 2220 pIdxInfo->estimatedCost = 5; 2221 }else{ 2222 pIdxInfo->idxNum = 0; 2223 pIdxInfo->estimatedCost = 1e50; 2224 } 2225 return SQLITE_OK; 2226 } 2227 2228 /* 2229 ** Open a new fuzzy-search cursor. 2230 */ 2231 static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ 2232 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2233 spellfix1_cursor *pCur; 2234 pCur = sqlite3_malloc64( sizeof(*pCur) ); 2235 if( pCur==0 ) return SQLITE_NOMEM; 2236 memset(pCur, 0, sizeof(*pCur)); 2237 pCur->pVTab = p; 2238 *ppCursor = &pCur->base; 2239 return SQLITE_OK; 2240 } 2241 2242 /* 2243 ** Adjust a distance measurement by the words rank in order to show 2244 ** preference to common words. 2245 */ 2246 static int spellfix1Score(int iDistance, int iRank){ 2247 int iLog2; 2248 for(iLog2=0; iRank>0; iLog2++, iRank>>=1){} 2249 return iDistance + 32 - iLog2; 2250 } 2251 2252 /* 2253 ** Compare two spellfix1_row objects for sorting purposes in qsort() such 2254 ** that they sort in order of increasing distance. 2255 */ 2256 static int SQLITE_CDECL spellfix1RowCompare(const void *A, const void *B){ 2257 const struct spellfix1_row *a = (const struct spellfix1_row*)A; 2258 const struct spellfix1_row *b = (const struct spellfix1_row*)B; 2259 return a->iScore - b->iScore; 2260 } 2261 2262 /* 2263 ** A structure used to pass information from spellfix1FilterForMatch() 2264 ** into spellfix1RunQuery(). 2265 */ 2266 typedef struct MatchQuery { 2267 spellfix1_cursor *pCur; /* The cursor being queried */ 2268 sqlite3_stmt *pStmt; /* shadow table query statment */ 2269 char zHash[SPELLFIX_MX_HASH]; /* The current phonehash for zPattern */ 2270 const char *zPattern; /* Transliterated input string */ 2271 int nPattern; /* Length of zPattern */ 2272 EditDist3FromString *pMatchStr3; /* Original unicode string */ 2273 EditDist3Config *pConfig3; /* Edit-distance cost coefficients */ 2274 const EditDist3Lang *pLang; /* The selected language coefficients */ 2275 int iLang; /* The language id */ 2276 int iScope; /* Default scope */ 2277 int iMaxDist; /* Maximum allowed edit distance, or -1 */ 2278 int rc; /* Error code */ 2279 int nRun; /* Number of prior runs for the same zPattern */ 2280 char azPrior[SPELLFIX_MX_RUN][SPELLFIX_MX_HASH]; /* Prior hashes */ 2281 } MatchQuery; 2282 2283 /* 2284 ** Run a query looking for the best matches against zPattern using 2285 ** zHash as the character class seed hash. 2286 */ 2287 static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ 2288 const char *zK1; 2289 const char *zWord; 2290 int iDist; 2291 int iRank; 2292 int iScore; 2293 int iWorst = 0; 2294 int idx; 2295 int idxWorst = -1; 2296 int i; 2297 int iScope = p->iScope; 2298 spellfix1_cursor *pCur = p->pCur; 2299 sqlite3_stmt *pStmt = p->pStmt; 2300 char zHash1[SPELLFIX_MX_HASH]; 2301 char zHash2[SPELLFIX_MX_HASH]; 2302 char *zClass; 2303 int nClass; 2304 int rc; 2305 2306 if( pCur->a==0 || p->rc ) return; /* Prior memory allocation failure */ 2307 zClass = (char*)phoneticHash((unsigned char*)zQuery, nQuery); 2308 if( zClass==0 ){ 2309 p->rc = SQLITE_NOMEM; 2310 return; 2311 } 2312 nClass = (int)strlen(zClass); 2313 if( nClass>SPELLFIX_MX_HASH-2 ){ 2314 nClass = SPELLFIX_MX_HASH-2; 2315 zClass[nClass] = 0; 2316 } 2317 if( nClass<=iScope ){ 2318 if( nClass>2 ){ 2319 iScope = nClass-1; 2320 }else{ 2321 iScope = nClass; 2322 } 2323 } 2324 memcpy(zHash1, zClass, iScope); 2325 sqlite3_free(zClass); 2326 zHash1[iScope] = 0; 2327 memcpy(zHash2, zHash1, iScope); 2328 zHash2[iScope] = 'Z'; 2329 zHash2[iScope+1] = 0; 2330 #if SPELLFIX_MX_RUN>1 2331 for(i=0; i<p->nRun; i++){ 2332 if( strcmp(p->azPrior[i], zHash1)==0 ) return; 2333 } 2334 #endif 2335 assert( p->nRun<SPELLFIX_MX_RUN ); 2336 memcpy(p->azPrior[p->nRun++], zHash1, iScope+1); 2337 if( sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC)==SQLITE_NOMEM 2338 || sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC)==SQLITE_NOMEM 2339 ){ 2340 p->rc = SQLITE_NOMEM; 2341 return; 2342 } 2343 #if SPELLFIX_MX_RUN>1 2344 for(i=0; i<pCur->nRow; i++){ 2345 if( pCur->a[i].iScore>iWorst ){ 2346 iWorst = pCur->a[i].iScore; 2347 idxWorst = i; 2348 } 2349 } 2350 #endif 2351 while( sqlite3_step(pStmt)==SQLITE_ROW ){ 2352 int iMatchlen = -1; 2353 iRank = sqlite3_column_int(pStmt, 2); 2354 if( p->pMatchStr3 ){ 2355 int nWord = sqlite3_column_bytes(pStmt, 1); 2356 zWord = (const char*)sqlite3_column_text(pStmt, 1); 2357 iDist = editDist3Core(p->pMatchStr3, zWord, nWord, p->pLang, &iMatchlen); 2358 }else{ 2359 zK1 = (const char*)sqlite3_column_text(pStmt, 3); 2360 if( zK1==0 ) continue; 2361 iDist = editdist1(p->zPattern, zK1, 0); 2362 } 2363 if( iDist<0 ){ 2364 p->rc = SQLITE_NOMEM; 2365 break; 2366 } 2367 pCur->nSearch++; 2368 2369 /* If there is a "distance < $dist" or "distance <= $dist" constraint, 2370 ** check if this row meets it. If not, jump back up to the top of the 2371 ** loop to process the next row. Otherwise, if the row does match the 2372 ** distance constraint, check if the pCur->a[] array is already full. 2373 ** If it is and no explicit "top = ?" constraint was present in the 2374 ** query, grow the array to ensure there is room for the new entry. */ 2375 assert( (p->iMaxDist>=0)==((pCur->idxNum & SPELLFIX_IDXNUM_DIST) ? 1 : 0) ); 2376 if( p->iMaxDist>=0 ){ 2377 if( iDist>p->iMaxDist ) continue; 2378 if( pCur->nRow>=pCur->nAlloc && (pCur->idxNum & SPELLFIX_IDXNUM_TOP)==0 ){ 2379 spellfix1ResizeCursor(pCur, pCur->nAlloc*2 + 10); 2380 if( pCur->a==0 ) break; 2381 } 2382 } 2383 2384 iScore = spellfix1Score(iDist,iRank); 2385 if( pCur->nRow<pCur->nAlloc ){ 2386 idx = pCur->nRow; 2387 }else if( iScore<iWorst ){ 2388 idx = idxWorst; 2389 sqlite3_free(pCur->a[idx].zWord); 2390 }else{ 2391 continue; 2392 } 2393 2394 pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); 2395 if( pCur->a[idx].zWord==0 ){ 2396 p->rc = SQLITE_NOMEM; 2397 break; 2398 } 2399 pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); 2400 pCur->a[idx].iRank = iRank; 2401 pCur->a[idx].iDistance = iDist; 2402 pCur->a[idx].iScore = iScore; 2403 pCur->a[idx].iMatchlen = iMatchlen; 2404 memcpy(pCur->a[idx].zHash, zHash1, iScope+1); 2405 if( pCur->nRow<pCur->nAlloc ) pCur->nRow++; 2406 if( pCur->nRow==pCur->nAlloc ){ 2407 iWorst = pCur->a[0].iScore; 2408 idxWorst = 0; 2409 for(i=1; i<pCur->nRow; i++){ 2410 iScore = pCur->a[i].iScore; 2411 if( iWorst<iScore ){ 2412 iWorst = iScore; 2413 idxWorst = i; 2414 } 2415 } 2416 } 2417 } 2418 rc = sqlite3_reset(pStmt); 2419 if( rc ) p->rc = rc; 2420 } 2421 2422 /* 2423 ** This version of the xFilter method work if the MATCH term is present 2424 ** and we are doing a scan. 2425 */ 2426 static int spellfix1FilterForMatch( 2427 spellfix1_cursor *pCur, 2428 int argc, 2429 sqlite3_value **argv 2430 ){ 2431 int idxNum = pCur->idxNum; 2432 const unsigned char *zMatchThis; /* RHS of the MATCH operator */ 2433 EditDist3FromString *pMatchStr3 = 0; /* zMatchThis as an editdist string */ 2434 char *zPattern; /* Transliteration of zMatchThis */ 2435 int nPattern; /* Length of zPattern */ 2436 int iLimit = 20; /* Max number of rows of output */ 2437 int iScope = 3; /* Use this many characters of zClass */ 2438 int iLang = 0; /* Language code */ 2439 char *zSql; /* SQL of shadow table query */ 2440 sqlite3_stmt *pStmt = 0; /* Shadow table query */ 2441 int rc; /* Result code */ 2442 int idx = 1; /* Next available filter parameter */ 2443 spellfix1_vtab *p = pCur->pVTab; /* The virtual table that owns pCur */ 2444 MatchQuery x; /* For passing info to RunQuery() */ 2445 2446 /* Load the cost table if we have not already done so */ 2447 if( p->zCostTable!=0 && p->pConfig3==0 ){ 2448 p->pConfig3 = sqlite3_malloc64( sizeof(p->pConfig3[0]) ); 2449 if( p->pConfig3==0 ) return SQLITE_NOMEM; 2450 memset(p->pConfig3, 0, sizeof(p->pConfig3[0])); 2451 rc = editDist3ConfigLoad(p->pConfig3, p->db, p->zCostTable); 2452 if( rc ) return rc; 2453 } 2454 memset(&x, 0, sizeof(x)); 2455 x.iScope = 3; /* Default scope if none specified by "WHERE scope=N" */ 2456 x.iMaxDist = -1; /* Maximum allowed edit distance */ 2457 2458 if( idxNum&2 ){ 2459 iLang = sqlite3_value_int(argv[idx++]); 2460 } 2461 if( idxNum&4 ){ 2462 iLimit = sqlite3_value_int(argv[idx++]); 2463 if( iLimit<1 ) iLimit = 1; 2464 } 2465 if( idxNum&8 ){ 2466 x.iScope = sqlite3_value_int(argv[idx++]); 2467 if( x.iScope<1 ) x.iScope = 1; 2468 if( x.iScope>SPELLFIX_MX_HASH-2 ) x.iScope = SPELLFIX_MX_HASH-2; 2469 } 2470 if( idxNum&(16|32) ){ 2471 x.iMaxDist = sqlite3_value_int(argv[idx++]); 2472 if( idxNum&16 ) x.iMaxDist--; 2473 if( x.iMaxDist<0 ) x.iMaxDist = 0; 2474 } 2475 spellfix1ResetCursor(pCur); 2476 spellfix1ResizeCursor(pCur, iLimit); 2477 zMatchThis = sqlite3_value_text(argv[0]); 2478 if( zMatchThis==0 ) return SQLITE_OK; 2479 if( p->pConfig3 ){ 2480 x.pLang = editDist3FindLang(p->pConfig3, iLang); 2481 pMatchStr3 = editDist3FromStringNew(x.pLang, (const char*)zMatchThis, -1); 2482 if( pMatchStr3==0 ){ 2483 x.rc = SQLITE_NOMEM; 2484 goto filter_exit; 2485 } 2486 }else{ 2487 x.pLang = 0; 2488 } 2489 zPattern = (char*)transliterate(zMatchThis, sqlite3_value_bytes(argv[0])); 2490 sqlite3_free(pCur->zPattern); 2491 pCur->zPattern = zPattern; 2492 if( zPattern==0 ){ 2493 x.rc = SQLITE_NOMEM; 2494 goto filter_exit; 2495 } 2496 nPattern = (int)strlen(zPattern); 2497 if( zPattern[nPattern-1]=='*' ) nPattern--; 2498 zSql = sqlite3_mprintf( 2499 "SELECT id, word, rank, coalesce(k1,word)" 2500 " FROM \"%w\".\"%w_vocab\"" 2501 " WHERE langid=%d AND k2>=?1 AND k2<?2", 2502 p->zDbName, p->zTableName, iLang 2503 ); 2504 if( zSql==0 ){ 2505 x.rc = SQLITE_NOMEM; 2506 pStmt = 0; 2507 goto filter_exit; 2508 } 2509 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); 2510 sqlite3_free(zSql); 2511 pCur->iLang = iLang; 2512 x.pCur = pCur; 2513 x.pStmt = pStmt; 2514 x.zPattern = zPattern; 2515 x.nPattern = nPattern; 2516 x.pMatchStr3 = pMatchStr3; 2517 x.iLang = iLang; 2518 x.rc = rc; 2519 x.pConfig3 = p->pConfig3; 2520 if( x.rc==SQLITE_OK ){ 2521 spellfix1RunQuery(&x, zPattern, nPattern); 2522 } 2523 2524 if( pCur->a ){ 2525 qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); 2526 pCur->iTop = iLimit; 2527 pCur->iScope = iScope; 2528 }else{ 2529 x.rc = SQLITE_NOMEM; 2530 } 2531 2532 filter_exit: 2533 sqlite3_finalize(pStmt); 2534 editDist3FromStringDelete(pMatchStr3); 2535 return x.rc; 2536 } 2537 2538 /* 2539 ** This version of xFilter handles a full-table scan case 2540 */ 2541 static int spellfix1FilterForFullScan( 2542 spellfix1_cursor *pCur, 2543 int argc, 2544 sqlite3_value **argv 2545 ){ 2546 int rc = SQLITE_OK; 2547 int idxNum = pCur->idxNum; 2548 char *zSql; 2549 spellfix1_vtab *pVTab = pCur->pVTab; 2550 spellfix1ResetCursor(pCur); 2551 assert( idxNum==0 || idxNum==64 ); 2552 zSql = sqlite3_mprintf( 2553 "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"%s", 2554 pVTab->zDbName, pVTab->zTableName, 2555 ((idxNum & 64) ? " WHERE rowid=?" : "") 2556 ); 2557 if( zSql==0 ) return SQLITE_NOMEM; 2558 rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); 2559 sqlite3_free(zSql); 2560 if( rc==SQLITE_OK && (idxNum & 64) ){ 2561 assert( argc==1 ); 2562 rc = sqlite3_bind_value(pCur->pFullScan, 1, argv[0]); 2563 } 2564 pCur->nRow = pCur->iRow = 0; 2565 if( rc==SQLITE_OK ){ 2566 rc = sqlite3_step(pCur->pFullScan); 2567 if( rc==SQLITE_ROW ){ pCur->iRow = -1; rc = SQLITE_OK; } 2568 if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } 2569 }else{ 2570 pCur->iRow = 0; 2571 } 2572 return rc; 2573 } 2574 2575 2576 /* 2577 ** Called to "rewind" a cursor back to the beginning so that 2578 ** it starts its output over again. Always called at least once 2579 ** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call. 2580 */ 2581 static int spellfix1Filter( 2582 sqlite3_vtab_cursor *cur, 2583 int idxNum, const char *idxStr, 2584 int argc, sqlite3_value **argv 2585 ){ 2586 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2587 int rc; 2588 pCur->idxNum = idxNum; 2589 if( idxNum & 1 ){ 2590 rc = spellfix1FilterForMatch(pCur, argc, argv); 2591 }else{ 2592 rc = spellfix1FilterForFullScan(pCur, argc, argv); 2593 } 2594 return rc; 2595 } 2596 2597 2598 /* 2599 ** Advance a cursor to its next row of output 2600 */ 2601 static int spellfix1Next(sqlite3_vtab_cursor *cur){ 2602 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2603 int rc = SQLITE_OK; 2604 if( pCur->iRow < pCur->nRow ){ 2605 if( pCur->pFullScan ){ 2606 rc = sqlite3_step(pCur->pFullScan); 2607 if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; 2608 if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; 2609 }else{ 2610 pCur->iRow++; 2611 } 2612 } 2613 return rc; 2614 } 2615 2616 /* 2617 ** Return TRUE if we are at the end-of-file 2618 */ 2619 static int spellfix1Eof(sqlite3_vtab_cursor *cur){ 2620 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2621 return pCur->iRow>=pCur->nRow; 2622 } 2623 2624 /* 2625 ** Return columns from the current row. 2626 */ 2627 static int spellfix1Column( 2628 sqlite3_vtab_cursor *cur, 2629 sqlite3_context *ctx, 2630 int i 2631 ){ 2632 spellfix1_cursor *pCur = (spellfix1_cursor*)cur; 2633 if( pCur->pFullScan ){ 2634 if( i<=SPELLFIX_COL_LANGID ){ 2635 sqlite3_result_value(ctx, sqlite3_column_value(pCur->pFullScan, i)); 2636 }else{ 2637 sqlite3_result_null(ctx); 2638 } 2639 return SQLITE_OK; 2640 } 2641 switch( i ){ 2642 case SPELLFIX_COL_WORD: { 2643 sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); 2644 break; 2645 } 2646 case SPELLFIX_COL_RANK: { 2647 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank); 2648 break; 2649 } 2650 case SPELLFIX_COL_DISTANCE: { 2651 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iDistance); 2652 break; 2653 } 2654 case SPELLFIX_COL_LANGID: { 2655 sqlite3_result_int(ctx, pCur->iLang); 2656 break; 2657 } 2658 case SPELLFIX_COL_SCORE: { 2659 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iScore); 2660 break; 2661 } 2662 case SPELLFIX_COL_MATCHLEN: { 2663 int iMatchlen = pCur->a[pCur->iRow].iMatchlen; 2664 if( iMatchlen<0 ){ 2665 int nPattern = (int)strlen(pCur->zPattern); 2666 char *zWord = pCur->a[pCur->iRow].zWord; 2667 int nWord = (int)strlen(zWord); 2668 2669 if( nPattern>0 && pCur->zPattern[nPattern-1]=='*' ){ 2670 char *zTranslit; 2671 int res; 2672 zTranslit = (char *)transliterate((unsigned char *)zWord, nWord); 2673 if( !zTranslit ) return SQLITE_NOMEM; 2674 res = editdist1(pCur->zPattern, zTranslit, &iMatchlen); 2675 sqlite3_free(zTranslit); 2676 if( res<0 ) return SQLITE_NOMEM; 2677 iMatchlen = translen_to_charlen(zWord, nWord, iMatchlen); 2678 }else{ 2679 iMatchlen = utf8Charlen(zWord, nWord); 2680 } 2681 } 2682 2683 sqlite3_result_int(ctx, iMatchlen); 2684 break; 2685 } 2686 case SPELLFIX_COL_PHONEHASH: { 2687 sqlite3_result_text(ctx, pCur->a[pCur->iRow].zHash, -1, SQLITE_STATIC); 2688 break; 2689 } 2690 case SPELLFIX_COL_TOP: { 2691 sqlite3_result_int(ctx, pCur->iTop); 2692 break; 2693 } 2694 case SPELLFIX_COL_SCOPE: { 2695 sqlite3_result_int(ctx, pCur->iScope); 2696 break; 2697 } 2698 case SPELLFIX_COL_SRCHCNT: { 2699 sqlite3_result_int(ctx, pCur->nSearch); 2700 break; 2701 } 2702 default: { 2703 sqlite3_result_null(ctx); 2704 break; 2705 } 2706 } 2707 return SQLITE_OK; 2708 } 2709 2710 /* 2711 ** The rowid. 2712 */ 2713 static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ 2714 spellfix1_cursor *pCur = (spellfix1_cursor*)cur; 2715 if( pCur->pFullScan ){ 2716 *pRowid = sqlite3_column_int64(pCur->pFullScan, 4); 2717 }else{ 2718 *pRowid = pCur->a[pCur->iRow].iRowid; 2719 } 2720 return SQLITE_OK; 2721 } 2722 2723 /* 2724 ** This function is called by the xUpdate() method. It returns a string 2725 ** containing the conflict mode that xUpdate() should use for the current 2726 ** operation. One of: "ROLLBACK", "IGNORE", "ABORT" or "REPLACE". 2727 */ 2728 static const char *spellfix1GetConflict(sqlite3 *db){ 2729 static const char *azConflict[] = { 2730 /* Note: Instead of "FAIL" - "ABORT". */ 2731 "ROLLBACK", "IGNORE", "ABORT", "ABORT", "REPLACE" 2732 }; 2733 int eConflict = sqlite3_vtab_on_conflict(db); 2734 2735 assert( eConflict==SQLITE_ROLLBACK || eConflict==SQLITE_IGNORE 2736 || eConflict==SQLITE_FAIL || eConflict==SQLITE_ABORT 2737 || eConflict==SQLITE_REPLACE 2738 ); 2739 assert( SQLITE_ROLLBACK==1 ); 2740 assert( SQLITE_IGNORE==2 ); 2741 assert( SQLITE_FAIL==3 ); 2742 assert( SQLITE_ABORT==4 ); 2743 assert( SQLITE_REPLACE==5 ); 2744 2745 return azConflict[eConflict-1]; 2746 } 2747 2748 /* 2749 ** The xUpdate() method. 2750 */ 2751 static int spellfix1Update( 2752 sqlite3_vtab *pVTab, 2753 int argc, 2754 sqlite3_value **argv, 2755 sqlite_int64 *pRowid 2756 ){ 2757 int rc = SQLITE_OK; 2758 sqlite3_int64 rowid, newRowid; 2759 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2760 sqlite3 *db = p->db; 2761 2762 if( argc==1 ){ 2763 /* A delete operation on the rowid given by argv[0] */ 2764 rowid = *pRowid = sqlite3_value_int64(argv[0]); 2765 spellfix1DbExec(&rc, db, "DELETE FROM \"%w\".\"%w_vocab\" " 2766 " WHERE id=%lld", 2767 p->zDbName, p->zTableName, rowid); 2768 }else{ 2769 const unsigned char *zWord = sqlite3_value_text(argv[SPELLFIX_COL_WORD+2]); 2770 int nWord = sqlite3_value_bytes(argv[SPELLFIX_COL_WORD+2]); 2771 int iLang = sqlite3_value_int(argv[SPELLFIX_COL_LANGID+2]); 2772 int iRank = sqlite3_value_int(argv[SPELLFIX_COL_RANK+2]); 2773 const unsigned char *zSoundslike = 2774 sqlite3_value_text(argv[SPELLFIX_COL_SOUNDSLIKE+2]); 2775 int nSoundslike = sqlite3_value_bytes(argv[SPELLFIX_COL_SOUNDSLIKE+2]); 2776 char *zK1, *zK2; 2777 int i; 2778 char c; 2779 const char *zConflict = spellfix1GetConflict(db); 2780 2781 if( zWord==0 ){ 2782 /* Inserts of the form: INSERT INTO table(command) VALUES('xyzzy'); 2783 ** cause zWord to be NULL, so we look at the "command" column to see 2784 ** what special actions to take */ 2785 const char *zCmd = 2786 (const char*)sqlite3_value_text(argv[SPELLFIX_COL_COMMAND+2]); 2787 if( zCmd==0 ){ 2788 pVTab->zErrMsg = sqlite3_mprintf("NOT NULL constraint failed: %s.word", 2789 p->zTableName); 2790 return SQLITE_CONSTRAINT_NOTNULL; 2791 } 2792 if( strcmp(zCmd,"reset")==0 ){ 2793 /* Reset the edit cost table (if there is one). */ 2794 editDist3ConfigDelete(p->pConfig3); 2795 p->pConfig3 = 0; 2796 return SQLITE_OK; 2797 } 2798 if( strncmp(zCmd,"edit_cost_table=",16)==0 ){ 2799 editDist3ConfigDelete(p->pConfig3); 2800 p->pConfig3 = 0; 2801 sqlite3_free(p->zCostTable); 2802 p->zCostTable = spellfix1Dequote(zCmd+16); 2803 if( p->zCostTable==0 ) return SQLITE_NOMEM; 2804 if( p->zCostTable[0]==0 || sqlite3_stricmp(p->zCostTable,"null")==0 ){ 2805 sqlite3_free(p->zCostTable); 2806 p->zCostTable = 0; 2807 } 2808 return SQLITE_OK; 2809 } 2810 pVTab->zErrMsg = sqlite3_mprintf("unknown value for %s.command: \"%w\"", 2811 p->zTableName, zCmd); 2812 return SQLITE_ERROR; 2813 } 2814 if( iRank<1 ) iRank = 1; 2815 if( zSoundslike ){ 2816 zK1 = (char*)transliterate(zSoundslike, nSoundslike); 2817 }else{ 2818 zK1 = (char*)transliterate(zWord, nWord); 2819 } 2820 if( zK1==0 ) return SQLITE_NOMEM; 2821 for(i=0; (c = zK1[i])!=0; i++){ 2822 if( c>='A' && c<='Z' ) zK1[i] += 'a' - 'A'; 2823 } 2824 zK2 = (char*)phoneticHash((const unsigned char*)zK1, i); 2825 if( zK2==0 ){ 2826 sqlite3_free(zK1); 2827 return SQLITE_NOMEM; 2828 } 2829 if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ 2830 if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ 2831 spellfix1DbExec(&rc, db, 2832 "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " 2833 "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)", 2834 p->zDbName, p->zTableName, 2835 iRank, iLang, zWord, zK1, zWord, zK2 2836 ); 2837 }else{ 2838 newRowid = sqlite3_value_int64(argv[1]); 2839 spellfix1DbExec(&rc, db, 2840 "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " 2841 "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)", 2842 zConflict, p->zDbName, p->zTableName, 2843 newRowid, iRank, iLang, zWord, zK1, zWord, zK2 2844 ); 2845 } 2846 *pRowid = sqlite3_last_insert_rowid(db); 2847 }else{ 2848 rowid = sqlite3_value_int64(argv[0]); 2849 newRowid = *pRowid = sqlite3_value_int64(argv[1]); 2850 spellfix1DbExec(&rc, db, 2851 "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," 2852 " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld", 2853 zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang, 2854 zWord, zK1, zWord, zK2, rowid 2855 ); 2856 } 2857 sqlite3_free(zK1); 2858 sqlite3_free(zK2); 2859 } 2860 return rc; 2861 } 2862 2863 /* 2864 ** Rename the spellfix1 table. 2865 */ 2866 static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ 2867 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2868 sqlite3 *db = p->db; 2869 int rc = SQLITE_OK; 2870 char *zNewName = sqlite3_mprintf("%s", zNew); 2871 if( zNewName==0 ){ 2872 return SQLITE_NOMEM; 2873 } 2874 spellfix1DbExec(&rc, db, 2875 "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"", 2876 p->zDbName, p->zTableName, zNewName 2877 ); 2878 if( rc==SQLITE_OK ){ 2879 sqlite3_free(p->zTableName); 2880 p->zTableName = zNewName; 2881 }else{ 2882 sqlite3_free(zNewName); 2883 } 2884 return rc; 2885 } 2886 2887 2888 /* 2889 ** A virtual table module that provides fuzzy search. 2890 */ 2891 static sqlite3_module spellfix1Module = { 2892 0, /* iVersion */ 2893 spellfix1Create, /* xCreate - handle CREATE VIRTUAL TABLE */ 2894 spellfix1Connect, /* xConnect - reconnected to an existing table */ 2895 spellfix1BestIndex, /* xBestIndex - figure out how to do a query */ 2896 spellfix1Disconnect, /* xDisconnect - close a connection */ 2897 spellfix1Destroy, /* xDestroy - handle DROP TABLE */ 2898 spellfix1Open, /* xOpen - open a cursor */ 2899 spellfix1Close, /* xClose - close a cursor */ 2900 spellfix1Filter, /* xFilter - configure scan constraints */ 2901 spellfix1Next, /* xNext - advance a cursor */ 2902 spellfix1Eof, /* xEof - check for end of scan */ 2903 spellfix1Column, /* xColumn - read data */ 2904 spellfix1Rowid, /* xRowid - read data */ 2905 spellfix1Update, /* xUpdate */ 2906 0, /* xBegin */ 2907 0, /* xSync */ 2908 0, /* xCommit */ 2909 0, /* xRollback */ 2910 0, /* xFindMethod */ 2911 spellfix1Rename, /* xRename */ 2912 }; 2913 2914 /* 2915 ** Register the various functions and the virtual table. 2916 */ 2917 static int spellfix1Register(sqlite3 *db){ 2918 int rc = SQLITE_OK; 2919 int i; 2920 rc = sqlite3_create_function(db, "spellfix1_translit", 1, 2921 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2922 transliterateSqlFunc, 0, 0); 2923 if( rc==SQLITE_OK ){ 2924 rc = sqlite3_create_function(db, "spellfix1_editdist", 2, 2925 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2926 editdistSqlFunc, 0, 0); 2927 } 2928 if( rc==SQLITE_OK ){ 2929 rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, 2930 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2931 phoneticHashSqlFunc, 0, 0); 2932 } 2933 if( rc==SQLITE_OK ){ 2934 rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, 2935 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2936 scriptCodeSqlFunc, 0, 0); 2937 } 2938 if( rc==SQLITE_OK ){ 2939 rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); 2940 } 2941 if( rc==SQLITE_OK ){ 2942 rc = editDist3Install(db); 2943 } 2944 2945 /* Verify sanity of the translit[] table */ 2946 for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){ 2947 assert( translit[i].cFrom<translit[i+1].cFrom ); 2948 } 2949 2950 return rc; 2951 } 2952 2953 #endif /* SQLITE_OMIT_VIRTUALTABLE */ 2954 2955 /* 2956 ** Extension load function. 2957 */ 2958 #ifdef _WIN32 2959 __declspec(dllexport) 2960 #endif 2961 int sqlite3_spellfix_init( 2962 sqlite3 *db, 2963 char **pzErrMsg, 2964 const sqlite3_api_routines *pApi 2965 ){ 2966 SQLITE_EXTENSION_INIT2(pApi); 2967 #ifndef SQLITE_OMIT_VIRTUALTABLE 2968 return spellfix1Register(db); 2969 #endif 2970 return SQLITE_OK; 2971 } 2972