1 /* 2 ** 2012 April 10 3 ** 4 ** The author disclaims copyright to this source code. In place of 5 ** a legal notice, here is a blessing: 6 ** 7 ** May you do good and not evil. 8 ** May you find forgiveness for yourself and forgive others. 9 ** May you share freely, never taking more than you give. 10 ** 11 ************************************************************************* 12 ** 13 ** This module implements the spellfix1 VIRTUAL TABLE that can be used 14 ** to search a large vocabulary for close matches. See separate 15 ** documentation (http://www.sqlite.org/spellfix1.html) for details. 16 */ 17 #include "sqlite3ext.h" 18 SQLITE_EXTENSION_INIT1 19 20 #ifndef SQLITE_AMALGAMATION 21 # if !defined(NDEBUG) && !defined(SQLITE_DEBUG) 22 # define NDEBUG 1 23 # endif 24 # if defined(NDEBUG) && defined(SQLITE_DEBUG) 25 # undef NDEBUG 26 # endif 27 # include <string.h> 28 # include <stdio.h> 29 # include <stdlib.h> 30 # include <assert.h> 31 # define ALWAYS(X) 1 32 # define NEVER(X) 0 33 typedef unsigned char u8; 34 typedef unsigned short u16; 35 #endif 36 #include <ctype.h> 37 38 #ifndef SQLITE_OMIT_VIRTUALTABLE 39 40 /* 41 ** Character classes for ASCII characters: 42 ** 43 ** 0 '' Silent letters: H W 44 ** 1 'A' Any vowel: A E I O U (Y) 45 ** 2 'B' A bilabeal stop or fricative: B F P V W 46 ** 3 'C' Other fricatives or back stops: C G J K Q S X Z 47 ** 4 'D' Alveolar stops: D T 48 ** 5 'H' Letter H at the beginning of a word 49 ** 6 'L' Glide: L 50 ** 7 'R' Semivowel: R 51 ** 8 'M' Nasals: M N 52 ** 9 'Y' Letter Y at the beginning of a word. 53 ** 10 '9' Digits: 0 1 2 3 4 5 6 7 8 9 54 ** 11 ' ' White space 55 ** 12 '?' Other. 56 */ 57 #define CCLASS_SILENT 0 58 #define CCLASS_VOWEL 1 59 #define CCLASS_B 2 60 #define CCLASS_C 3 61 #define CCLASS_D 4 62 #define CCLASS_H 5 63 #define CCLASS_L 6 64 #define CCLASS_R 7 65 #define CCLASS_M 8 66 #define CCLASS_Y 9 67 #define CCLASS_DIGIT 10 68 #define CCLASS_SPACE 11 69 #define CCLASS_OTHER 12 70 71 /* 72 ** The following table gives the character class for non-initial ASCII 73 ** characters. 74 */ 75 static const unsigned char midClass[] = { 76 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 77 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 78 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 79 /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 80 /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, 81 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 82 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 83 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 84 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 85 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 86 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, 87 /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, 88 /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, 89 /* ' */ CCLASS_SILENT, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, 90 /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, 91 /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, 92 /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, 93 /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, 94 /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, 95 /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, 96 /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, 97 /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, 98 /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, 99 /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, 100 /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, 101 /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, 102 /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, 103 /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, 104 /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, 105 /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_VOWEL, 106 /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, 107 /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, 108 /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, 109 /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, 110 /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, 111 /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, 112 /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, 113 /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, 114 /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, 115 /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, 116 /* x */ CCLASS_C, /* y */ CCLASS_VOWEL, /* z */ CCLASS_C, 117 /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, 118 /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, 119 }; 120 /* 121 ** This tables gives the character class for ASCII characters that form the 122 ** initial character of a word. The only difference from midClass is with 123 ** the letters H, W, and Y. 124 */ 125 static const unsigned char initClass[] = { 126 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 127 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 128 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 129 /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 130 /* */ CCLASS_SPACE, /* */ CCLASS_SPACE, /* */ CCLASS_OTHER, 131 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 132 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 133 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 134 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 135 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, 136 /* */ CCLASS_OTHER, /* */ CCLASS_OTHER, /* */ CCLASS_SPACE, 137 /* ! */ CCLASS_OTHER, /* " */ CCLASS_OTHER, /* # */ CCLASS_OTHER, 138 /* $ */ CCLASS_OTHER, /* % */ CCLASS_OTHER, /* & */ CCLASS_OTHER, 139 /* ' */ CCLASS_OTHER, /* ( */ CCLASS_OTHER, /* ) */ CCLASS_OTHER, 140 /* * */ CCLASS_OTHER, /* + */ CCLASS_OTHER, /* , */ CCLASS_OTHER, 141 /* - */ CCLASS_OTHER, /* . */ CCLASS_OTHER, /* / */ CCLASS_OTHER, 142 /* 0 */ CCLASS_DIGIT, /* 1 */ CCLASS_DIGIT, /* 2 */ CCLASS_DIGIT, 143 /* 3 */ CCLASS_DIGIT, /* 4 */ CCLASS_DIGIT, /* 5 */ CCLASS_DIGIT, 144 /* 6 */ CCLASS_DIGIT, /* 7 */ CCLASS_DIGIT, /* 8 */ CCLASS_DIGIT, 145 /* 9 */ CCLASS_DIGIT, /* : */ CCLASS_OTHER, /* ; */ CCLASS_OTHER, 146 /* < */ CCLASS_OTHER, /* = */ CCLASS_OTHER, /* > */ CCLASS_OTHER, 147 /* ? */ CCLASS_OTHER, /* @ */ CCLASS_OTHER, /* A */ CCLASS_VOWEL, 148 /* B */ CCLASS_B, /* C */ CCLASS_C, /* D */ CCLASS_D, 149 /* E */ CCLASS_VOWEL, /* F */ CCLASS_B, /* G */ CCLASS_C, 150 /* H */ CCLASS_SILENT, /* I */ CCLASS_VOWEL, /* J */ CCLASS_C, 151 /* K */ CCLASS_C, /* L */ CCLASS_L, /* M */ CCLASS_M, 152 /* N */ CCLASS_M, /* O */ CCLASS_VOWEL, /* P */ CCLASS_B, 153 /* Q */ CCLASS_C, /* R */ CCLASS_R, /* S */ CCLASS_C, 154 /* T */ CCLASS_D, /* U */ CCLASS_VOWEL, /* V */ CCLASS_B, 155 /* W */ CCLASS_B, /* X */ CCLASS_C, /* Y */ CCLASS_Y, 156 /* Z */ CCLASS_C, /* [ */ CCLASS_OTHER, /* \ */ CCLASS_OTHER, 157 /* ] */ CCLASS_OTHER, /* ^ */ CCLASS_OTHER, /* _ */ CCLASS_OTHER, 158 /* ` */ CCLASS_OTHER, /* a */ CCLASS_VOWEL, /* b */ CCLASS_B, 159 /* c */ CCLASS_C, /* d */ CCLASS_D, /* e */ CCLASS_VOWEL, 160 /* f */ CCLASS_B, /* g */ CCLASS_C, /* h */ CCLASS_SILENT, 161 /* i */ CCLASS_VOWEL, /* j */ CCLASS_C, /* k */ CCLASS_C, 162 /* l */ CCLASS_L, /* m */ CCLASS_M, /* n */ CCLASS_M, 163 /* o */ CCLASS_VOWEL, /* p */ CCLASS_B, /* q */ CCLASS_C, 164 /* r */ CCLASS_R, /* s */ CCLASS_C, /* t */ CCLASS_D, 165 /* u */ CCLASS_VOWEL, /* v */ CCLASS_B, /* w */ CCLASS_B, 166 /* x */ CCLASS_C, /* y */ CCLASS_Y, /* z */ CCLASS_C, 167 /* { */ CCLASS_OTHER, /* | */ CCLASS_OTHER, /* } */ CCLASS_OTHER, 168 /* ~ */ CCLASS_OTHER, /* */ CCLASS_OTHER, 169 }; 170 171 /* 172 ** Mapping from the character class number (0-13) to a symbol for each 173 ** character class. Note that initClass[] can be used to map the class 174 ** symbol back into the class number. 175 */ 176 static const unsigned char className[] = ".ABCDHLRMY9 ?"; 177 178 /* 179 ** Generate a "phonetic hash" from a string of ASCII characters 180 ** in zIn[0..nIn-1]. 181 ** 182 ** * Map characters by character class as defined above. 183 ** * Omit double-letters 184 ** * Omit vowels beside R and L 185 ** * Omit T when followed by CH 186 ** * Omit W when followed by R 187 ** * Omit D when followed by J or G 188 ** * Omit K in KN or G in GN at the beginning of a word 189 ** 190 ** Space to hold the result is obtained from sqlite3_malloc() 191 ** 192 ** Return NULL if memory allocation fails. 193 */ 194 static unsigned char *phoneticHash(const unsigned char *zIn, int nIn){ 195 unsigned char *zOut = sqlite3_malloc64( nIn + 1 ); 196 int i; 197 int nOut = 0; 198 char cPrev = 0x77; 199 char cPrevX = 0x77; 200 const unsigned char *aClass = initClass; 201 202 if( zOut==0 ) return 0; 203 if( nIn>2 ){ 204 switch( zIn[0] ){ 205 case 'g': 206 case 'k': { 207 if( zIn[1]=='n' ){ zIn++; nIn--; } 208 break; 209 } 210 } 211 } 212 for(i=0; i<nIn; i++){ 213 unsigned char c = zIn[i]; 214 if( i+1<nIn ){ 215 if( c=='w' && zIn[i+1]=='r' ) continue; 216 if( c=='d' && (zIn[i+1]=='j' || zIn[i+1]=='g') ) continue; 217 if( i+2<nIn ){ 218 if( c=='t' && zIn[i+1]=='c' && zIn[i+2]=='h' ) continue; 219 } 220 } 221 c = aClass[c&0x7f]; 222 if( c==CCLASS_SPACE ) continue; 223 if( c==CCLASS_OTHER && cPrev!=CCLASS_DIGIT ) continue; 224 aClass = midClass; 225 if( c==CCLASS_VOWEL && (cPrevX==CCLASS_R || cPrevX==CCLASS_L) ){ 226 continue; /* No vowels beside L or R */ 227 } 228 if( (c==CCLASS_R || c==CCLASS_L) && cPrevX==CCLASS_VOWEL ){ 229 nOut--; /* No vowels beside L or R */ 230 } 231 cPrev = c; 232 if( c==CCLASS_SILENT ) continue; 233 cPrevX = c; 234 c = className[c]; 235 assert( nOut>=0 ); 236 if( nOut==0 || c!=zOut[nOut-1] ) zOut[nOut++] = c; 237 } 238 zOut[nOut] = 0; 239 return zOut; 240 } 241 242 /* 243 ** This is an SQL function wrapper around phoneticHash(). See 244 ** the description of phoneticHash() for additional information. 245 */ 246 static void phoneticHashSqlFunc( 247 sqlite3_context *context, 248 int argc, 249 sqlite3_value **argv 250 ){ 251 const unsigned char *zIn; 252 unsigned char *zOut; 253 254 zIn = sqlite3_value_text(argv[0]); 255 if( zIn==0 ) return; 256 zOut = phoneticHash(zIn, sqlite3_value_bytes(argv[0])); 257 if( zOut==0 ){ 258 sqlite3_result_error_nomem(context); 259 }else{ 260 sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); 261 } 262 } 263 264 /* 265 ** Return the character class number for a character given its 266 ** context. 267 */ 268 static char characterClass(char cPrev, char c){ 269 return cPrev==0 ? initClass[c&0x7f] : midClass[c&0x7f]; 270 } 271 272 /* 273 ** Return the cost of inserting or deleting character c immediately 274 ** following character cPrev. If cPrev==0, that means c is the first 275 ** character of the word. 276 */ 277 static int insertOrDeleteCost(char cPrev, char c, char cNext){ 278 char classC = characterClass(cPrev, c); 279 char classCprev; 280 281 if( classC==CCLASS_SILENT ){ 282 /* Insert or delete "silent" characters such as H or W */ 283 return 1; 284 } 285 if( cPrev==c ){ 286 /* Repeated characters, or miss a repeat */ 287 return 10; 288 } 289 if( classC==CCLASS_VOWEL && (cPrev=='r' || cNext=='r') ){ 290 return 20; /* Insert a vowel before or after 'r' */ 291 } 292 classCprev = characterClass(cPrev, cPrev); 293 if( classC==classCprev ){ 294 if( classC==CCLASS_VOWEL ){ 295 /* Remove or add a new vowel to a vowel cluster */ 296 return 15; 297 }else{ 298 /* Remove or add a consonant not in the same class */ 299 return 50; 300 } 301 } 302 303 /* any other character insertion or deletion */ 304 return 100; 305 } 306 307 /* 308 ** Divide the insertion cost by this factor when appending to the 309 ** end of the word. 310 */ 311 #define FINAL_INS_COST_DIV 4 312 313 /* 314 ** Return the cost of substituting cTo in place of cFrom assuming 315 ** the previous character is cPrev. If cPrev==0 then cTo is the first 316 ** character of the word. 317 */ 318 static int substituteCost(char cPrev, char cFrom, char cTo){ 319 char classFrom, classTo; 320 if( cFrom==cTo ){ 321 /* Exact match */ 322 return 0; 323 } 324 if( cFrom==(cTo^0x20) && ((cTo>='A' && cTo<='Z') || (cTo>='a' && cTo<='z')) ){ 325 /* differ only in case */ 326 return 0; 327 } 328 classFrom = characterClass(cPrev, cFrom); 329 classTo = characterClass(cPrev, cTo); 330 if( classFrom==classTo ){ 331 /* Same character class */ 332 return 40; 333 } 334 if( classFrom>=CCLASS_B && classFrom<=CCLASS_Y 335 && classTo>=CCLASS_B && classTo<=CCLASS_Y ){ 336 /* Convert from one consonant to another, but in a different class */ 337 return 75; 338 } 339 /* Any other subsitution */ 340 return 100; 341 } 342 343 /* 344 ** Given two strings zA and zB which are pure ASCII, return the cost 345 ** of transforming zA into zB. If zA ends with '*' assume that it is 346 ** a prefix of zB and give only minimal penalty for extra characters 347 ** on the end of zB. 348 ** 349 ** Smaller numbers mean a closer match. 350 ** 351 ** Negative values indicate an error: 352 ** -1 One of the inputs is NULL 353 ** -2 Non-ASCII characters on input 354 ** -3 Unable to allocate memory 355 ** 356 ** If pnMatch is not NULL, then *pnMatch is set to the number of bytes 357 ** of zB that matched the pattern in zA. If zA does not end with a '*', 358 ** then this value is always the number of bytes in zB (i.e. strlen(zB)). 359 ** If zA does end in a '*', then it is the number of bytes in the prefix 360 ** of zB that was deemed to match zA. 361 */ 362 static int editdist1(const char *zA, const char *zB, int *pnMatch){ 363 int nA, nB; /* Number of characters in zA[] and zB[] */ 364 int xA, xB; /* Loop counters for zA[] and zB[] */ 365 char cA = 0, cB; /* Current character of zA and zB */ 366 char cAprev, cBprev; /* Previous character of zA and zB */ 367 char cAnext, cBnext; /* Next character in zA and zB */ 368 int d; /* North-west cost value */ 369 int dc = 0; /* North-west character value */ 370 int res; /* Final result */ 371 int *m; /* The cost matrix */ 372 char *cx; /* Corresponding character values */ 373 int *toFree = 0; /* Malloced space */ 374 int nMatch = 0; 375 int mStack[60+15]; /* Stack space to use if not too much is needed */ 376 377 /* Early out if either input is NULL */ 378 if( zA==0 || zB==0 ) return -1; 379 380 /* Skip any common prefix */ 381 while( zA[0] && zA[0]==zB[0] ){ dc = zA[0]; zA++; zB++; nMatch++; } 382 if( pnMatch ) *pnMatch = nMatch; 383 if( zA[0]==0 && zB[0]==0 ) return 0; 384 385 #if 0 386 printf("A=\"%s\" B=\"%s\" dc=%c\n", zA, zB, dc?dc:' '); 387 #endif 388 389 /* Verify input strings and measure their lengths */ 390 for(nA=0; zA[nA]; nA++){ 391 if( zA[nA]&0x80 ) return -2; 392 } 393 for(nB=0; zB[nB]; nB++){ 394 if( zB[nB]&0x80 ) return -2; 395 } 396 397 /* Special processing if either string is empty */ 398 if( nA==0 ){ 399 cBprev = (char)dc; 400 for(xB=res=0; (cB = zB[xB])!=0; xB++){ 401 res += insertOrDeleteCost(cBprev, cB, zB[xB+1])/FINAL_INS_COST_DIV; 402 cBprev = cB; 403 } 404 return res; 405 } 406 if( nB==0 ){ 407 cAprev = (char)dc; 408 for(xA=res=0; (cA = zA[xA])!=0; xA++){ 409 res += insertOrDeleteCost(cAprev, cA, zA[xA+1]); 410 cAprev = cA; 411 } 412 return res; 413 } 414 415 /* A is a prefix of B */ 416 if( zA[0]=='*' && zA[1]==0 ) return 0; 417 418 /* Allocate and initialize the Wagner matrix */ 419 if( nB<(sizeof(mStack)*4)/(sizeof(mStack[0])*5) ){ 420 m = mStack; 421 }else{ 422 m = toFree = sqlite3_malloc64( (nB+1)*5*sizeof(m[0])/4 ); 423 if( m==0 ) return -3; 424 } 425 cx = (char*)&m[nB+1]; 426 427 /* Compute the Wagner edit distance */ 428 m[0] = 0; 429 cx[0] = (char)dc; 430 cBprev = (char)dc; 431 for(xB=1; xB<=nB; xB++){ 432 cBnext = zB[xB]; 433 cB = zB[xB-1]; 434 cx[xB] = cB; 435 m[xB] = m[xB-1] + insertOrDeleteCost(cBprev, cB, cBnext); 436 cBprev = cB; 437 } 438 cAprev = (char)dc; 439 for(xA=1; xA<=nA; xA++){ 440 int lastA = (xA==nA); 441 cA = zA[xA-1]; 442 cAnext = zA[xA]; 443 if( cA=='*' && lastA ) break; 444 d = m[0]; 445 dc = cx[0]; 446 m[0] = d + insertOrDeleteCost(cAprev, cA, cAnext); 447 cBprev = 0; 448 for(xB=1; xB<=nB; xB++){ 449 int totalCost, insCost, delCost, subCost, ncx; 450 cB = zB[xB-1]; 451 cBnext = zB[xB]; 452 453 /* Cost to insert cB */ 454 insCost = insertOrDeleteCost(cx[xB-1], cB, cBnext); 455 if( lastA ) insCost /= FINAL_INS_COST_DIV; 456 457 /* Cost to delete cA */ 458 delCost = insertOrDeleteCost(cx[xB], cA, cBnext); 459 460 /* Cost to substitute cA->cB */ 461 subCost = substituteCost(cx[xB-1], cA, cB); 462 463 /* Best cost */ 464 totalCost = insCost + m[xB-1]; 465 ncx = cB; 466 if( (delCost + m[xB])<totalCost ){ 467 totalCost = delCost + m[xB]; 468 ncx = cA; 469 } 470 if( (subCost + d)<totalCost ){ 471 totalCost = subCost + d; 472 } 473 474 #if 0 475 printf("%d,%d d=%4d u=%4d r=%4d dc=%c cA=%c cB=%c" 476 " ins=%4d del=%4d sub=%4d t=%4d ncx=%c\n", 477 xA, xB, d, m[xB], m[xB-1], dc?dc:' ', cA, cB, 478 insCost, delCost, subCost, totalCost, ncx?ncx:' '); 479 #endif 480 481 /* Update the matrix */ 482 d = m[xB]; 483 dc = cx[xB]; 484 m[xB] = totalCost; 485 cx[xB] = (char)ncx; 486 cBprev = cB; 487 } 488 cAprev = cA; 489 } 490 491 /* Free the wagner matrix and return the result */ 492 if( cA=='*' ){ 493 res = m[1]; 494 for(xB=1; xB<=nB; xB++){ 495 if( m[xB]<res ){ 496 res = m[xB]; 497 if( pnMatch ) *pnMatch = xB+nMatch; 498 } 499 } 500 }else{ 501 res = m[nB]; 502 /* In the current implementation, pnMatch is always NULL if zA does 503 ** not end in "*" */ 504 assert( pnMatch==0 ); 505 } 506 sqlite3_free(toFree); 507 return res; 508 } 509 510 /* 511 ** Function: editdist(A,B) 512 ** 513 ** Return the cost of transforming string A into string B. Both strings 514 ** must be pure ASCII text. If A ends with '*' then it is assumed to be 515 ** a prefix of B and extra characters on the end of B have minimal additional 516 ** cost. 517 */ 518 static void editdistSqlFunc( 519 sqlite3_context *context, 520 int argc, 521 sqlite3_value **argv 522 ){ 523 int res = editdist1( 524 (const char*)sqlite3_value_text(argv[0]), 525 (const char*)sqlite3_value_text(argv[1]), 526 0); 527 if( res<0 ){ 528 if( res==(-3) ){ 529 sqlite3_result_error_nomem(context); 530 }else if( res==(-2) ){ 531 sqlite3_result_error(context, "non-ASCII input to editdist()", -1); 532 }else{ 533 sqlite3_result_error(context, "NULL input to editdist()", -1); 534 } 535 }else{ 536 sqlite3_result_int(context, res); 537 } 538 } 539 540 /* End of the fixed-cost edit distance implementation 541 ****************************************************************************** 542 ***************************************************************************** 543 ** Begin: Configurable cost unicode edit distance routines 544 */ 545 /* Forward declaration of structures */ 546 typedef struct EditDist3Cost EditDist3Cost; 547 typedef struct EditDist3Config EditDist3Config; 548 typedef struct EditDist3Point EditDist3Point; 549 typedef struct EditDist3From EditDist3From; 550 typedef struct EditDist3FromString EditDist3FromString; 551 typedef struct EditDist3To EditDist3To; 552 typedef struct EditDist3ToString EditDist3ToString; 553 typedef struct EditDist3Lang EditDist3Lang; 554 555 556 /* 557 ** An entry in the edit cost table 558 */ 559 struct EditDist3Cost { 560 EditDist3Cost *pNext; /* Next cost element */ 561 u8 nFrom; /* Number of bytes in aFrom */ 562 u8 nTo; /* Number of bytes in aTo */ 563 u16 iCost; /* Cost of this transformation */ 564 char a[4] ; /* FROM string followed by TO string */ 565 /* Additional TO and FROM string bytes appended as necessary */ 566 }; 567 568 /* 569 ** Edit costs for a particular language ID 570 */ 571 struct EditDist3Lang { 572 int iLang; /* Language ID */ 573 int iInsCost; /* Default insertion cost */ 574 int iDelCost; /* Default deletion cost */ 575 int iSubCost; /* Default substitution cost */ 576 EditDist3Cost *pCost; /* Costs */ 577 }; 578 579 580 /* 581 ** The default EditDist3Lang object, with default costs. 582 */ 583 static const EditDist3Lang editDist3Lang = { 0, 100, 100, 150, 0 }; 584 585 /* 586 ** Complete configuration 587 */ 588 struct EditDist3Config { 589 int nLang; /* Number of language IDs. Size of a[] */ 590 EditDist3Lang *a; /* One for each distinct language ID */ 591 }; 592 593 /* 594 ** Extra information about each character in the FROM string. 595 */ 596 struct EditDist3From { 597 int nSubst; /* Number of substitution cost entries */ 598 int nDel; /* Number of deletion cost entries */ 599 int nByte; /* Number of bytes in this character */ 600 EditDist3Cost **apSubst; /* Array of substitution costs for this element */ 601 EditDist3Cost **apDel; /* Array of deletion cost entries */ 602 }; 603 604 /* 605 ** A precompiled FROM string. 606 * 607 ** In the common case we expect the FROM string to be reused multiple times. 608 ** In other words, the common case will be to measure the edit distance 609 ** from a single origin string to multiple target strings. 610 */ 611 struct EditDist3FromString { 612 char *z; /* The complete text of the FROM string */ 613 int n; /* Number of characters in the FROM string */ 614 int isPrefix; /* True if ends with '*' character */ 615 EditDist3From *a; /* Extra info about each char of the FROM string */ 616 }; 617 618 /* 619 ** Extra information about each character in the TO string. 620 */ 621 struct EditDist3To { 622 int nIns; /* Number of insertion cost entries */ 623 int nByte; /* Number of bytes in this character */ 624 EditDist3Cost **apIns; /* Array of deletion cost entries */ 625 }; 626 627 /* 628 ** A precompiled FROM string 629 */ 630 struct EditDist3ToString { 631 char *z; /* The complete text of the TO string */ 632 int n; /* Number of characters in the TO string */ 633 EditDist3To *a; /* Extra info about each char of the TO string */ 634 }; 635 636 /* 637 ** Clear or delete an instance of the object that records all edit-distance 638 ** weights. 639 */ 640 static void editDist3ConfigClear(EditDist3Config *p){ 641 int i; 642 if( p==0 ) return; 643 for(i=0; i<p->nLang; i++){ 644 EditDist3Cost *pCost, *pNext; 645 pCost = p->a[i].pCost; 646 while( pCost ){ 647 pNext = pCost->pNext; 648 sqlite3_free(pCost); 649 pCost = pNext; 650 } 651 } 652 sqlite3_free(p->a); 653 memset(p, 0, sizeof(*p)); 654 } 655 static void editDist3ConfigDelete(void *pIn){ 656 EditDist3Config *p = (EditDist3Config*)pIn; 657 editDist3ConfigClear(p); 658 sqlite3_free(p); 659 } 660 661 /* 662 ** Load all edit-distance weights from a table. 663 */ 664 static int editDist3ConfigLoad( 665 EditDist3Config *p, /* The edit distance configuration to load */ 666 sqlite3 *db, /* Load from this database */ 667 const char *zTable /* Name of the table from which to load */ 668 ){ 669 sqlite3_stmt *pStmt; 670 int rc, rc2; 671 char *zSql; 672 int iLangPrev = -9999; 673 EditDist3Lang *pLang = 0; 674 675 zSql = sqlite3_mprintf("SELECT iLang, cFrom, cTo, iCost" 676 " FROM \"%w\" WHERE iLang>=0 ORDER BY iLang", zTable); 677 if( zSql==0 ) return SQLITE_NOMEM; 678 rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); 679 sqlite3_free(zSql); 680 if( rc ) return rc; 681 editDist3ConfigClear(p); 682 while( sqlite3_step(pStmt)==SQLITE_ROW ){ 683 int iLang = sqlite3_column_int(pStmt, 0); 684 const char *zFrom = (const char*)sqlite3_column_text(pStmt, 1); 685 int nFrom = zFrom ? sqlite3_column_bytes(pStmt, 1) : 0; 686 const char *zTo = (const char*)sqlite3_column_text(pStmt, 2); 687 int nTo = zTo ? sqlite3_column_bytes(pStmt, 2) : 0; 688 int iCost = sqlite3_column_int(pStmt, 3); 689 690 assert( zFrom!=0 || nFrom==0 ); 691 assert( zTo!=0 || nTo==0 ); 692 if( nFrom>100 || nTo>100 ) continue; 693 if( iCost<0 ) continue; 694 if( pLang==0 || iLang!=iLangPrev ){ 695 EditDist3Lang *pNew; 696 pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0])); 697 if( pNew==0 ){ rc = SQLITE_NOMEM; break; } 698 p->a = pNew; 699 pLang = &p->a[p->nLang]; 700 p->nLang++; 701 pLang->iLang = iLang; 702 pLang->iInsCost = 100; 703 pLang->iDelCost = 100; 704 pLang->iSubCost = 150; 705 pLang->pCost = 0; 706 iLangPrev = iLang; 707 } 708 if( nFrom==1 && zFrom[0]=='?' && nTo==0 ){ 709 pLang->iDelCost = iCost; 710 }else if( nFrom==0 && nTo==1 && zTo[0]=='?' ){ 711 pLang->iInsCost = iCost; 712 }else if( nFrom==1 && nTo==1 && zFrom[0]=='?' && zTo[0]=='?' ){ 713 pLang->iSubCost = iCost; 714 }else{ 715 EditDist3Cost *pCost; 716 int nExtra = nFrom + nTo - 4; 717 if( nExtra<0 ) nExtra = 0; 718 pCost = sqlite3_malloc64( sizeof(*pCost) + nExtra ); 719 if( pCost==0 ){ rc = SQLITE_NOMEM; break; } 720 pCost->nFrom = (u8)nFrom; 721 pCost->nTo = (u8)nTo; 722 pCost->iCost = (u16)iCost; 723 memcpy(pCost->a, zFrom, nFrom); 724 memcpy(pCost->a + nFrom, zTo, nTo); 725 pCost->pNext = pLang->pCost; 726 pLang->pCost = pCost; 727 } 728 } 729 rc2 = sqlite3_finalize(pStmt); 730 if( rc==SQLITE_OK ) rc = rc2; 731 return rc; 732 } 733 734 /* 735 ** Return the length (in bytes) of a utf-8 character. Or return a maximum 736 ** of N. 737 */ 738 static int utf8Len(unsigned char c, int N){ 739 int len = 1; 740 if( c>0x7f ){ 741 if( (c&0xe0)==0xc0 ){ 742 len = 2; 743 }else if( (c&0xf0)==0xe0 ){ 744 len = 3; 745 }else{ 746 len = 4; 747 } 748 } 749 if( len>N ) len = N; 750 return len; 751 } 752 753 /* 754 ** Return TRUE (non-zero) if the To side of the given cost matches 755 ** the given string. 756 */ 757 static int matchTo(EditDist3Cost *p, const char *z, int n){ 758 if( p->nTo>n ) return 0; 759 if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0; 760 return 1; 761 } 762 763 /* 764 ** Return TRUE (non-zero) if the From side of the given cost matches 765 ** the given string. 766 */ 767 static int matchFrom(EditDist3Cost *p, const char *z, int n){ 768 assert( p->nFrom<=n ); 769 if( strncmp(p->a, z, p->nFrom)!=0 ) return 0; 770 return 1; 771 } 772 773 /* 774 ** Return TRUE (non-zero) of the next FROM character and the next TO 775 ** character are the same. 776 */ 777 static int matchFromTo( 778 EditDist3FromString *pStr, /* Left hand string */ 779 int n1, /* Index of comparison character on the left */ 780 const char *z2, /* Right-handl comparison character */ 781 int n2 /* Bytes remaining in z2[] */ 782 ){ 783 int b1 = pStr->a[n1].nByte; 784 if( b1>n2 ) return 0; 785 if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0; 786 return 1; 787 } 788 789 /* 790 ** Delete an EditDist3FromString objecct 791 */ 792 static void editDist3FromStringDelete(EditDist3FromString *p){ 793 int i; 794 if( p ){ 795 for(i=0; i<p->n; i++){ 796 sqlite3_free(p->a[i].apDel); 797 sqlite3_free(p->a[i].apSubst); 798 } 799 sqlite3_free(p); 800 } 801 } 802 803 /* 804 ** Create a EditDist3FromString object. 805 */ 806 static EditDist3FromString *editDist3FromStringNew( 807 const EditDist3Lang *pLang, 808 const char *z, 809 int n 810 ){ 811 EditDist3FromString *pStr; 812 EditDist3Cost *p; 813 int i; 814 815 if( z==0 ) return 0; 816 if( n<0 ) n = (int)strlen(z); 817 pStr = sqlite3_malloc64( sizeof(*pStr) + sizeof(pStr->a[0])*n + n + 1 ); 818 if( pStr==0 ) return 0; 819 pStr->a = (EditDist3From*)&pStr[1]; 820 memset(pStr->a, 0, sizeof(pStr->a[0])*n); 821 pStr->n = n; 822 pStr->z = (char*)&pStr->a[n]; 823 memcpy(pStr->z, z, n+1); 824 if( n && z[n-1]=='*' ){ 825 pStr->isPrefix = 1; 826 n--; 827 pStr->n--; 828 pStr->z[n] = 0; 829 }else{ 830 pStr->isPrefix = 0; 831 } 832 833 for(i=0; i<n; i++){ 834 EditDist3From *pFrom = &pStr->a[i]; 835 memset(pFrom, 0, sizeof(*pFrom)); 836 pFrom->nByte = utf8Len((unsigned char)z[i], n-i); 837 for(p=pLang->pCost; p; p=p->pNext){ 838 EditDist3Cost **apNew; 839 if( i+p->nFrom>n ) continue; 840 if( matchFrom(p, z+i, n-i)==0 ) continue; 841 if( p->nTo==0 ){ 842 apNew = sqlite3_realloc64(pFrom->apDel, 843 sizeof(*apNew)*(pFrom->nDel+1)); 844 if( apNew==0 ) break; 845 pFrom->apDel = apNew; 846 apNew[pFrom->nDel++] = p; 847 }else{ 848 apNew = sqlite3_realloc64(pFrom->apSubst, 849 sizeof(*apNew)*(pFrom->nSubst+1)); 850 if( apNew==0 ) break; 851 pFrom->apSubst = apNew; 852 apNew[pFrom->nSubst++] = p; 853 } 854 } 855 if( p ){ 856 editDist3FromStringDelete(pStr); 857 pStr = 0; 858 break; 859 } 860 } 861 return pStr; 862 } 863 864 /* 865 ** Update entry m[i] such that it is the minimum of its current value 866 ** and m[j]+iCost. 867 ** 868 ** If the iCost is 1,000,000 or greater, then consider the cost to be 869 ** infinite and skip the update. 870 */ 871 static void updateCost( 872 unsigned int *m, 873 int i, 874 int j, 875 int iCost 876 ){ 877 assert( iCost>=0 ); 878 if( iCost<10000 ){ 879 unsigned int b = m[j] + iCost; 880 if( b<m[i] ) m[i] = b; 881 } 882 } 883 884 /* 885 ** How much stack space (int bytes) to use for Wagner matrix in 886 ** editDist3Core(). If more space than this is required, the entire 887 ** matrix is taken from the heap. To reduce the load on the memory 888 ** allocator, make this value as large as practical for the 889 ** architecture in use. 890 */ 891 #ifndef SQLITE_SPELLFIX_STACKALLOC_SZ 892 # define SQLITE_SPELLFIX_STACKALLOC_SZ (1024) 893 #endif 894 895 /* Compute the edit distance between two strings. 896 ** 897 ** If an error occurs, return a negative number which is the error code. 898 ** 899 ** If pnMatch is not NULL, then *pnMatch is set to the number of characters 900 ** (not bytes) in z2 that matched the search pattern in *pFrom. If pFrom does 901 ** not contain the pattern for a prefix-search, then this is always the number 902 ** of characters in z2. If pFrom does contain a prefix search pattern, then 903 ** it is the number of characters in the prefix of z2 that was deemed to 904 ** match pFrom. 905 */ 906 static int editDist3Core( 907 EditDist3FromString *pFrom, /* The FROM string */ 908 const char *z2, /* The TO string */ 909 int n2, /* Length of the TO string */ 910 const EditDist3Lang *pLang, /* Edit weights for a particular language ID */ 911 int *pnMatch /* OUT: Characters in matched prefix */ 912 ){ 913 int k, n; 914 int i1, b1; 915 int i2, b2; 916 EditDist3FromString f = *pFrom; 917 EditDist3To *a2; 918 unsigned int *m; 919 unsigned int *pToFree; 920 int szRow; 921 EditDist3Cost *p; 922 int res; 923 sqlite3_uint64 nByte; 924 unsigned int stackSpace[SQLITE_SPELLFIX_STACKALLOC_SZ/sizeof(unsigned int)]; 925 926 /* allocate the Wagner matrix and the aTo[] array for the TO string */ 927 n = (f.n+1)*(n2+1); 928 n = (n+1)&~1; 929 nByte = n*sizeof(m[0]) + sizeof(a2[0])*n2; 930 if( nByte<=sizeof(stackSpace) ){ 931 m = stackSpace; 932 pToFree = 0; 933 }else{ 934 m = pToFree = sqlite3_malloc64( nByte ); 935 if( m==0 ) return -1; /* Out of memory */ 936 } 937 a2 = (EditDist3To*)&m[n]; 938 memset(a2, 0, sizeof(a2[0])*n2); 939 940 /* Fill in the a1[] matrix for all characters of the TO string */ 941 for(i2=0; i2<n2; i2++){ 942 a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2); 943 for(p=pLang->pCost; p; p=p->pNext){ 944 EditDist3Cost **apNew; 945 if( p->nFrom>0 ) continue; 946 if( i2+p->nTo>n2 ) continue; 947 if( matchTo(p, z2+i2, n2-i2)==0 ) continue; 948 a2[i2].nIns++; 949 apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); 950 if( apNew==0 ){ 951 res = -1; /* Out of memory */ 952 goto editDist3Abort; 953 } 954 a2[i2].apIns = apNew; 955 a2[i2].apIns[a2[i2].nIns-1] = p; 956 } 957 } 958 959 /* Prepare to compute the minimum edit distance */ 960 szRow = f.n+1; 961 memset(m, 0x01, (n2+1)*szRow*sizeof(m[0])); 962 m[0] = 0; 963 964 /* First fill in the top-row of the matrix with FROM deletion costs */ 965 for(i1=0; i1<f.n; i1 += b1){ 966 b1 = f.a[i1].nByte; 967 updateCost(m, i1+b1, i1, pLang->iDelCost); 968 for(k=0; k<f.a[i1].nDel; k++){ 969 p = f.a[i1].apDel[k]; 970 updateCost(m, i1+p->nFrom, i1, p->iCost); 971 } 972 } 973 974 /* Fill in all subsequent rows, top-to-bottom, left-to-right */ 975 for(i2=0; i2<n2; i2 += b2){ 976 int rx; /* Starting index for current row */ 977 int rxp; /* Starting index for previous row */ 978 b2 = a2[i2].nByte; 979 rx = szRow*(i2+b2); 980 rxp = szRow*i2; 981 updateCost(m, rx, rxp, pLang->iInsCost); 982 for(k=0; k<a2[i2].nIns; k++){ 983 p = a2[i2].apIns[k]; 984 updateCost(m, szRow*(i2+p->nTo), rxp, p->iCost); 985 } 986 for(i1=0; i1<f.n; i1+=b1){ 987 int cx; /* Index of current cell */ 988 int cxp; /* Index of cell immediately to the left */ 989 int cxd; /* Index of cell to the left and one row above */ 990 int cxu; /* Index of cell immediately above */ 991 b1 = f.a[i1].nByte; 992 cxp = rx + i1; 993 cx = cxp + b1; 994 cxd = rxp + i1; 995 cxu = cxd + b1; 996 updateCost(m, cx, cxp, pLang->iDelCost); 997 for(k=0; k<f.a[i1].nDel; k++){ 998 p = f.a[i1].apDel[k]; 999 updateCost(m, cxp+p->nFrom, cxp, p->iCost); 1000 } 1001 updateCost(m, cx, cxu, pLang->iInsCost); 1002 if( matchFromTo(&f, i1, z2+i2, n2-i2) ){ 1003 updateCost(m, cx, cxd, 0); 1004 } 1005 updateCost(m, cx, cxd, pLang->iSubCost); 1006 for(k=0; k<f.a[i1].nSubst; k++){ 1007 p = f.a[i1].apSubst[k]; 1008 if( matchTo(p, z2+i2, n2-i2) ){ 1009 updateCost(m, cxd+p->nFrom+szRow*p->nTo, cxd, p->iCost); 1010 } 1011 } 1012 } 1013 } 1014 1015 #if 0 /* Enable for debugging */ 1016 printf(" ^"); 1017 for(i1=0; i1<f.n; i1++) printf(" %c-%2x", f.z[i1], f.z[i1]&0xff); 1018 printf("\n ^:"); 1019 for(i1=0; i1<szRow; i1++){ 1020 int v = m[i1]; 1021 if( v>9999 ) printf(" ****"); 1022 else printf(" %4d", v); 1023 } 1024 printf("\n"); 1025 for(i2=0; i2<n2; i2++){ 1026 printf("%c-%02x:", z2[i2], z2[i2]&0xff); 1027 for(i1=0; i1<szRow; i1++){ 1028 int v = m[(i2+1)*szRow+i1]; 1029 if( v>9999 ) printf(" ****"); 1030 else printf(" %4d", v); 1031 } 1032 printf("\n"); 1033 } 1034 #endif 1035 1036 /* Free memory allocations and return the result */ 1037 res = (int)m[szRow*(n2+1)-1]; 1038 n = n2; 1039 if( f.isPrefix ){ 1040 for(i2=1; i2<=n2; i2++){ 1041 int b = m[szRow*i2-1]; 1042 if( b<=res ){ 1043 res = b; 1044 n = i2 - 1; 1045 } 1046 } 1047 } 1048 if( pnMatch ){ 1049 int nExtra = 0; 1050 for(k=0; k<n; k++){ 1051 if( (z2[k] & 0xc0)==0x80 ) nExtra++; 1052 } 1053 *pnMatch = n - nExtra; 1054 } 1055 1056 editDist3Abort: 1057 for(i2=0; i2<n2; i2++) sqlite3_free(a2[i2].apIns); 1058 sqlite3_free(pToFree); 1059 return res; 1060 } 1061 1062 /* 1063 ** Get an appropriate EditDist3Lang object. 1064 */ 1065 static const EditDist3Lang *editDist3FindLang( 1066 EditDist3Config *pConfig, 1067 int iLang 1068 ){ 1069 int i; 1070 for(i=0; i<pConfig->nLang; i++){ 1071 if( pConfig->a[i].iLang==iLang ) return &pConfig->a[i]; 1072 } 1073 return &editDist3Lang; 1074 } 1075 1076 /* 1077 ** Function: editdist3(A,B,iLang) 1078 ** editdist3(tablename) 1079 ** 1080 ** Return the cost of transforming string A into string B using edit 1081 ** weights for iLang. 1082 ** 1083 ** The second form loads edit weights into memory from a table. 1084 */ 1085 static void editDist3SqlFunc( 1086 sqlite3_context *context, 1087 int argc, 1088 sqlite3_value **argv 1089 ){ 1090 EditDist3Config *pConfig = (EditDist3Config*)sqlite3_user_data(context); 1091 sqlite3 *db = sqlite3_context_db_handle(context); 1092 int rc; 1093 if( argc==1 ){ 1094 const char *zTable = (const char*)sqlite3_value_text(argv[0]); 1095 rc = editDist3ConfigLoad(pConfig, db, zTable); 1096 if( rc ) sqlite3_result_error_code(context, rc); 1097 }else{ 1098 const char *zA = (const char*)sqlite3_value_text(argv[0]); 1099 const char *zB = (const char*)sqlite3_value_text(argv[1]); 1100 int nA = sqlite3_value_bytes(argv[0]); 1101 int nB = sqlite3_value_bytes(argv[1]); 1102 int iLang = argc==3 ? sqlite3_value_int(argv[2]) : 0; 1103 const EditDist3Lang *pLang = editDist3FindLang(pConfig, iLang); 1104 EditDist3FromString *pFrom; 1105 int dist; 1106 1107 pFrom = editDist3FromStringNew(pLang, zA, nA); 1108 if( pFrom==0 ){ 1109 sqlite3_result_error_nomem(context); 1110 return; 1111 } 1112 dist = editDist3Core(pFrom, zB, nB, pLang, 0); 1113 editDist3FromStringDelete(pFrom); 1114 if( dist==(-1) ){ 1115 sqlite3_result_error_nomem(context); 1116 }else{ 1117 sqlite3_result_int(context, dist); 1118 } 1119 } 1120 } 1121 1122 /* 1123 ** Register the editDist3 function with SQLite 1124 */ 1125 static int editDist3Install(sqlite3 *db){ 1126 int rc; 1127 EditDist3Config *pConfig = sqlite3_malloc64( sizeof(*pConfig) ); 1128 if( pConfig==0 ) return SQLITE_NOMEM; 1129 memset(pConfig, 0, sizeof(*pConfig)); 1130 rc = sqlite3_create_function_v2(db, "editdist3", 1131 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1132 editDist3SqlFunc, 0, 0, 0); 1133 if( rc==SQLITE_OK ){ 1134 rc = sqlite3_create_function_v2(db, "editdist3", 1135 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1136 editDist3SqlFunc, 0, 0, 0); 1137 } 1138 if( rc==SQLITE_OK ){ 1139 rc = sqlite3_create_function_v2(db, "editdist3", 1140 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, 1141 editDist3SqlFunc, 0, 0, editDist3ConfigDelete); 1142 }else{ 1143 sqlite3_free(pConfig); 1144 } 1145 return rc; 1146 } 1147 /* End configurable cost unicode edit distance routines 1148 ****************************************************************************** 1149 ****************************************************************************** 1150 ** Begin transliterate unicode-to-ascii implementation 1151 */ 1152 1153 #if !SQLITE_AMALGAMATION 1154 /* 1155 ** This lookup table is used to help decode the first byte of 1156 ** a multi-byte UTF8 character. 1157 */ 1158 static const unsigned char sqlite3Utf8Trans1[] = { 1159 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1160 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 1161 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 1162 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 1163 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1164 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 1165 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 1166 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, 1167 }; 1168 #endif 1169 1170 /* 1171 ** Return the value of the first UTF-8 character in the string. 1172 */ 1173 static int utf8Read(const unsigned char *z, int n, int *pSize){ 1174 int c, i; 1175 1176 /* All callers to this routine (in the current implementation) 1177 ** always have n>0. */ 1178 if( NEVER(n==0) ){ 1179 c = i = 0; 1180 }else{ 1181 c = z[0]; 1182 i = 1; 1183 if( c>=0xc0 ){ 1184 c = sqlite3Utf8Trans1[c-0xc0]; 1185 while( i<n && (z[i] & 0xc0)==0x80 ){ 1186 c = (c<<6) + (0x3f & z[i++]); 1187 } 1188 } 1189 } 1190 *pSize = i; 1191 return c; 1192 } 1193 1194 /* 1195 ** Return the number of characters in the utf-8 string in the nIn byte 1196 ** buffer pointed to by zIn. 1197 */ 1198 static int utf8Charlen(const char *zIn, int nIn){ 1199 int i; 1200 int nChar = 0; 1201 for(i=0; i<nIn; nChar++){ 1202 int sz; 1203 utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); 1204 i += sz; 1205 } 1206 return nChar; 1207 } 1208 1209 typedef struct Transliteration Transliteration; 1210 struct Transliteration { 1211 unsigned short int cFrom; 1212 unsigned char cTo0, cTo1, cTo2, cTo3; 1213 }; 1214 1215 /* 1216 ** Table of translations from unicode characters into ASCII. 1217 */ 1218 static const Transliteration translit[] = { 1219 { 0x00A0, 0x20, 0x00, 0x00, 0x00 }, /* to */ 1220 { 0x00B5, 0x75, 0x00, 0x00, 0x00 }, /* µ to u */ 1221 { 0x00C0, 0x41, 0x00, 0x00, 0x00 }, /* À to A */ 1222 { 0x00C1, 0x41, 0x00, 0x00, 0x00 }, /* Á to A */ 1223 { 0x00C2, 0x41, 0x00, 0x00, 0x00 }, /*  to A */ 1224 { 0x00C3, 0x41, 0x00, 0x00, 0x00 }, /* à to A */ 1225 { 0x00C4, 0x41, 0x65, 0x00, 0x00 }, /* Ä to Ae */ 1226 { 0x00C5, 0x41, 0x61, 0x00, 0x00 }, /* Å to Aa */ 1227 { 0x00C6, 0x41, 0x45, 0x00, 0x00 }, /* Æ to AE */ 1228 { 0x00C7, 0x43, 0x00, 0x00, 0x00 }, /* Ç to C */ 1229 { 0x00C8, 0x45, 0x00, 0x00, 0x00 }, /* È to E */ 1230 { 0x00C9, 0x45, 0x00, 0x00, 0x00 }, /* É to E */ 1231 { 0x00CA, 0x45, 0x00, 0x00, 0x00 }, /* Ê to E */ 1232 { 0x00CB, 0x45, 0x00, 0x00, 0x00 }, /* Ë to E */ 1233 { 0x00CC, 0x49, 0x00, 0x00, 0x00 }, /* Ì to I */ 1234 { 0x00CD, 0x49, 0x00, 0x00, 0x00 }, /* Í to I */ 1235 { 0x00CE, 0x49, 0x00, 0x00, 0x00 }, /* Î to I */ 1236 { 0x00CF, 0x49, 0x00, 0x00, 0x00 }, /* Ï to I */ 1237 { 0x00D0, 0x44, 0x00, 0x00, 0x00 }, /* Ð to D */ 1238 { 0x00D1, 0x4E, 0x00, 0x00, 0x00 }, /* Ñ to N */ 1239 { 0x00D2, 0x4F, 0x00, 0x00, 0x00 }, /* Ò to O */ 1240 { 0x00D3, 0x4F, 0x00, 0x00, 0x00 }, /* Ó to O */ 1241 { 0x00D4, 0x4F, 0x00, 0x00, 0x00 }, /* Ô to O */ 1242 { 0x00D5, 0x4F, 0x00, 0x00, 0x00 }, /* Õ to O */ 1243 { 0x00D6, 0x4F, 0x65, 0x00, 0x00 }, /* Ö to Oe */ 1244 { 0x00D7, 0x78, 0x00, 0x00, 0x00 }, /* × to x */ 1245 { 0x00D8, 0x4F, 0x00, 0x00, 0x00 }, /* Ø to O */ 1246 { 0x00D9, 0x55, 0x00, 0x00, 0x00 }, /* Ù to U */ 1247 { 0x00DA, 0x55, 0x00, 0x00, 0x00 }, /* Ú to U */ 1248 { 0x00DB, 0x55, 0x00, 0x00, 0x00 }, /* Û to U */ 1249 { 0x00DC, 0x55, 0x65, 0x00, 0x00 }, /* Ü to Ue */ 1250 { 0x00DD, 0x59, 0x00, 0x00, 0x00 }, /* Ý to Y */ 1251 { 0x00DE, 0x54, 0x68, 0x00, 0x00 }, /* Þ to Th */ 1252 { 0x00DF, 0x73, 0x73, 0x00, 0x00 }, /* ß to ss */ 1253 { 0x00E0, 0x61, 0x00, 0x00, 0x00 }, /* à to a */ 1254 { 0x00E1, 0x61, 0x00, 0x00, 0x00 }, /* á to a */ 1255 { 0x00E2, 0x61, 0x00, 0x00, 0x00 }, /* â to a */ 1256 { 0x00E3, 0x61, 0x00, 0x00, 0x00 }, /* ã to a */ 1257 { 0x00E4, 0x61, 0x65, 0x00, 0x00 }, /* ä to ae */ 1258 { 0x00E5, 0x61, 0x61, 0x00, 0x00 }, /* å to aa */ 1259 { 0x00E6, 0x61, 0x65, 0x00, 0x00 }, /* æ to ae */ 1260 { 0x00E7, 0x63, 0x00, 0x00, 0x00 }, /* ç to c */ 1261 { 0x00E8, 0x65, 0x00, 0x00, 0x00 }, /* è to e */ 1262 { 0x00E9, 0x65, 0x00, 0x00, 0x00 }, /* é to e */ 1263 { 0x00EA, 0x65, 0x00, 0x00, 0x00 }, /* ê to e */ 1264 { 0x00EB, 0x65, 0x00, 0x00, 0x00 }, /* ë to e */ 1265 { 0x00EC, 0x69, 0x00, 0x00, 0x00 }, /* ì to i */ 1266 { 0x00ED, 0x69, 0x00, 0x00, 0x00 }, /* í to i */ 1267 { 0x00EE, 0x69, 0x00, 0x00, 0x00 }, /* î to i */ 1268 { 0x00EF, 0x69, 0x00, 0x00, 0x00 }, /* ï to i */ 1269 { 0x00F0, 0x64, 0x00, 0x00, 0x00 }, /* ð to d */ 1270 { 0x00F1, 0x6E, 0x00, 0x00, 0x00 }, /* ñ to n */ 1271 { 0x00F2, 0x6F, 0x00, 0x00, 0x00 }, /* ò to o */ 1272 { 0x00F3, 0x6F, 0x00, 0x00, 0x00 }, /* ó to o */ 1273 { 0x00F4, 0x6F, 0x00, 0x00, 0x00 }, /* ô to o */ 1274 { 0x00F5, 0x6F, 0x00, 0x00, 0x00 }, /* õ to o */ 1275 { 0x00F6, 0x6F, 0x65, 0x00, 0x00 }, /* ö to oe */ 1276 { 0x00F7, 0x3A, 0x00, 0x00, 0x00 }, /* ÷ to : */ 1277 { 0x00F8, 0x6F, 0x00, 0x00, 0x00 }, /* ø to o */ 1278 { 0x00F9, 0x75, 0x00, 0x00, 0x00 }, /* ù to u */ 1279 { 0x00FA, 0x75, 0x00, 0x00, 0x00 }, /* ú to u */ 1280 { 0x00FB, 0x75, 0x00, 0x00, 0x00 }, /* û to u */ 1281 { 0x00FC, 0x75, 0x65, 0x00, 0x00 }, /* ü to ue */ 1282 { 0x00FD, 0x79, 0x00, 0x00, 0x00 }, /* ý to y */ 1283 { 0x00FE, 0x74, 0x68, 0x00, 0x00 }, /* þ to th */ 1284 { 0x00FF, 0x79, 0x00, 0x00, 0x00 }, /* ÿ to y */ 1285 { 0x0100, 0x41, 0x00, 0x00, 0x00 }, /* Ā to A */ 1286 { 0x0101, 0x61, 0x00, 0x00, 0x00 }, /* ā to a */ 1287 { 0x0102, 0x41, 0x00, 0x00, 0x00 }, /* Ă to A */ 1288 { 0x0103, 0x61, 0x00, 0x00, 0x00 }, /* ă to a */ 1289 { 0x0104, 0x41, 0x00, 0x00, 0x00 }, /* Ą to A */ 1290 { 0x0105, 0x61, 0x00, 0x00, 0x00 }, /* ą to a */ 1291 { 0x0106, 0x43, 0x00, 0x00, 0x00 }, /* Ć to C */ 1292 { 0x0107, 0x63, 0x00, 0x00, 0x00 }, /* ć to c */ 1293 { 0x0108, 0x43, 0x68, 0x00, 0x00 }, /* Ĉ to Ch */ 1294 { 0x0109, 0x63, 0x68, 0x00, 0x00 }, /* ĉ to ch */ 1295 { 0x010A, 0x43, 0x00, 0x00, 0x00 }, /* Ċ to C */ 1296 { 0x010B, 0x63, 0x00, 0x00, 0x00 }, /* ċ to c */ 1297 { 0x010C, 0x43, 0x00, 0x00, 0x00 }, /* Č to C */ 1298 { 0x010D, 0x63, 0x00, 0x00, 0x00 }, /* č to c */ 1299 { 0x010E, 0x44, 0x00, 0x00, 0x00 }, /* Ď to D */ 1300 { 0x010F, 0x64, 0x00, 0x00, 0x00 }, /* ď to d */ 1301 { 0x0110, 0x44, 0x00, 0x00, 0x00 }, /* Đ to D */ 1302 { 0x0111, 0x64, 0x00, 0x00, 0x00 }, /* đ to d */ 1303 { 0x0112, 0x45, 0x00, 0x00, 0x00 }, /* Ē to E */ 1304 { 0x0113, 0x65, 0x00, 0x00, 0x00 }, /* ē to e */ 1305 { 0x0114, 0x45, 0x00, 0x00, 0x00 }, /* Ĕ to E */ 1306 { 0x0115, 0x65, 0x00, 0x00, 0x00 }, /* ĕ to e */ 1307 { 0x0116, 0x45, 0x00, 0x00, 0x00 }, /* Ė to E */ 1308 { 0x0117, 0x65, 0x00, 0x00, 0x00 }, /* ė to e */ 1309 { 0x0118, 0x45, 0x00, 0x00, 0x00 }, /* Ę to E */ 1310 { 0x0119, 0x65, 0x00, 0x00, 0x00 }, /* ę to e */ 1311 { 0x011A, 0x45, 0x00, 0x00, 0x00 }, /* Ě to E */ 1312 { 0x011B, 0x65, 0x00, 0x00, 0x00 }, /* ě to e */ 1313 { 0x011C, 0x47, 0x68, 0x00, 0x00 }, /* Ĝ to Gh */ 1314 { 0x011D, 0x67, 0x68, 0x00, 0x00 }, /* ĝ to gh */ 1315 { 0x011E, 0x47, 0x00, 0x00, 0x00 }, /* Ğ to G */ 1316 { 0x011F, 0x67, 0x00, 0x00, 0x00 }, /* ğ to g */ 1317 { 0x0120, 0x47, 0x00, 0x00, 0x00 }, /* Ġ to G */ 1318 { 0x0121, 0x67, 0x00, 0x00, 0x00 }, /* ġ to g */ 1319 { 0x0122, 0x47, 0x00, 0x00, 0x00 }, /* Ģ to G */ 1320 { 0x0123, 0x67, 0x00, 0x00, 0x00 }, /* ģ to g */ 1321 { 0x0124, 0x48, 0x68, 0x00, 0x00 }, /* Ĥ to Hh */ 1322 { 0x0125, 0x68, 0x68, 0x00, 0x00 }, /* ĥ to hh */ 1323 { 0x0126, 0x48, 0x00, 0x00, 0x00 }, /* Ħ to H */ 1324 { 0x0127, 0x68, 0x00, 0x00, 0x00 }, /* ħ to h */ 1325 { 0x0128, 0x49, 0x00, 0x00, 0x00 }, /* Ĩ to I */ 1326 { 0x0129, 0x69, 0x00, 0x00, 0x00 }, /* ĩ to i */ 1327 { 0x012A, 0x49, 0x00, 0x00, 0x00 }, /* Ī to I */ 1328 { 0x012B, 0x69, 0x00, 0x00, 0x00 }, /* ī to i */ 1329 { 0x012C, 0x49, 0x00, 0x00, 0x00 }, /* Ĭ to I */ 1330 { 0x012D, 0x69, 0x00, 0x00, 0x00 }, /* ĭ to i */ 1331 { 0x012E, 0x49, 0x00, 0x00, 0x00 }, /* Į to I */ 1332 { 0x012F, 0x69, 0x00, 0x00, 0x00 }, /* į to i */ 1333 { 0x0130, 0x49, 0x00, 0x00, 0x00 }, /* İ to I */ 1334 { 0x0131, 0x69, 0x00, 0x00, 0x00 }, /* ı to i */ 1335 { 0x0132, 0x49, 0x4A, 0x00, 0x00 }, /* IJ to IJ */ 1336 { 0x0133, 0x69, 0x6A, 0x00, 0x00 }, /* ij to ij */ 1337 { 0x0134, 0x4A, 0x68, 0x00, 0x00 }, /* Ĵ to Jh */ 1338 { 0x0135, 0x6A, 0x68, 0x00, 0x00 }, /* ĵ to jh */ 1339 { 0x0136, 0x4B, 0x00, 0x00, 0x00 }, /* Ķ to K */ 1340 { 0x0137, 0x6B, 0x00, 0x00, 0x00 }, /* ķ to k */ 1341 { 0x0138, 0x6B, 0x00, 0x00, 0x00 }, /* ĸ to k */ 1342 { 0x0139, 0x4C, 0x00, 0x00, 0x00 }, /* Ĺ to L */ 1343 { 0x013A, 0x6C, 0x00, 0x00, 0x00 }, /* ĺ to l */ 1344 { 0x013B, 0x4C, 0x00, 0x00, 0x00 }, /* Ļ to L */ 1345 { 0x013C, 0x6C, 0x00, 0x00, 0x00 }, /* ļ to l */ 1346 { 0x013D, 0x4C, 0x00, 0x00, 0x00 }, /* Ľ to L */ 1347 { 0x013E, 0x6C, 0x00, 0x00, 0x00 }, /* ľ to l */ 1348 { 0x013F, 0x4C, 0x2E, 0x00, 0x00 }, /* Ŀ to L. */ 1349 { 0x0140, 0x6C, 0x2E, 0x00, 0x00 }, /* ŀ to l. */ 1350 { 0x0141, 0x4C, 0x00, 0x00, 0x00 }, /* Ł to L */ 1351 { 0x0142, 0x6C, 0x00, 0x00, 0x00 }, /* ł to l */ 1352 { 0x0143, 0x4E, 0x00, 0x00, 0x00 }, /* Ń to N */ 1353 { 0x0144, 0x6E, 0x00, 0x00, 0x00 }, /* ń to n */ 1354 { 0x0145, 0x4E, 0x00, 0x00, 0x00 }, /* Ņ to N */ 1355 { 0x0146, 0x6E, 0x00, 0x00, 0x00 }, /* ņ to n */ 1356 { 0x0147, 0x4E, 0x00, 0x00, 0x00 }, /* Ň to N */ 1357 { 0x0148, 0x6E, 0x00, 0x00, 0x00 }, /* ň to n */ 1358 { 0x0149, 0x27, 0x6E, 0x00, 0x00 }, /* ʼn to 'n */ 1359 { 0x014A, 0x4E, 0x47, 0x00, 0x00 }, /* Ŋ to NG */ 1360 { 0x014B, 0x6E, 0x67, 0x00, 0x00 }, /* ŋ to ng */ 1361 { 0x014C, 0x4F, 0x00, 0x00, 0x00 }, /* Ō to O */ 1362 { 0x014D, 0x6F, 0x00, 0x00, 0x00 }, /* ō to o */ 1363 { 0x014E, 0x4F, 0x00, 0x00, 0x00 }, /* Ŏ to O */ 1364 { 0x014F, 0x6F, 0x00, 0x00, 0x00 }, /* ŏ to o */ 1365 { 0x0150, 0x4F, 0x00, 0x00, 0x00 }, /* Ő to O */ 1366 { 0x0151, 0x6F, 0x00, 0x00, 0x00 }, /* ő to o */ 1367 { 0x0152, 0x4F, 0x45, 0x00, 0x00 }, /* Œ to OE */ 1368 { 0x0153, 0x6F, 0x65, 0x00, 0x00 }, /* œ to oe */ 1369 { 0x0154, 0x52, 0x00, 0x00, 0x00 }, /* Ŕ to R */ 1370 { 0x0155, 0x72, 0x00, 0x00, 0x00 }, /* ŕ to r */ 1371 { 0x0156, 0x52, 0x00, 0x00, 0x00 }, /* Ŗ to R */ 1372 { 0x0157, 0x72, 0x00, 0x00, 0x00 }, /* ŗ to r */ 1373 { 0x0158, 0x52, 0x00, 0x00, 0x00 }, /* Ř to R */ 1374 { 0x0159, 0x72, 0x00, 0x00, 0x00 }, /* ř to r */ 1375 { 0x015A, 0x53, 0x00, 0x00, 0x00 }, /* Ś to S */ 1376 { 0x015B, 0x73, 0x00, 0x00, 0x00 }, /* ś to s */ 1377 { 0x015C, 0x53, 0x68, 0x00, 0x00 }, /* Ŝ to Sh */ 1378 { 0x015D, 0x73, 0x68, 0x00, 0x00 }, /* ŝ to sh */ 1379 { 0x015E, 0x53, 0x00, 0x00, 0x00 }, /* Ş to S */ 1380 { 0x015F, 0x73, 0x00, 0x00, 0x00 }, /* ş to s */ 1381 { 0x0160, 0x53, 0x00, 0x00, 0x00 }, /* Š to S */ 1382 { 0x0161, 0x73, 0x00, 0x00, 0x00 }, /* š to s */ 1383 { 0x0162, 0x54, 0x00, 0x00, 0x00 }, /* Ţ to T */ 1384 { 0x0163, 0x74, 0x00, 0x00, 0x00 }, /* ţ to t */ 1385 { 0x0164, 0x54, 0x00, 0x00, 0x00 }, /* Ť to T */ 1386 { 0x0165, 0x74, 0x00, 0x00, 0x00 }, /* ť to t */ 1387 { 0x0166, 0x54, 0x00, 0x00, 0x00 }, /* Ŧ to T */ 1388 { 0x0167, 0x74, 0x00, 0x00, 0x00 }, /* ŧ to t */ 1389 { 0x0168, 0x55, 0x00, 0x00, 0x00 }, /* Ũ to U */ 1390 { 0x0169, 0x75, 0x00, 0x00, 0x00 }, /* ũ to u */ 1391 { 0x016A, 0x55, 0x00, 0x00, 0x00 }, /* Ū to U */ 1392 { 0x016B, 0x75, 0x00, 0x00, 0x00 }, /* ū to u */ 1393 { 0x016C, 0x55, 0x00, 0x00, 0x00 }, /* Ŭ to U */ 1394 { 0x016D, 0x75, 0x00, 0x00, 0x00 }, /* ŭ to u */ 1395 { 0x016E, 0x55, 0x00, 0x00, 0x00 }, /* Ů to U */ 1396 { 0x016F, 0x75, 0x00, 0x00, 0x00 }, /* ů to u */ 1397 { 0x0170, 0x55, 0x00, 0x00, 0x00 }, /* Ű to U */ 1398 { 0x0171, 0x75, 0x00, 0x00, 0x00 }, /* ű to u */ 1399 { 0x0172, 0x55, 0x00, 0x00, 0x00 }, /* Ų to U */ 1400 { 0x0173, 0x75, 0x00, 0x00, 0x00 }, /* ų to u */ 1401 { 0x0174, 0x57, 0x00, 0x00, 0x00 }, /* Ŵ to W */ 1402 { 0x0175, 0x77, 0x00, 0x00, 0x00 }, /* ŵ to w */ 1403 { 0x0176, 0x59, 0x00, 0x00, 0x00 }, /* Ŷ to Y */ 1404 { 0x0177, 0x79, 0x00, 0x00, 0x00 }, /* ŷ to y */ 1405 { 0x0178, 0x59, 0x00, 0x00, 0x00 }, /* Ÿ to Y */ 1406 { 0x0179, 0x5A, 0x00, 0x00, 0x00 }, /* Ź to Z */ 1407 { 0x017A, 0x7A, 0x00, 0x00, 0x00 }, /* ź to z */ 1408 { 0x017B, 0x5A, 0x00, 0x00, 0x00 }, /* Ż to Z */ 1409 { 0x017C, 0x7A, 0x00, 0x00, 0x00 }, /* ż to z */ 1410 { 0x017D, 0x5A, 0x00, 0x00, 0x00 }, /* Ž to Z */ 1411 { 0x017E, 0x7A, 0x00, 0x00, 0x00 }, /* ž to z */ 1412 { 0x017F, 0x73, 0x00, 0x00, 0x00 }, /* ſ to s */ 1413 { 0x0192, 0x66, 0x00, 0x00, 0x00 }, /* ƒ to f */ 1414 { 0x0218, 0x53, 0x00, 0x00, 0x00 }, /* Ș to S */ 1415 { 0x0219, 0x73, 0x00, 0x00, 0x00 }, /* ș to s */ 1416 { 0x021A, 0x54, 0x00, 0x00, 0x00 }, /* Ț to T */ 1417 { 0x021B, 0x74, 0x00, 0x00, 0x00 }, /* ț to t */ 1418 { 0x0386, 0x41, 0x00, 0x00, 0x00 }, /* Ά to A */ 1419 { 0x0388, 0x45, 0x00, 0x00, 0x00 }, /* Έ to E */ 1420 { 0x0389, 0x49, 0x00, 0x00, 0x00 }, /* Ή to I */ 1421 { 0x038A, 0x49, 0x00, 0x00, 0x00 }, /* Ί to I */ 1422 { 0x038C, 0x4f, 0x00, 0x00, 0x00 }, /* Ό to O */ 1423 { 0x038E, 0x59, 0x00, 0x00, 0x00 }, /* Ύ to Y */ 1424 { 0x038F, 0x4f, 0x00, 0x00, 0x00 }, /* Ώ to O */ 1425 { 0x0390, 0x69, 0x00, 0x00, 0x00 }, /* ΐ to i */ 1426 { 0x0391, 0x41, 0x00, 0x00, 0x00 }, /* Α to A */ 1427 { 0x0392, 0x42, 0x00, 0x00, 0x00 }, /* Β to B */ 1428 { 0x0393, 0x47, 0x00, 0x00, 0x00 }, /* Γ to G */ 1429 { 0x0394, 0x44, 0x00, 0x00, 0x00 }, /* Δ to D */ 1430 { 0x0395, 0x45, 0x00, 0x00, 0x00 }, /* Ε to E */ 1431 { 0x0396, 0x5a, 0x00, 0x00, 0x00 }, /* Ζ to Z */ 1432 { 0x0397, 0x49, 0x00, 0x00, 0x00 }, /* Η to I */ 1433 { 0x0398, 0x54, 0x68, 0x00, 0x00 }, /* Θ to Th */ 1434 { 0x0399, 0x49, 0x00, 0x00, 0x00 }, /* Ι to I */ 1435 { 0x039A, 0x4b, 0x00, 0x00, 0x00 }, /* Κ to K */ 1436 { 0x039B, 0x4c, 0x00, 0x00, 0x00 }, /* Λ to L */ 1437 { 0x039C, 0x4d, 0x00, 0x00, 0x00 }, /* Μ to M */ 1438 { 0x039D, 0x4e, 0x00, 0x00, 0x00 }, /* Ν to N */ 1439 { 0x039E, 0x58, 0x00, 0x00, 0x00 }, /* Ξ to X */ 1440 { 0x039F, 0x4f, 0x00, 0x00, 0x00 }, /* Ο to O */ 1441 { 0x03A0, 0x50, 0x00, 0x00, 0x00 }, /* Π to P */ 1442 { 0x03A1, 0x52, 0x00, 0x00, 0x00 }, /* Ρ to R */ 1443 { 0x03A3, 0x53, 0x00, 0x00, 0x00 }, /* Σ to S */ 1444 { 0x03A4, 0x54, 0x00, 0x00, 0x00 }, /* Τ to T */ 1445 { 0x03A5, 0x59, 0x00, 0x00, 0x00 }, /* Υ to Y */ 1446 { 0x03A6, 0x46, 0x00, 0x00, 0x00 }, /* Φ to F */ 1447 { 0x03A7, 0x43, 0x68, 0x00, 0x00 }, /* Χ to Ch */ 1448 { 0x03A8, 0x50, 0x73, 0x00, 0x00 }, /* Ψ to Ps */ 1449 { 0x03A9, 0x4f, 0x00, 0x00, 0x00 }, /* Ω to O */ 1450 { 0x03AA, 0x49, 0x00, 0x00, 0x00 }, /* Ϊ to I */ 1451 { 0x03AB, 0x59, 0x00, 0x00, 0x00 }, /* Ϋ to Y */ 1452 { 0x03AC, 0x61, 0x00, 0x00, 0x00 }, /* ά to a */ 1453 { 0x03AD, 0x65, 0x00, 0x00, 0x00 }, /* έ to e */ 1454 { 0x03AE, 0x69, 0x00, 0x00, 0x00 }, /* ή to i */ 1455 { 0x03AF, 0x69, 0x00, 0x00, 0x00 }, /* ί to i */ 1456 { 0x03B1, 0x61, 0x00, 0x00, 0x00 }, /* α to a */ 1457 { 0x03B2, 0x62, 0x00, 0x00, 0x00 }, /* β to b */ 1458 { 0x03B3, 0x67, 0x00, 0x00, 0x00 }, /* γ to g */ 1459 { 0x03B4, 0x64, 0x00, 0x00, 0x00 }, /* δ to d */ 1460 { 0x03B5, 0x65, 0x00, 0x00, 0x00 }, /* ε to e */ 1461 { 0x03B6, 0x7a, 0x00, 0x00, 0x00 }, /* ζ to z */ 1462 { 0x03B7, 0x69, 0x00, 0x00, 0x00 }, /* η to i */ 1463 { 0x03B8, 0x74, 0x68, 0x00, 0x00 }, /* θ to th */ 1464 { 0x03B9, 0x69, 0x00, 0x00, 0x00 }, /* ι to i */ 1465 { 0x03BA, 0x6b, 0x00, 0x00, 0x00 }, /* κ to k */ 1466 { 0x03BB, 0x6c, 0x00, 0x00, 0x00 }, /* λ to l */ 1467 { 0x03BC, 0x6d, 0x00, 0x00, 0x00 }, /* μ to m */ 1468 { 0x03BD, 0x6e, 0x00, 0x00, 0x00 }, /* ν to n */ 1469 { 0x03BE, 0x78, 0x00, 0x00, 0x00 }, /* ξ to x */ 1470 { 0x03BF, 0x6f, 0x00, 0x00, 0x00 }, /* ο to o */ 1471 { 0x03C0, 0x70, 0x00, 0x00, 0x00 }, /* π to p */ 1472 { 0x03C1, 0x72, 0x00, 0x00, 0x00 }, /* ρ to r */ 1473 { 0x03C3, 0x73, 0x00, 0x00, 0x00 }, /* σ to s */ 1474 { 0x03C4, 0x74, 0x00, 0x00, 0x00 }, /* τ to t */ 1475 { 0x03C5, 0x79, 0x00, 0x00, 0x00 }, /* υ to y */ 1476 { 0x03C6, 0x66, 0x00, 0x00, 0x00 }, /* φ to f */ 1477 { 0x03C7, 0x63, 0x68, 0x00, 0x00 }, /* χ to ch */ 1478 { 0x03C8, 0x70, 0x73, 0x00, 0x00 }, /* ψ to ps */ 1479 { 0x03C9, 0x6f, 0x00, 0x00, 0x00 }, /* ω to o */ 1480 { 0x03CA, 0x69, 0x00, 0x00, 0x00 }, /* ϊ to i */ 1481 { 0x03CB, 0x79, 0x00, 0x00, 0x00 }, /* ϋ to y */ 1482 { 0x03CC, 0x6f, 0x00, 0x00, 0x00 }, /* ό to o */ 1483 { 0x03CD, 0x79, 0x00, 0x00, 0x00 }, /* ύ to y */ 1484 { 0x03CE, 0x69, 0x00, 0x00, 0x00 }, /* ώ to i */ 1485 { 0x0400, 0x45, 0x00, 0x00, 0x00 }, /* Ѐ to E */ 1486 { 0x0401, 0x45, 0x00, 0x00, 0x00 }, /* Ё to E */ 1487 { 0x0402, 0x44, 0x00, 0x00, 0x00 }, /* Ђ to D */ 1488 { 0x0403, 0x47, 0x00, 0x00, 0x00 }, /* Ѓ to G */ 1489 { 0x0404, 0x45, 0x00, 0x00, 0x00 }, /* Є to E */ 1490 { 0x0405, 0x5a, 0x00, 0x00, 0x00 }, /* Ѕ to Z */ 1491 { 0x0406, 0x49, 0x00, 0x00, 0x00 }, /* І to I */ 1492 { 0x0407, 0x49, 0x00, 0x00, 0x00 }, /* Ї to I */ 1493 { 0x0408, 0x4a, 0x00, 0x00, 0x00 }, /* Ј to J */ 1494 { 0x0409, 0x49, 0x00, 0x00, 0x00 }, /* Љ to I */ 1495 { 0x040A, 0x4e, 0x00, 0x00, 0x00 }, /* Њ to N */ 1496 { 0x040B, 0x44, 0x00, 0x00, 0x00 }, /* Ћ to D */ 1497 { 0x040C, 0x4b, 0x00, 0x00, 0x00 }, /* Ќ to K */ 1498 { 0x040D, 0x49, 0x00, 0x00, 0x00 }, /* Ѝ to I */ 1499 { 0x040E, 0x55, 0x00, 0x00, 0x00 }, /* Ў to U */ 1500 { 0x040F, 0x44, 0x00, 0x00, 0x00 }, /* Џ to D */ 1501 { 0x0410, 0x41, 0x00, 0x00, 0x00 }, /* А to A */ 1502 { 0x0411, 0x42, 0x00, 0x00, 0x00 }, /* Б to B */ 1503 { 0x0412, 0x56, 0x00, 0x00, 0x00 }, /* В to V */ 1504 { 0x0413, 0x47, 0x00, 0x00, 0x00 }, /* Г to G */ 1505 { 0x0414, 0x44, 0x00, 0x00, 0x00 }, /* Д to D */ 1506 { 0x0415, 0x45, 0x00, 0x00, 0x00 }, /* Е to E */ 1507 { 0x0416, 0x5a, 0x68, 0x00, 0x00 }, /* Ж to Zh */ 1508 { 0x0417, 0x5a, 0x00, 0x00, 0x00 }, /* З to Z */ 1509 { 0x0418, 0x49, 0x00, 0x00, 0x00 }, /* И to I */ 1510 { 0x0419, 0x49, 0x00, 0x00, 0x00 }, /* Й to I */ 1511 { 0x041A, 0x4b, 0x00, 0x00, 0x00 }, /* К to K */ 1512 { 0x041B, 0x4c, 0x00, 0x00, 0x00 }, /* Л to L */ 1513 { 0x041C, 0x4d, 0x00, 0x00, 0x00 }, /* М to M */ 1514 { 0x041D, 0x4e, 0x00, 0x00, 0x00 }, /* Н to N */ 1515 { 0x041E, 0x4f, 0x00, 0x00, 0x00 }, /* О to O */ 1516 { 0x041F, 0x50, 0x00, 0x00, 0x00 }, /* П to P */ 1517 { 0x0420, 0x52, 0x00, 0x00, 0x00 }, /* Р to R */ 1518 { 0x0421, 0x53, 0x00, 0x00, 0x00 }, /* С to S */ 1519 { 0x0422, 0x54, 0x00, 0x00, 0x00 }, /* Т to T */ 1520 { 0x0423, 0x55, 0x00, 0x00, 0x00 }, /* У to U */ 1521 { 0x0424, 0x46, 0x00, 0x00, 0x00 }, /* Ф to F */ 1522 { 0x0425, 0x4b, 0x68, 0x00, 0x00 }, /* Х to Kh */ 1523 { 0x0426, 0x54, 0x63, 0x00, 0x00 }, /* Ц to Tc */ 1524 { 0x0427, 0x43, 0x68, 0x00, 0x00 }, /* Ч to Ch */ 1525 { 0x0428, 0x53, 0x68, 0x00, 0x00 }, /* Ш to Sh */ 1526 { 0x0429, 0x53, 0x68, 0x63, 0x68 }, /* Щ to Shch */ 1527 { 0x042A, 0x61, 0x00, 0x00, 0x00 }, /* to A */ 1528 { 0x042B, 0x59, 0x00, 0x00, 0x00 }, /* Ы to Y */ 1529 { 0x042C, 0x59, 0x00, 0x00, 0x00 }, /* to Y */ 1530 { 0x042D, 0x45, 0x00, 0x00, 0x00 }, /* Э to E */ 1531 { 0x042E, 0x49, 0x75, 0x00, 0x00 }, /* Ю to Iu */ 1532 { 0x042F, 0x49, 0x61, 0x00, 0x00 }, /* Я to Ia */ 1533 { 0x0430, 0x61, 0x00, 0x00, 0x00 }, /* а to a */ 1534 { 0x0431, 0x62, 0x00, 0x00, 0x00 }, /* б to b */ 1535 { 0x0432, 0x76, 0x00, 0x00, 0x00 }, /* в to v */ 1536 { 0x0433, 0x67, 0x00, 0x00, 0x00 }, /* г to g */ 1537 { 0x0434, 0x64, 0x00, 0x00, 0x00 }, /* д to d */ 1538 { 0x0435, 0x65, 0x00, 0x00, 0x00 }, /* е to e */ 1539 { 0x0436, 0x7a, 0x68, 0x00, 0x00 }, /* ж to zh */ 1540 { 0x0437, 0x7a, 0x00, 0x00, 0x00 }, /* з to z */ 1541 { 0x0438, 0x69, 0x00, 0x00, 0x00 }, /* и to i */ 1542 { 0x0439, 0x69, 0x00, 0x00, 0x00 }, /* й to i */ 1543 { 0x043A, 0x6b, 0x00, 0x00, 0x00 }, /* к to k */ 1544 { 0x043B, 0x6c, 0x00, 0x00, 0x00 }, /* л to l */ 1545 { 0x043C, 0x6d, 0x00, 0x00, 0x00 }, /* м to m */ 1546 { 0x043D, 0x6e, 0x00, 0x00, 0x00 }, /* н to n */ 1547 { 0x043E, 0x6f, 0x00, 0x00, 0x00 }, /* о to o */ 1548 { 0x043F, 0x70, 0x00, 0x00, 0x00 }, /* п to p */ 1549 { 0x0440, 0x72, 0x00, 0x00, 0x00 }, /* р to r */ 1550 { 0x0441, 0x73, 0x00, 0x00, 0x00 }, /* с to s */ 1551 { 0x0442, 0x74, 0x00, 0x00, 0x00 }, /* т to t */ 1552 { 0x0443, 0x75, 0x00, 0x00, 0x00 }, /* у to u */ 1553 { 0x0444, 0x66, 0x00, 0x00, 0x00 }, /* ф to f */ 1554 { 0x0445, 0x6b, 0x68, 0x00, 0x00 }, /* х to kh */ 1555 { 0x0446, 0x74, 0x63, 0x00, 0x00 }, /* ц to tc */ 1556 { 0x0447, 0x63, 0x68, 0x00, 0x00 }, /* ч to ch */ 1557 { 0x0448, 0x73, 0x68, 0x00, 0x00 }, /* ш to sh */ 1558 { 0x0449, 0x73, 0x68, 0x63, 0x68 }, /* щ to shch */ 1559 { 0x044A, 0x61, 0x00, 0x00, 0x00 }, /* to a */ 1560 { 0x044B, 0x79, 0x00, 0x00, 0x00 }, /* ы to y */ 1561 { 0x044C, 0x79, 0x00, 0x00, 0x00 }, /* to y */ 1562 { 0x044D, 0x65, 0x00, 0x00, 0x00 }, /* э to e */ 1563 { 0x044E, 0x69, 0x75, 0x00, 0x00 }, /* ю to iu */ 1564 { 0x044F, 0x69, 0x61, 0x00, 0x00 }, /* я to ia */ 1565 { 0x0450, 0x65, 0x00, 0x00, 0x00 }, /* ѐ to e */ 1566 { 0x0451, 0x65, 0x00, 0x00, 0x00 }, /* ё to e */ 1567 { 0x0452, 0x64, 0x00, 0x00, 0x00 }, /* ђ to d */ 1568 { 0x0453, 0x67, 0x00, 0x00, 0x00 }, /* ѓ to g */ 1569 { 0x0454, 0x65, 0x00, 0x00, 0x00 }, /* є to e */ 1570 { 0x0455, 0x7a, 0x00, 0x00, 0x00 }, /* ѕ to z */ 1571 { 0x0456, 0x69, 0x00, 0x00, 0x00 }, /* і to i */ 1572 { 0x0457, 0x69, 0x00, 0x00, 0x00 }, /* ї to i */ 1573 { 0x0458, 0x6a, 0x00, 0x00, 0x00 }, /* ј to j */ 1574 { 0x0459, 0x69, 0x00, 0x00, 0x00 }, /* љ to i */ 1575 { 0x045A, 0x6e, 0x00, 0x00, 0x00 }, /* њ to n */ 1576 { 0x045B, 0x64, 0x00, 0x00, 0x00 }, /* ћ to d */ 1577 { 0x045C, 0x6b, 0x00, 0x00, 0x00 }, /* ќ to k */ 1578 { 0x045D, 0x69, 0x00, 0x00, 0x00 }, /* ѝ to i */ 1579 { 0x045E, 0x75, 0x00, 0x00, 0x00 }, /* ў to u */ 1580 { 0x045F, 0x64, 0x00, 0x00, 0x00 }, /* џ to d */ 1581 { 0x1E02, 0x42, 0x00, 0x00, 0x00 }, /* Ḃ to B */ 1582 { 0x1E03, 0x62, 0x00, 0x00, 0x00 }, /* ḃ to b */ 1583 { 0x1E0A, 0x44, 0x00, 0x00, 0x00 }, /* Ḋ to D */ 1584 { 0x1E0B, 0x64, 0x00, 0x00, 0x00 }, /* ḋ to d */ 1585 { 0x1E1E, 0x46, 0x00, 0x00, 0x00 }, /* Ḟ to F */ 1586 { 0x1E1F, 0x66, 0x00, 0x00, 0x00 }, /* ḟ to f */ 1587 { 0x1E40, 0x4D, 0x00, 0x00, 0x00 }, /* Ṁ to M */ 1588 { 0x1E41, 0x6D, 0x00, 0x00, 0x00 }, /* ṁ to m */ 1589 { 0x1E56, 0x50, 0x00, 0x00, 0x00 }, /* Ṗ to P */ 1590 { 0x1E57, 0x70, 0x00, 0x00, 0x00 }, /* ṗ to p */ 1591 { 0x1E60, 0x53, 0x00, 0x00, 0x00 }, /* Ṡ to S */ 1592 { 0x1E61, 0x73, 0x00, 0x00, 0x00 }, /* ṡ to s */ 1593 { 0x1E6A, 0x54, 0x00, 0x00, 0x00 }, /* Ṫ to T */ 1594 { 0x1E6B, 0x74, 0x00, 0x00, 0x00 }, /* ṫ to t */ 1595 { 0x1E80, 0x57, 0x00, 0x00, 0x00 }, /* Ẁ to W */ 1596 { 0x1E81, 0x77, 0x00, 0x00, 0x00 }, /* ẁ to w */ 1597 { 0x1E82, 0x57, 0x00, 0x00, 0x00 }, /* Ẃ to W */ 1598 { 0x1E83, 0x77, 0x00, 0x00, 0x00 }, /* ẃ to w */ 1599 { 0x1E84, 0x57, 0x00, 0x00, 0x00 }, /* Ẅ to W */ 1600 { 0x1E85, 0x77, 0x00, 0x00, 0x00 }, /* ẅ to w */ 1601 { 0x1EF2, 0x59, 0x00, 0x00, 0x00 }, /* Ỳ to Y */ 1602 { 0x1EF3, 0x79, 0x00, 0x00, 0x00 }, /* ỳ to y */ 1603 { 0xFB00, 0x66, 0x66, 0x00, 0x00 }, /* ff to ff */ 1604 { 0xFB01, 0x66, 0x69, 0x00, 0x00 }, /* fi to fi */ 1605 { 0xFB02, 0x66, 0x6C, 0x00, 0x00 }, /* fl to fl */ 1606 { 0xFB05, 0x73, 0x74, 0x00, 0x00 }, /* ſt to st */ 1607 { 0xFB06, 0x73, 0x74, 0x00, 0x00 }, /* st to st */ 1608 }; 1609 1610 static const Transliteration *spellfixFindTranslit(int c, int *pxTop){ 1611 *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1; 1612 return translit; 1613 } 1614 1615 /* 1616 ** Convert the input string from UTF-8 into pure ASCII by converting 1617 ** all non-ASCII characters to some combination of characters in the 1618 ** ASCII subset. 1619 ** 1620 ** The returned string might contain more characters than the input. 1621 ** 1622 ** Space to hold the returned string comes from sqlite3_malloc() and 1623 ** should be freed by the caller. 1624 */ 1625 static unsigned char *transliterate(const unsigned char *zIn, int nIn){ 1626 unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 ); 1627 int c, sz, nOut; 1628 if( zOut==0 ) return 0; 1629 nOut = 0; 1630 while( nIn>0 ){ 1631 c = utf8Read(zIn, nIn, &sz); 1632 zIn += sz; 1633 nIn -= sz; 1634 if( c<=127 ){ 1635 zOut[nOut++] = (unsigned char)c; 1636 }else{ 1637 int xTop, xBtm, x; 1638 const Transliteration *tbl = spellfixFindTranslit(c, &xTop); 1639 xBtm = 0; 1640 while( xTop>=xBtm ){ 1641 x = (xTop + xBtm)/2; 1642 if( tbl[x].cFrom==c ){ 1643 zOut[nOut++] = tbl[x].cTo0; 1644 if( tbl[x].cTo1 ){ 1645 zOut[nOut++] = tbl[x].cTo1; 1646 if( tbl[x].cTo2 ){ 1647 zOut[nOut++] = tbl[x].cTo2; 1648 if( tbl[x].cTo3 ){ 1649 zOut[nOut++] = tbl[x].cTo3; 1650 } 1651 } 1652 } 1653 c = 0; 1654 break; 1655 }else if( tbl[x].cFrom>c ){ 1656 xTop = x-1; 1657 }else{ 1658 xBtm = x+1; 1659 } 1660 } 1661 if( c ) zOut[nOut++] = '?'; 1662 } 1663 } 1664 zOut[nOut] = 0; 1665 return zOut; 1666 } 1667 1668 /* 1669 ** Return the number of characters in the shortest prefix of the input 1670 ** string that transliterates to an ASCII string nTrans bytes or longer. 1671 ** Or, if the transliteration of the input string is less than nTrans 1672 ** bytes in size, return the number of characters in the input string. 1673 */ 1674 static int translen_to_charlen(const char *zIn, int nIn, int nTrans){ 1675 int i, c, sz, nOut; 1676 int nChar; 1677 1678 i = nOut = 0; 1679 for(nChar=0; i<nIn && nOut<nTrans; nChar++){ 1680 c = utf8Read((const unsigned char *)&zIn[i], nIn-i, &sz); 1681 i += sz; 1682 1683 nOut++; 1684 if( c>=128 ){ 1685 int xTop, xBtm, x; 1686 const Transliteration *tbl = spellfixFindTranslit(c, &xTop); 1687 xBtm = 0; 1688 while( xTop>=xBtm ){ 1689 x = (xTop + xBtm)/2; 1690 if( tbl[x].cFrom==c ){ 1691 if( tbl[x].cTo1 ){ 1692 nOut++; 1693 if( tbl[x].cTo2 ){ 1694 nOut++; 1695 if( tbl[x].cTo3 ){ 1696 nOut++; 1697 } 1698 } 1699 } 1700 break; 1701 }else if( tbl[x].cFrom>c ){ 1702 xTop = x-1; 1703 }else{ 1704 xBtm = x+1; 1705 } 1706 } 1707 } 1708 } 1709 1710 return nChar; 1711 } 1712 1713 1714 /* 1715 ** spellfix1_translit(X) 1716 ** 1717 ** Convert a string that contains non-ASCII Roman characters into 1718 ** pure ASCII. 1719 */ 1720 static void transliterateSqlFunc( 1721 sqlite3_context *context, 1722 int argc, 1723 sqlite3_value **argv 1724 ){ 1725 const unsigned char *zIn = sqlite3_value_text(argv[0]); 1726 int nIn = sqlite3_value_bytes(argv[0]); 1727 unsigned char *zOut = transliterate(zIn, nIn); 1728 if( zOut==0 ){ 1729 sqlite3_result_error_nomem(context); 1730 }else{ 1731 sqlite3_result_text(context, (char*)zOut, -1, sqlite3_free); 1732 } 1733 } 1734 1735 /* 1736 ** spellfix1_scriptcode(X) 1737 ** 1738 ** Try to determine the dominant script used by the word X and return 1739 ** its ISO 15924 numeric code. 1740 ** 1741 ** The current implementation only understands the following scripts: 1742 ** 1743 ** 215 (Latin) 1744 ** 220 (Cyrillic) 1745 ** 200 (Greek) 1746 ** 1747 ** This routine will return 998 if the input X contains characters from 1748 ** two or more of the above scripts or 999 if X contains no characters 1749 ** from any of the above scripts. 1750 */ 1751 static void scriptCodeSqlFunc( 1752 sqlite3_context *context, 1753 int argc, 1754 sqlite3_value **argv 1755 ){ 1756 const unsigned char *zIn = sqlite3_value_text(argv[0]); 1757 int nIn = sqlite3_value_bytes(argv[0]); 1758 int c, sz; 1759 int scriptMask = 0; 1760 int res; 1761 int seenDigit = 0; 1762 # define SCRIPT_LATIN 0x0001 1763 # define SCRIPT_CYRILLIC 0x0002 1764 # define SCRIPT_GREEK 0x0004 1765 # define SCRIPT_HEBREW 0x0008 1766 # define SCRIPT_ARABIC 0x0010 1767 1768 while( nIn>0 ){ 1769 c = utf8Read(zIn, nIn, &sz); 1770 zIn += sz; 1771 nIn -= sz; 1772 if( c<0x02af ){ 1773 if( c>=0x80 || midClass[c&0x7f]<CCLASS_DIGIT ){ 1774 scriptMask |= SCRIPT_LATIN; 1775 }else if( c>='0' && c<='9' ){ 1776 seenDigit = 1; 1777 } 1778 }else if( c>=0x0400 && c<=0x04ff ){ 1779 scriptMask |= SCRIPT_CYRILLIC; 1780 }else if( c>=0x0386 && c<=0x03ce ){ 1781 scriptMask |= SCRIPT_GREEK; 1782 }else if( c>=0x0590 && c<=0x05ff ){ 1783 scriptMask |= SCRIPT_HEBREW; 1784 }else if( c>=0x0600 && c<=0x06ff ){ 1785 scriptMask |= SCRIPT_ARABIC; 1786 } 1787 } 1788 if( scriptMask==0 && seenDigit ) scriptMask = SCRIPT_LATIN; 1789 switch( scriptMask ){ 1790 case 0: res = 999; break; 1791 case SCRIPT_LATIN: res = 215; break; 1792 case SCRIPT_CYRILLIC: res = 220; break; 1793 case SCRIPT_GREEK: res = 200; break; 1794 case SCRIPT_HEBREW: res = 125; break; 1795 case SCRIPT_ARABIC: res = 160; break; 1796 default: res = 998; break; 1797 } 1798 sqlite3_result_int(context, res); 1799 } 1800 1801 /* End transliterate 1802 ****************************************************************************** 1803 ****************************************************************************** 1804 ** Begin spellfix1 virtual table. 1805 */ 1806 1807 /* Maximum length of a phonehash used for querying the shadow table */ 1808 #define SPELLFIX_MX_HASH 32 1809 1810 /* Maximum number of hash strings to examine per query */ 1811 #define SPELLFIX_MX_RUN 1 1812 1813 typedef struct spellfix1_vtab spellfix1_vtab; 1814 typedef struct spellfix1_cursor spellfix1_cursor; 1815 1816 /* Fuzzy-search virtual table object */ 1817 struct spellfix1_vtab { 1818 sqlite3_vtab base; /* Base class - must be first */ 1819 sqlite3 *db; /* Database connection */ 1820 char *zDbName; /* Name of database holding this table */ 1821 char *zTableName; /* Name of the virtual table */ 1822 char *zCostTable; /* Table holding edit-distance cost numbers */ 1823 EditDist3Config *pConfig3; /* Parsed edit distance costs */ 1824 }; 1825 1826 /* Fuzzy-search cursor object */ 1827 struct spellfix1_cursor { 1828 sqlite3_vtab_cursor base; /* Base class - must be first */ 1829 spellfix1_vtab *pVTab; /* The table to which this cursor belongs */ 1830 char *zPattern; /* rhs of MATCH clause */ 1831 int idxNum; /* idxNum value passed to xFilter() */ 1832 int nRow; /* Number of rows of content */ 1833 int nAlloc; /* Number of allocated rows */ 1834 int iRow; /* Current row of content */ 1835 int iLang; /* Value of the langid= constraint */ 1836 int iTop; /* Value of the top= constraint */ 1837 int iScope; /* Value of the scope= constraint */ 1838 int nSearch; /* Number of vocabulary items checked */ 1839 sqlite3_stmt *pFullScan; /* Shadow query for a full table scan */ 1840 struct spellfix1_row { /* For each row of content */ 1841 sqlite3_int64 iRowid; /* Rowid for this row */ 1842 char *zWord; /* Text for this row */ 1843 int iRank; /* Rank for this row */ 1844 int iDistance; /* Distance from pattern for this row */ 1845 int iScore; /* Score for sorting */ 1846 int iMatchlen; /* Value of matchlen column (or -1) */ 1847 char zHash[SPELLFIX_MX_HASH]; /* the phonehash used for this match */ 1848 } *a; 1849 }; 1850 1851 /* 1852 ** Construct one or more SQL statements from the format string given 1853 ** and then evaluate those statements. The success code is written 1854 ** into *pRc. 1855 ** 1856 ** If *pRc is initially non-zero then this routine is a no-op. 1857 */ 1858 static void spellfix1DbExec( 1859 int *pRc, /* Success code */ 1860 sqlite3 *db, /* Database in which to run SQL */ 1861 const char *zFormat, /* Format string for SQL */ 1862 ... /* Arguments to the format string */ 1863 ){ 1864 va_list ap; 1865 char *zSql; 1866 if( *pRc ) return; 1867 va_start(ap, zFormat); 1868 zSql = sqlite3_vmprintf(zFormat, ap); 1869 va_end(ap); 1870 if( zSql==0 ){ 1871 *pRc = SQLITE_NOMEM; 1872 }else{ 1873 *pRc = sqlite3_exec(db, zSql, 0, 0, 0); 1874 sqlite3_free(zSql); 1875 } 1876 } 1877 1878 /* 1879 ** xDisconnect/xDestroy method for the fuzzy-search module. 1880 */ 1881 static int spellfix1Uninit(int isDestroy, sqlite3_vtab *pVTab){ 1882 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 1883 int rc = SQLITE_OK; 1884 if( isDestroy ){ 1885 sqlite3 *db = p->db; 1886 spellfix1DbExec(&rc, db, "DROP TABLE IF EXISTS \"%w\".\"%w_vocab\"", 1887 p->zDbName, p->zTableName); 1888 } 1889 if( rc==SQLITE_OK ){ 1890 sqlite3_free(p->zTableName); 1891 editDist3ConfigDelete(p->pConfig3); 1892 sqlite3_free(p->zCostTable); 1893 sqlite3_free(p); 1894 } 1895 return rc; 1896 } 1897 static int spellfix1Disconnect(sqlite3_vtab *pVTab){ 1898 return spellfix1Uninit(0, pVTab); 1899 } 1900 static int spellfix1Destroy(sqlite3_vtab *pVTab){ 1901 return spellfix1Uninit(1, pVTab); 1902 } 1903 1904 /* 1905 ** Make a copy of a string. Remove leading and trailing whitespace 1906 ** and dequote it. 1907 */ 1908 static char *spellfix1Dequote(const char *zIn){ 1909 char *zOut; 1910 int i, j; 1911 char c; 1912 while( isspace((unsigned char)zIn[0]) ) zIn++; 1913 zOut = sqlite3_mprintf("%s", zIn); 1914 if( zOut==0 ) return 0; 1915 i = (int)strlen(zOut); 1916 #if 0 /* The parser will never leave spaces at the end */ 1917 while( i>0 && isspace(zOut[i-1]) ){ i--; } 1918 #endif 1919 zOut[i] = 0; 1920 c = zOut[0]; 1921 if( c=='\'' || c=='"' ){ 1922 for(i=1, j=0; ALWAYS(zOut[i]); i++){ 1923 zOut[j++] = zOut[i]; 1924 if( zOut[i]==c ){ 1925 if( zOut[i+1]==c ){ 1926 i++; 1927 }else{ 1928 zOut[j-1] = 0; 1929 break; 1930 } 1931 } 1932 } 1933 } 1934 return zOut; 1935 } 1936 1937 1938 /* 1939 ** xConnect/xCreate method for the spellfix1 module. Arguments are: 1940 ** 1941 ** argv[0] -> module name ("spellfix1") 1942 ** argv[1] -> database name 1943 ** argv[2] -> table name 1944 ** argv[3].. -> optional arguments (i.e. "edit_cost_table" parameter) 1945 */ 1946 static int spellfix1Init( 1947 int isCreate, 1948 sqlite3 *db, 1949 void *pAux, 1950 int argc, const char *const*argv, 1951 sqlite3_vtab **ppVTab, 1952 char **pzErr 1953 ){ 1954 spellfix1_vtab *pNew = 0; 1955 /* const char *zModule = argv[0]; // not used */ 1956 const char *zDbName = argv[1]; 1957 const char *zTableName = argv[2]; 1958 int nDbName; 1959 int rc = SQLITE_OK; 1960 int i; 1961 1962 nDbName = (int)strlen(zDbName); 1963 pNew = sqlite3_malloc64( sizeof(*pNew) + nDbName + 1); 1964 if( pNew==0 ){ 1965 rc = SQLITE_NOMEM; 1966 }else{ 1967 memset(pNew, 0, sizeof(*pNew)); 1968 pNew->zDbName = (char*)&pNew[1]; 1969 memcpy(pNew->zDbName, zDbName, nDbName+1); 1970 pNew->zTableName = sqlite3_mprintf("%s", zTableName); 1971 pNew->db = db; 1972 if( pNew->zTableName==0 ){ 1973 rc = SQLITE_NOMEM; 1974 }else{ 1975 rc = sqlite3_declare_vtab(db, 1976 "CREATE TABLE x(word,rank,distance,langid, " 1977 "score, matchlen, phonehash HIDDEN, " 1978 "top HIDDEN, scope HIDDEN, srchcnt HIDDEN, " 1979 "soundslike HIDDEN, command HIDDEN)" 1980 ); 1981 #define SPELLFIX_COL_WORD 0 1982 #define SPELLFIX_COL_RANK 1 1983 #define SPELLFIX_COL_DISTANCE 2 1984 #define SPELLFIX_COL_LANGID 3 1985 #define SPELLFIX_COL_SCORE 4 1986 #define SPELLFIX_COL_MATCHLEN 5 1987 #define SPELLFIX_COL_PHONEHASH 6 1988 #define SPELLFIX_COL_TOP 7 1989 #define SPELLFIX_COL_SCOPE 8 1990 #define SPELLFIX_COL_SRCHCNT 9 1991 #define SPELLFIX_COL_SOUNDSLIKE 10 1992 #define SPELLFIX_COL_COMMAND 11 1993 } 1994 if( rc==SQLITE_OK && isCreate ){ 1995 spellfix1DbExec(&rc, db, 1996 "CREATE TABLE IF NOT EXISTS \"%w\".\"%w_vocab\"(\n" 1997 " id INTEGER PRIMARY KEY,\n" 1998 " rank INT,\n" 1999 " langid INT,\n" 2000 " word TEXT,\n" 2001 " k1 TEXT,\n" 2002 " k2 TEXT\n" 2003 ");\n", 2004 zDbName, zTableName 2005 ); 2006 spellfix1DbExec(&rc, db, 2007 "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" " 2008 "ON \"%w_vocab\"(langid,k2);", 2009 zDbName, zTableName, zTableName 2010 ); 2011 } 2012 for(i=3; rc==SQLITE_OK && i<argc; i++){ 2013 if( strncmp(argv[i],"edit_cost_table=",16)==0 && pNew->zCostTable==0 ){ 2014 pNew->zCostTable = spellfix1Dequote(&argv[i][16]); 2015 if( pNew->zCostTable==0 ) rc = SQLITE_NOMEM; 2016 continue; 2017 } 2018 *pzErr = sqlite3_mprintf("bad argument to spellfix1(): \"%s\"", argv[i]); 2019 rc = SQLITE_ERROR; 2020 } 2021 } 2022 2023 if( rc && pNew ){ 2024 *ppVTab = 0; 2025 spellfix1Uninit(0, &pNew->base); 2026 }else{ 2027 *ppVTab = (sqlite3_vtab *)pNew; 2028 } 2029 return rc; 2030 } 2031 2032 /* 2033 ** The xConnect and xCreate methods 2034 */ 2035 static int spellfix1Connect( 2036 sqlite3 *db, 2037 void *pAux, 2038 int argc, const char *const*argv, 2039 sqlite3_vtab **ppVTab, 2040 char **pzErr 2041 ){ 2042 return spellfix1Init(0, db, pAux, argc, argv, ppVTab, pzErr); 2043 } 2044 static int spellfix1Create( 2045 sqlite3 *db, 2046 void *pAux, 2047 int argc, const char *const*argv, 2048 sqlite3_vtab **ppVTab, 2049 char **pzErr 2050 ){ 2051 return spellfix1Init(1, db, pAux, argc, argv, ppVTab, pzErr); 2052 } 2053 2054 /* 2055 ** Clear all of the content from a cursor. 2056 */ 2057 static void spellfix1ResetCursor(spellfix1_cursor *pCur){ 2058 int i; 2059 for(i=0; i<pCur->nRow; i++){ 2060 sqlite3_free(pCur->a[i].zWord); 2061 } 2062 pCur->nRow = 0; 2063 pCur->iRow = 0; 2064 pCur->nSearch = 0; 2065 if( pCur->pFullScan ){ 2066 sqlite3_finalize(pCur->pFullScan); 2067 pCur->pFullScan = 0; 2068 } 2069 } 2070 2071 /* 2072 ** Resize the cursor to hold up to N rows of content 2073 */ 2074 static void spellfix1ResizeCursor(spellfix1_cursor *pCur, int N){ 2075 struct spellfix1_row *aNew; 2076 assert( N>=pCur->nRow ); 2077 aNew = sqlite3_realloc64(pCur->a, sizeof(pCur->a[0])*N); 2078 if( aNew==0 && N>0 ){ 2079 spellfix1ResetCursor(pCur); 2080 sqlite3_free(pCur->a); 2081 pCur->nAlloc = 0; 2082 pCur->a = 0; 2083 }else{ 2084 pCur->nAlloc = N; 2085 pCur->a = aNew; 2086 } 2087 } 2088 2089 2090 /* 2091 ** Close a fuzzy-search cursor. 2092 */ 2093 static int spellfix1Close(sqlite3_vtab_cursor *cur){ 2094 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2095 spellfix1ResetCursor(pCur); 2096 spellfix1ResizeCursor(pCur, 0); 2097 sqlite3_free(pCur->zPattern); 2098 sqlite3_free(pCur); 2099 return SQLITE_OK; 2100 } 2101 2102 #define SPELLFIX_IDXNUM_MATCH 0x01 /* word MATCH $str */ 2103 #define SPELLFIX_IDXNUM_LANGID 0x02 /* langid == $langid */ 2104 #define SPELLFIX_IDXNUM_TOP 0x04 /* top = $top */ 2105 #define SPELLFIX_IDXNUM_SCOPE 0x08 /* scope = $scope */ 2106 #define SPELLFIX_IDXNUM_DISTLT 0x10 /* distance < $distance */ 2107 #define SPELLFIX_IDXNUM_DISTLE 0x20 /* distance <= $distance */ 2108 #define SPELLFIX_IDXNUM_ROWID 0x40 /* rowid = $rowid */ 2109 #define SPELLFIX_IDXNUM_DIST (0x10|0x20) /* DISTLT and DISTLE */ 2110 2111 /* 2112 ** 2113 ** The plan number is a bitmask of the SPELLFIX_IDXNUM_* values defined 2114 ** above. 2115 ** 2116 ** filter.argv[*] values contains $str, $langid, $top, $scope and $rowid 2117 ** if specified and in that order. 2118 */ 2119 static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ 2120 int iPlan = 0; 2121 int iLangTerm = -1; 2122 int iTopTerm = -1; 2123 int iScopeTerm = -1; 2124 int iDistTerm = -1; 2125 int iRowidTerm = -1; 2126 int i; 2127 const struct sqlite3_index_constraint *pConstraint; 2128 pConstraint = pIdxInfo->aConstraint; 2129 for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){ 2130 if( pConstraint->usable==0 ) continue; 2131 2132 /* Terms of the form: word MATCH $str */ 2133 if( (iPlan & SPELLFIX_IDXNUM_MATCH)==0 2134 && pConstraint->iColumn==SPELLFIX_COL_WORD 2135 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH 2136 ){ 2137 iPlan |= SPELLFIX_IDXNUM_MATCH; 2138 pIdxInfo->aConstraintUsage[i].argvIndex = 1; 2139 pIdxInfo->aConstraintUsage[i].omit = 1; 2140 } 2141 2142 /* Terms of the form: langid = $langid */ 2143 if( (iPlan & SPELLFIX_IDXNUM_LANGID)==0 2144 && pConstraint->iColumn==SPELLFIX_COL_LANGID 2145 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2146 ){ 2147 iPlan |= SPELLFIX_IDXNUM_LANGID; 2148 iLangTerm = i; 2149 } 2150 2151 /* Terms of the form: top = $top */ 2152 if( (iPlan & SPELLFIX_IDXNUM_TOP)==0 2153 && pConstraint->iColumn==SPELLFIX_COL_TOP 2154 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2155 ){ 2156 iPlan |= SPELLFIX_IDXNUM_TOP; 2157 iTopTerm = i; 2158 } 2159 2160 /* Terms of the form: scope = $scope */ 2161 if( (iPlan & SPELLFIX_IDXNUM_SCOPE)==0 2162 && pConstraint->iColumn==SPELLFIX_COL_SCOPE 2163 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2164 ){ 2165 iPlan |= SPELLFIX_IDXNUM_SCOPE; 2166 iScopeTerm = i; 2167 } 2168 2169 /* Terms of the form: distance < $dist or distance <= $dist */ 2170 if( (iPlan & SPELLFIX_IDXNUM_DIST)==0 2171 && pConstraint->iColumn==SPELLFIX_COL_DISTANCE 2172 && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT 2173 || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) 2174 ){ 2175 if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ){ 2176 iPlan |= SPELLFIX_IDXNUM_DISTLT; 2177 }else{ 2178 iPlan |= SPELLFIX_IDXNUM_DISTLE; 2179 } 2180 iDistTerm = i; 2181 } 2182 2183 /* Terms of the form: distance < $dist or distance <= $dist */ 2184 if( (iPlan & SPELLFIX_IDXNUM_ROWID)==0 2185 && pConstraint->iColumn<0 2186 && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ 2187 ){ 2188 iPlan |= SPELLFIX_IDXNUM_ROWID; 2189 iRowidTerm = i; 2190 } 2191 } 2192 if( iPlan&SPELLFIX_IDXNUM_MATCH ){ 2193 int idx = 2; 2194 pIdxInfo->idxNum = iPlan; 2195 if( pIdxInfo->nOrderBy==1 2196 && pIdxInfo->aOrderBy[0].iColumn==SPELLFIX_COL_SCORE 2197 && pIdxInfo->aOrderBy[0].desc==0 2198 ){ 2199 pIdxInfo->orderByConsumed = 1; /* Default order by iScore */ 2200 } 2201 if( iPlan&SPELLFIX_IDXNUM_LANGID ){ 2202 pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx++; 2203 pIdxInfo->aConstraintUsage[iLangTerm].omit = 1; 2204 } 2205 if( iPlan&SPELLFIX_IDXNUM_TOP ){ 2206 pIdxInfo->aConstraintUsage[iTopTerm].argvIndex = idx++; 2207 pIdxInfo->aConstraintUsage[iTopTerm].omit = 1; 2208 } 2209 if( iPlan&SPELLFIX_IDXNUM_SCOPE ){ 2210 pIdxInfo->aConstraintUsage[iScopeTerm].argvIndex = idx++; 2211 pIdxInfo->aConstraintUsage[iScopeTerm].omit = 1; 2212 } 2213 if( iPlan&SPELLFIX_IDXNUM_DIST ){ 2214 pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = idx++; 2215 pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; 2216 } 2217 pIdxInfo->estimatedCost = 1e5; 2218 }else if( (iPlan & SPELLFIX_IDXNUM_ROWID) ){ 2219 pIdxInfo->idxNum = SPELLFIX_IDXNUM_ROWID; 2220 pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; 2221 pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; 2222 pIdxInfo->estimatedCost = 5; 2223 }else{ 2224 pIdxInfo->idxNum = 0; 2225 pIdxInfo->estimatedCost = 1e50; 2226 } 2227 return SQLITE_OK; 2228 } 2229 2230 /* 2231 ** Open a new fuzzy-search cursor. 2232 */ 2233 static int spellfix1Open(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ 2234 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2235 spellfix1_cursor *pCur; 2236 pCur = sqlite3_malloc64( sizeof(*pCur) ); 2237 if( pCur==0 ) return SQLITE_NOMEM; 2238 memset(pCur, 0, sizeof(*pCur)); 2239 pCur->pVTab = p; 2240 *ppCursor = &pCur->base; 2241 return SQLITE_OK; 2242 } 2243 2244 /* 2245 ** Adjust a distance measurement by the words rank in order to show 2246 ** preference to common words. 2247 */ 2248 static int spellfix1Score(int iDistance, int iRank){ 2249 int iLog2; 2250 for(iLog2=0; iRank>0; iLog2++, iRank>>=1){} 2251 return iDistance + 32 - iLog2; 2252 } 2253 2254 /* 2255 ** Compare two spellfix1_row objects for sorting purposes in qsort() such 2256 ** that they sort in order of increasing distance. 2257 */ 2258 static int SQLITE_CDECL spellfix1RowCompare(const void *A, const void *B){ 2259 const struct spellfix1_row *a = (const struct spellfix1_row*)A; 2260 const struct spellfix1_row *b = (const struct spellfix1_row*)B; 2261 return a->iScore - b->iScore; 2262 } 2263 2264 /* 2265 ** A structure used to pass information from spellfix1FilterForMatch() 2266 ** into spellfix1RunQuery(). 2267 */ 2268 typedef struct MatchQuery { 2269 spellfix1_cursor *pCur; /* The cursor being queried */ 2270 sqlite3_stmt *pStmt; /* shadow table query statment */ 2271 char zHash[SPELLFIX_MX_HASH]; /* The current phonehash for zPattern */ 2272 const char *zPattern; /* Transliterated input string */ 2273 int nPattern; /* Length of zPattern */ 2274 EditDist3FromString *pMatchStr3; /* Original unicode string */ 2275 EditDist3Config *pConfig3; /* Edit-distance cost coefficients */ 2276 const EditDist3Lang *pLang; /* The selected language coefficients */ 2277 int iLang; /* The language id */ 2278 int iScope; /* Default scope */ 2279 int iMaxDist; /* Maximum allowed edit distance, or -1 */ 2280 int rc; /* Error code */ 2281 int nRun; /* Number of prior runs for the same zPattern */ 2282 char azPrior[SPELLFIX_MX_RUN][SPELLFIX_MX_HASH]; /* Prior hashes */ 2283 } MatchQuery; 2284 2285 /* 2286 ** Run a query looking for the best matches against zPattern using 2287 ** zHash as the character class seed hash. 2288 */ 2289 static void spellfix1RunQuery(MatchQuery *p, const char *zQuery, int nQuery){ 2290 const char *zK1; 2291 const char *zWord; 2292 int iDist; 2293 int iRank; 2294 int iScore; 2295 int iWorst = 0; 2296 int idx; 2297 int idxWorst = -1; 2298 int i; 2299 int iScope = p->iScope; 2300 spellfix1_cursor *pCur = p->pCur; 2301 sqlite3_stmt *pStmt = p->pStmt; 2302 char zHash1[SPELLFIX_MX_HASH]; 2303 char zHash2[SPELLFIX_MX_HASH]; 2304 char *zClass; 2305 int nClass; 2306 int rc; 2307 2308 if( pCur->a==0 || p->rc ) return; /* Prior memory allocation failure */ 2309 zClass = (char*)phoneticHash((unsigned char*)zQuery, nQuery); 2310 if( zClass==0 ){ 2311 p->rc = SQLITE_NOMEM; 2312 return; 2313 } 2314 nClass = (int)strlen(zClass); 2315 if( nClass>SPELLFIX_MX_HASH-2 ){ 2316 nClass = SPELLFIX_MX_HASH-2; 2317 zClass[nClass] = 0; 2318 } 2319 if( nClass<=iScope ){ 2320 if( nClass>2 ){ 2321 iScope = nClass-1; 2322 }else{ 2323 iScope = nClass; 2324 } 2325 } 2326 memcpy(zHash1, zClass, iScope); 2327 sqlite3_free(zClass); 2328 zHash1[iScope] = 0; 2329 memcpy(zHash2, zHash1, iScope); 2330 zHash2[iScope] = 'Z'; 2331 zHash2[iScope+1] = 0; 2332 #if SPELLFIX_MX_RUN>1 2333 for(i=0; i<p->nRun; i++){ 2334 if( strcmp(p->azPrior[i], zHash1)==0 ) return; 2335 } 2336 #endif 2337 assert( p->nRun<SPELLFIX_MX_RUN ); 2338 memcpy(p->azPrior[p->nRun++], zHash1, iScope+1); 2339 if( sqlite3_bind_text(pStmt, 1, zHash1, -1, SQLITE_STATIC)==SQLITE_NOMEM 2340 || sqlite3_bind_text(pStmt, 2, zHash2, -1, SQLITE_STATIC)==SQLITE_NOMEM 2341 ){ 2342 p->rc = SQLITE_NOMEM; 2343 return; 2344 } 2345 #if SPELLFIX_MX_RUN>1 2346 for(i=0; i<pCur->nRow; i++){ 2347 if( pCur->a[i].iScore>iWorst ){ 2348 iWorst = pCur->a[i].iScore; 2349 idxWorst = i; 2350 } 2351 } 2352 #endif 2353 while( sqlite3_step(pStmt)==SQLITE_ROW ){ 2354 int iMatchlen = -1; 2355 iRank = sqlite3_column_int(pStmt, 2); 2356 if( p->pMatchStr3 ){ 2357 int nWord = sqlite3_column_bytes(pStmt, 1); 2358 zWord = (const char*)sqlite3_column_text(pStmt, 1); 2359 iDist = editDist3Core(p->pMatchStr3, zWord, nWord, p->pLang, &iMatchlen); 2360 }else{ 2361 zK1 = (const char*)sqlite3_column_text(pStmt, 3); 2362 if( zK1==0 ) continue; 2363 iDist = editdist1(p->zPattern, zK1, 0); 2364 } 2365 if( iDist<0 ){ 2366 p->rc = SQLITE_NOMEM; 2367 break; 2368 } 2369 pCur->nSearch++; 2370 2371 /* If there is a "distance < $dist" or "distance <= $dist" constraint, 2372 ** check if this row meets it. If not, jump back up to the top of the 2373 ** loop to process the next row. Otherwise, if the row does match the 2374 ** distance constraint, check if the pCur->a[] array is already full. 2375 ** If it is and no explicit "top = ?" constraint was present in the 2376 ** query, grow the array to ensure there is room for the new entry. */ 2377 assert( (p->iMaxDist>=0)==((pCur->idxNum & SPELLFIX_IDXNUM_DIST) ? 1 : 0) ); 2378 if( p->iMaxDist>=0 ){ 2379 if( iDist>p->iMaxDist ) continue; 2380 if( pCur->nRow>=pCur->nAlloc && (pCur->idxNum & SPELLFIX_IDXNUM_TOP)==0 ){ 2381 spellfix1ResizeCursor(pCur, pCur->nAlloc*2 + 10); 2382 if( pCur->a==0 ) break; 2383 } 2384 } 2385 2386 iScore = spellfix1Score(iDist,iRank); 2387 if( pCur->nRow<pCur->nAlloc ){ 2388 idx = pCur->nRow; 2389 }else if( iScore<iWorst ){ 2390 idx = idxWorst; 2391 sqlite3_free(pCur->a[idx].zWord); 2392 }else{ 2393 continue; 2394 } 2395 2396 pCur->a[idx].zWord = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); 2397 if( pCur->a[idx].zWord==0 ){ 2398 p->rc = SQLITE_NOMEM; 2399 break; 2400 } 2401 pCur->a[idx].iRowid = sqlite3_column_int64(pStmt, 0); 2402 pCur->a[idx].iRank = iRank; 2403 pCur->a[idx].iDistance = iDist; 2404 pCur->a[idx].iScore = iScore; 2405 pCur->a[idx].iMatchlen = iMatchlen; 2406 memcpy(pCur->a[idx].zHash, zHash1, iScope+1); 2407 if( pCur->nRow<pCur->nAlloc ) pCur->nRow++; 2408 if( pCur->nRow==pCur->nAlloc ){ 2409 iWorst = pCur->a[0].iScore; 2410 idxWorst = 0; 2411 for(i=1; i<pCur->nRow; i++){ 2412 iScore = pCur->a[i].iScore; 2413 if( iWorst<iScore ){ 2414 iWorst = iScore; 2415 idxWorst = i; 2416 } 2417 } 2418 } 2419 } 2420 rc = sqlite3_reset(pStmt); 2421 if( rc ) p->rc = rc; 2422 } 2423 2424 /* 2425 ** This version of the xFilter method work if the MATCH term is present 2426 ** and we are doing a scan. 2427 */ 2428 static int spellfix1FilterForMatch( 2429 spellfix1_cursor *pCur, 2430 int argc, 2431 sqlite3_value **argv 2432 ){ 2433 int idxNum = pCur->idxNum; 2434 const unsigned char *zMatchThis; /* RHS of the MATCH operator */ 2435 EditDist3FromString *pMatchStr3 = 0; /* zMatchThis as an editdist string */ 2436 char *zPattern; /* Transliteration of zMatchThis */ 2437 int nPattern; /* Length of zPattern */ 2438 int iLimit = 20; /* Max number of rows of output */ 2439 int iScope = 3; /* Use this many characters of zClass */ 2440 int iLang = 0; /* Language code */ 2441 char *zSql; /* SQL of shadow table query */ 2442 sqlite3_stmt *pStmt = 0; /* Shadow table query */ 2443 int rc; /* Result code */ 2444 int idx = 1; /* Next available filter parameter */ 2445 spellfix1_vtab *p = pCur->pVTab; /* The virtual table that owns pCur */ 2446 MatchQuery x; /* For passing info to RunQuery() */ 2447 2448 /* Load the cost table if we have not already done so */ 2449 if( p->zCostTable!=0 && p->pConfig3==0 ){ 2450 p->pConfig3 = sqlite3_malloc64( sizeof(p->pConfig3[0]) ); 2451 if( p->pConfig3==0 ) return SQLITE_NOMEM; 2452 memset(p->pConfig3, 0, sizeof(p->pConfig3[0])); 2453 rc = editDist3ConfigLoad(p->pConfig3, p->db, p->zCostTable); 2454 if( rc ) return rc; 2455 } 2456 memset(&x, 0, sizeof(x)); 2457 x.iScope = 3; /* Default scope if none specified by "WHERE scope=N" */ 2458 x.iMaxDist = -1; /* Maximum allowed edit distance */ 2459 2460 if( idxNum&2 ){ 2461 iLang = sqlite3_value_int(argv[idx++]); 2462 } 2463 if( idxNum&4 ){ 2464 iLimit = sqlite3_value_int(argv[idx++]); 2465 if( iLimit<1 ) iLimit = 1; 2466 } 2467 if( idxNum&8 ){ 2468 x.iScope = sqlite3_value_int(argv[idx++]); 2469 if( x.iScope<1 ) x.iScope = 1; 2470 if( x.iScope>SPELLFIX_MX_HASH-2 ) x.iScope = SPELLFIX_MX_HASH-2; 2471 } 2472 if( idxNum&(16|32) ){ 2473 x.iMaxDist = sqlite3_value_int(argv[idx++]); 2474 if( idxNum&16 ) x.iMaxDist--; 2475 if( x.iMaxDist<0 ) x.iMaxDist = 0; 2476 } 2477 spellfix1ResetCursor(pCur); 2478 spellfix1ResizeCursor(pCur, iLimit); 2479 zMatchThis = sqlite3_value_text(argv[0]); 2480 if( zMatchThis==0 ) return SQLITE_OK; 2481 if( p->pConfig3 ){ 2482 x.pLang = editDist3FindLang(p->pConfig3, iLang); 2483 pMatchStr3 = editDist3FromStringNew(x.pLang, (const char*)zMatchThis, -1); 2484 if( pMatchStr3==0 ){ 2485 x.rc = SQLITE_NOMEM; 2486 goto filter_exit; 2487 } 2488 }else{ 2489 x.pLang = 0; 2490 } 2491 zPattern = (char*)transliterate(zMatchThis, sqlite3_value_bytes(argv[0])); 2492 sqlite3_free(pCur->zPattern); 2493 pCur->zPattern = zPattern; 2494 if( zPattern==0 ){ 2495 x.rc = SQLITE_NOMEM; 2496 goto filter_exit; 2497 } 2498 nPattern = (int)strlen(zPattern); 2499 if( zPattern[nPattern-1]=='*' ) nPattern--; 2500 zSql = sqlite3_mprintf( 2501 "SELECT id, word, rank, coalesce(k1,word)" 2502 " FROM \"%w\".\"%w_vocab\"" 2503 " WHERE langid=%d AND k2>=?1 AND k2<?2", 2504 p->zDbName, p->zTableName, iLang 2505 ); 2506 if( zSql==0 ){ 2507 x.rc = SQLITE_NOMEM; 2508 pStmt = 0; 2509 goto filter_exit; 2510 } 2511 rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); 2512 sqlite3_free(zSql); 2513 pCur->iLang = iLang; 2514 x.pCur = pCur; 2515 x.pStmt = pStmt; 2516 x.zPattern = zPattern; 2517 x.nPattern = nPattern; 2518 x.pMatchStr3 = pMatchStr3; 2519 x.iLang = iLang; 2520 x.rc = rc; 2521 x.pConfig3 = p->pConfig3; 2522 if( x.rc==SQLITE_OK ){ 2523 spellfix1RunQuery(&x, zPattern, nPattern); 2524 } 2525 2526 if( pCur->a ){ 2527 qsort(pCur->a, pCur->nRow, sizeof(pCur->a[0]), spellfix1RowCompare); 2528 pCur->iTop = iLimit; 2529 pCur->iScope = iScope; 2530 }else{ 2531 x.rc = SQLITE_NOMEM; 2532 } 2533 2534 filter_exit: 2535 sqlite3_finalize(pStmt); 2536 editDist3FromStringDelete(pMatchStr3); 2537 return x.rc; 2538 } 2539 2540 /* 2541 ** This version of xFilter handles a full-table scan case 2542 */ 2543 static int spellfix1FilterForFullScan( 2544 spellfix1_cursor *pCur, 2545 int argc, 2546 sqlite3_value **argv 2547 ){ 2548 int rc = SQLITE_OK; 2549 int idxNum = pCur->idxNum; 2550 char *zSql; 2551 spellfix1_vtab *pVTab = pCur->pVTab; 2552 spellfix1ResetCursor(pCur); 2553 assert( idxNum==0 || idxNum==64 ); 2554 zSql = sqlite3_mprintf( 2555 "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"%s", 2556 pVTab->zDbName, pVTab->zTableName, 2557 ((idxNum & 64) ? " WHERE rowid=?" : "") 2558 ); 2559 if( zSql==0 ) return SQLITE_NOMEM; 2560 rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); 2561 sqlite3_free(zSql); 2562 if( rc==SQLITE_OK && (idxNum & 64) ){ 2563 assert( argc==1 ); 2564 rc = sqlite3_bind_value(pCur->pFullScan, 1, argv[0]); 2565 } 2566 pCur->nRow = pCur->iRow = 0; 2567 if( rc==SQLITE_OK ){ 2568 rc = sqlite3_step(pCur->pFullScan); 2569 if( rc==SQLITE_ROW ){ pCur->iRow = -1; rc = SQLITE_OK; } 2570 if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } 2571 }else{ 2572 pCur->iRow = 0; 2573 } 2574 return rc; 2575 } 2576 2577 2578 /* 2579 ** Called to "rewind" a cursor back to the beginning so that 2580 ** it starts its output over again. Always called at least once 2581 ** prior to any spellfix1Column, spellfix1Rowid, or spellfix1Eof call. 2582 */ 2583 static int spellfix1Filter( 2584 sqlite3_vtab_cursor *cur, 2585 int idxNum, const char *idxStr, 2586 int argc, sqlite3_value **argv 2587 ){ 2588 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2589 int rc; 2590 pCur->idxNum = idxNum; 2591 if( idxNum & 1 ){ 2592 rc = spellfix1FilterForMatch(pCur, argc, argv); 2593 }else{ 2594 rc = spellfix1FilterForFullScan(pCur, argc, argv); 2595 } 2596 return rc; 2597 } 2598 2599 2600 /* 2601 ** Advance a cursor to its next row of output 2602 */ 2603 static int spellfix1Next(sqlite3_vtab_cursor *cur){ 2604 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2605 int rc = SQLITE_OK; 2606 if( pCur->iRow < pCur->nRow ){ 2607 if( pCur->pFullScan ){ 2608 rc = sqlite3_step(pCur->pFullScan); 2609 if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; 2610 if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; 2611 }else{ 2612 pCur->iRow++; 2613 } 2614 } 2615 return rc; 2616 } 2617 2618 /* 2619 ** Return TRUE if we are at the end-of-file 2620 */ 2621 static int spellfix1Eof(sqlite3_vtab_cursor *cur){ 2622 spellfix1_cursor *pCur = (spellfix1_cursor *)cur; 2623 return pCur->iRow>=pCur->nRow; 2624 } 2625 2626 /* 2627 ** Return columns from the current row. 2628 */ 2629 static int spellfix1Column( 2630 sqlite3_vtab_cursor *cur, 2631 sqlite3_context *ctx, 2632 int i 2633 ){ 2634 spellfix1_cursor *pCur = (spellfix1_cursor*)cur; 2635 if( pCur->pFullScan ){ 2636 if( i<=SPELLFIX_COL_LANGID ){ 2637 sqlite3_result_value(ctx, sqlite3_column_value(pCur->pFullScan, i)); 2638 }else{ 2639 sqlite3_result_null(ctx); 2640 } 2641 return SQLITE_OK; 2642 } 2643 switch( i ){ 2644 case SPELLFIX_COL_WORD: { 2645 sqlite3_result_text(ctx, pCur->a[pCur->iRow].zWord, -1, SQLITE_STATIC); 2646 break; 2647 } 2648 case SPELLFIX_COL_RANK: { 2649 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iRank); 2650 break; 2651 } 2652 case SPELLFIX_COL_DISTANCE: { 2653 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iDistance); 2654 break; 2655 } 2656 case SPELLFIX_COL_LANGID: { 2657 sqlite3_result_int(ctx, pCur->iLang); 2658 break; 2659 } 2660 case SPELLFIX_COL_SCORE: { 2661 sqlite3_result_int(ctx, pCur->a[pCur->iRow].iScore); 2662 break; 2663 } 2664 case SPELLFIX_COL_MATCHLEN: { 2665 int iMatchlen = pCur->a[pCur->iRow].iMatchlen; 2666 if( iMatchlen<0 ){ 2667 int nPattern = (int)strlen(pCur->zPattern); 2668 char *zWord = pCur->a[pCur->iRow].zWord; 2669 int nWord = (int)strlen(zWord); 2670 2671 if( nPattern>0 && pCur->zPattern[nPattern-1]=='*' ){ 2672 char *zTranslit; 2673 int res; 2674 zTranslit = (char *)transliterate((unsigned char *)zWord, nWord); 2675 if( !zTranslit ) return SQLITE_NOMEM; 2676 res = editdist1(pCur->zPattern, zTranslit, &iMatchlen); 2677 sqlite3_free(zTranslit); 2678 if( res<0 ) return SQLITE_NOMEM; 2679 iMatchlen = translen_to_charlen(zWord, nWord, iMatchlen); 2680 }else{ 2681 iMatchlen = utf8Charlen(zWord, nWord); 2682 } 2683 } 2684 2685 sqlite3_result_int(ctx, iMatchlen); 2686 break; 2687 } 2688 case SPELLFIX_COL_PHONEHASH: { 2689 sqlite3_result_text(ctx, pCur->a[pCur->iRow].zHash, -1, SQLITE_STATIC); 2690 break; 2691 } 2692 case SPELLFIX_COL_TOP: { 2693 sqlite3_result_int(ctx, pCur->iTop); 2694 break; 2695 } 2696 case SPELLFIX_COL_SCOPE: { 2697 sqlite3_result_int(ctx, pCur->iScope); 2698 break; 2699 } 2700 case SPELLFIX_COL_SRCHCNT: { 2701 sqlite3_result_int(ctx, pCur->nSearch); 2702 break; 2703 } 2704 default: { 2705 sqlite3_result_null(ctx); 2706 break; 2707 } 2708 } 2709 return SQLITE_OK; 2710 } 2711 2712 /* 2713 ** The rowid. 2714 */ 2715 static int spellfix1Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ 2716 spellfix1_cursor *pCur = (spellfix1_cursor*)cur; 2717 if( pCur->pFullScan ){ 2718 *pRowid = sqlite3_column_int64(pCur->pFullScan, 4); 2719 }else{ 2720 *pRowid = pCur->a[pCur->iRow].iRowid; 2721 } 2722 return SQLITE_OK; 2723 } 2724 2725 /* 2726 ** This function is called by the xUpdate() method. It returns a string 2727 ** containing the conflict mode that xUpdate() should use for the current 2728 ** operation. One of: "ROLLBACK", "IGNORE", "ABORT" or "REPLACE". 2729 */ 2730 static const char *spellfix1GetConflict(sqlite3 *db){ 2731 static const char *azConflict[] = { 2732 /* Note: Instead of "FAIL" - "ABORT". */ 2733 "ROLLBACK", "IGNORE", "ABORT", "ABORT", "REPLACE" 2734 }; 2735 int eConflict = sqlite3_vtab_on_conflict(db); 2736 2737 assert( eConflict==SQLITE_ROLLBACK || eConflict==SQLITE_IGNORE 2738 || eConflict==SQLITE_FAIL || eConflict==SQLITE_ABORT 2739 || eConflict==SQLITE_REPLACE 2740 ); 2741 assert( SQLITE_ROLLBACK==1 ); 2742 assert( SQLITE_IGNORE==2 ); 2743 assert( SQLITE_FAIL==3 ); 2744 assert( SQLITE_ABORT==4 ); 2745 assert( SQLITE_REPLACE==5 ); 2746 2747 return azConflict[eConflict-1]; 2748 } 2749 2750 /* 2751 ** The xUpdate() method. 2752 */ 2753 static int spellfix1Update( 2754 sqlite3_vtab *pVTab, 2755 int argc, 2756 sqlite3_value **argv, 2757 sqlite_int64 *pRowid 2758 ){ 2759 int rc = SQLITE_OK; 2760 sqlite3_int64 rowid, newRowid; 2761 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2762 sqlite3 *db = p->db; 2763 2764 if( argc==1 ){ 2765 /* A delete operation on the rowid given by argv[0] */ 2766 rowid = *pRowid = sqlite3_value_int64(argv[0]); 2767 spellfix1DbExec(&rc, db, "DELETE FROM \"%w\".\"%w_vocab\" " 2768 " WHERE id=%lld", 2769 p->zDbName, p->zTableName, rowid); 2770 }else{ 2771 const unsigned char *zWord = sqlite3_value_text(argv[SPELLFIX_COL_WORD+2]); 2772 int nWord = sqlite3_value_bytes(argv[SPELLFIX_COL_WORD+2]); 2773 int iLang = sqlite3_value_int(argv[SPELLFIX_COL_LANGID+2]); 2774 int iRank = sqlite3_value_int(argv[SPELLFIX_COL_RANK+2]); 2775 const unsigned char *zSoundslike = 2776 sqlite3_value_text(argv[SPELLFIX_COL_SOUNDSLIKE+2]); 2777 int nSoundslike = sqlite3_value_bytes(argv[SPELLFIX_COL_SOUNDSLIKE+2]); 2778 char *zK1, *zK2; 2779 int i; 2780 char c; 2781 const char *zConflict = spellfix1GetConflict(db); 2782 2783 if( zWord==0 ){ 2784 /* Inserts of the form: INSERT INTO table(command) VALUES('xyzzy'); 2785 ** cause zWord to be NULL, so we look at the "command" column to see 2786 ** what special actions to take */ 2787 const char *zCmd = 2788 (const char*)sqlite3_value_text(argv[SPELLFIX_COL_COMMAND+2]); 2789 if( zCmd==0 ){ 2790 pVTab->zErrMsg = sqlite3_mprintf("NOT NULL constraint failed: %s.word", 2791 p->zTableName); 2792 return SQLITE_CONSTRAINT_NOTNULL; 2793 } 2794 if( strcmp(zCmd,"reset")==0 ){ 2795 /* Reset the edit cost table (if there is one). */ 2796 editDist3ConfigDelete(p->pConfig3); 2797 p->pConfig3 = 0; 2798 return SQLITE_OK; 2799 } 2800 if( strncmp(zCmd,"edit_cost_table=",16)==0 ){ 2801 editDist3ConfigDelete(p->pConfig3); 2802 p->pConfig3 = 0; 2803 sqlite3_free(p->zCostTable); 2804 p->zCostTable = spellfix1Dequote(zCmd+16); 2805 if( p->zCostTable==0 ) return SQLITE_NOMEM; 2806 if( p->zCostTable[0]==0 || sqlite3_stricmp(p->zCostTable,"null")==0 ){ 2807 sqlite3_free(p->zCostTable); 2808 p->zCostTable = 0; 2809 } 2810 return SQLITE_OK; 2811 } 2812 pVTab->zErrMsg = sqlite3_mprintf("unknown value for %s.command: \"%w\"", 2813 p->zTableName, zCmd); 2814 return SQLITE_ERROR; 2815 } 2816 if( iRank<1 ) iRank = 1; 2817 if( zSoundslike ){ 2818 zK1 = (char*)transliterate(zSoundslike, nSoundslike); 2819 }else{ 2820 zK1 = (char*)transliterate(zWord, nWord); 2821 } 2822 if( zK1==0 ) return SQLITE_NOMEM; 2823 for(i=0; (c = zK1[i])!=0; i++){ 2824 if( c>='A' && c<='Z' ) zK1[i] += 'a' - 'A'; 2825 } 2826 zK2 = (char*)phoneticHash((const unsigned char*)zK1, i); 2827 if( zK2==0 ){ 2828 sqlite3_free(zK1); 2829 return SQLITE_NOMEM; 2830 } 2831 if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ 2832 if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ 2833 spellfix1DbExec(&rc, db, 2834 "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " 2835 "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)", 2836 p->zDbName, p->zTableName, 2837 iRank, iLang, zWord, zK1, zWord, zK2 2838 ); 2839 }else{ 2840 newRowid = sqlite3_value_int64(argv[1]); 2841 spellfix1DbExec(&rc, db, 2842 "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " 2843 "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)", 2844 zConflict, p->zDbName, p->zTableName, 2845 newRowid, iRank, iLang, zWord, zK1, zWord, zK2 2846 ); 2847 } 2848 *pRowid = sqlite3_last_insert_rowid(db); 2849 }else{ 2850 rowid = sqlite3_value_int64(argv[0]); 2851 newRowid = *pRowid = sqlite3_value_int64(argv[1]); 2852 spellfix1DbExec(&rc, db, 2853 "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," 2854 " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld", 2855 zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang, 2856 zWord, zK1, zWord, zK2, rowid 2857 ); 2858 } 2859 sqlite3_free(zK1); 2860 sqlite3_free(zK2); 2861 } 2862 return rc; 2863 } 2864 2865 /* 2866 ** Rename the spellfix1 table. 2867 */ 2868 static int spellfix1Rename(sqlite3_vtab *pVTab, const char *zNew){ 2869 spellfix1_vtab *p = (spellfix1_vtab*)pVTab; 2870 sqlite3 *db = p->db; 2871 int rc = SQLITE_OK; 2872 char *zNewName = sqlite3_mprintf("%s", zNew); 2873 if( zNewName==0 ){ 2874 return SQLITE_NOMEM; 2875 } 2876 spellfix1DbExec(&rc, db, 2877 "ALTER TABLE \"%w\".\"%w_vocab\" RENAME TO \"%w_vocab\"", 2878 p->zDbName, p->zTableName, zNewName 2879 ); 2880 if( rc==SQLITE_OK ){ 2881 sqlite3_free(p->zTableName); 2882 p->zTableName = zNewName; 2883 }else{ 2884 sqlite3_free(zNewName); 2885 } 2886 return rc; 2887 } 2888 2889 2890 /* 2891 ** A virtual table module that provides fuzzy search. 2892 */ 2893 static sqlite3_module spellfix1Module = { 2894 0, /* iVersion */ 2895 spellfix1Create, /* xCreate - handle CREATE VIRTUAL TABLE */ 2896 spellfix1Connect, /* xConnect - reconnected to an existing table */ 2897 spellfix1BestIndex, /* xBestIndex - figure out how to do a query */ 2898 spellfix1Disconnect, /* xDisconnect - close a connection */ 2899 spellfix1Destroy, /* xDestroy - handle DROP TABLE */ 2900 spellfix1Open, /* xOpen - open a cursor */ 2901 spellfix1Close, /* xClose - close a cursor */ 2902 spellfix1Filter, /* xFilter - configure scan constraints */ 2903 spellfix1Next, /* xNext - advance a cursor */ 2904 spellfix1Eof, /* xEof - check for end of scan */ 2905 spellfix1Column, /* xColumn - read data */ 2906 spellfix1Rowid, /* xRowid - read data */ 2907 spellfix1Update, /* xUpdate */ 2908 0, /* xBegin */ 2909 0, /* xSync */ 2910 0, /* xCommit */ 2911 0, /* xRollback */ 2912 0, /* xFindMethod */ 2913 spellfix1Rename, /* xRename */ 2914 }; 2915 2916 /* 2917 ** Register the various functions and the virtual table. 2918 */ 2919 static int spellfix1Register(sqlite3 *db){ 2920 int rc = SQLITE_OK; 2921 int i; 2922 rc = sqlite3_create_function(db, "spellfix1_translit", 1, 2923 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2924 transliterateSqlFunc, 0, 0); 2925 if( rc==SQLITE_OK ){ 2926 rc = sqlite3_create_function(db, "spellfix1_editdist", 2, 2927 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2928 editdistSqlFunc, 0, 0); 2929 } 2930 if( rc==SQLITE_OK ){ 2931 rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, 2932 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2933 phoneticHashSqlFunc, 0, 0); 2934 } 2935 if( rc==SQLITE_OK ){ 2936 rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, 2937 SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, 2938 scriptCodeSqlFunc, 0, 0); 2939 } 2940 if( rc==SQLITE_OK ){ 2941 rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); 2942 } 2943 if( rc==SQLITE_OK ){ 2944 rc = editDist3Install(db); 2945 } 2946 2947 /* Verify sanity of the translit[] table */ 2948 for(i=0; i<sizeof(translit)/sizeof(translit[0])-1; i++){ 2949 assert( translit[i].cFrom<translit[i+1].cFrom ); 2950 } 2951 2952 return rc; 2953 } 2954 2955 #endif /* SQLITE_OMIT_VIRTUALTABLE */ 2956 2957 /* 2958 ** Extension load function. 2959 */ 2960 #ifdef _WIN32 2961 __declspec(dllexport) 2962 #endif 2963 int sqlite3_spellfix_init( 2964 sqlite3 *db, 2965 char **pzErrMsg, 2966 const sqlite3_api_routines *pApi 2967 ){ 2968 SQLITE_EXTENSION_INIT2(pApi); 2969 #ifndef SQLITE_OMIT_VIRTUALTABLE 2970 return spellfix1Register(db); 2971 #endif 2972 return SQLITE_OK; 2973 } 2974