1 /* vi:set ts=8 sts=4 sw=4 noet: 2 * 3 * VIM - Vi IMproved by Bram Moolenaar 4 * 5 * Do ":help uganda" in Vim to read copying and usage conditions. 6 * Do ":help credits" in Vim to see a list of people who contributed. 7 * See README.txt for an overview of the Vim source code. 8 */ 9 10 /* 11 * spellfile.c: code for reading and writing spell files. 12 * 13 * See spell.c for information about spell checking. 14 */ 15 16 /* 17 * Vim spell file format: <HEADER> 18 * <SECTIONS> 19 * <LWORDTREE> 20 * <KWORDTREE> 21 * <PREFIXTREE> 22 * 23 * <HEADER>: <fileID> <versionnr> 24 * 25 * <fileID> 8 bytes "VIMspell" 26 * <versionnr> 1 byte VIMSPELLVERSION 27 * 28 * 29 * Sections make it possible to add information to the .spl file without 30 * making it incompatible with previous versions. There are two kinds of 31 * sections: 32 * 1. Not essential for correct spell checking. E.g. for making suggestions. 33 * These are skipped when not supported. 34 * 2. Optional information, but essential for spell checking when present. 35 * E.g. conditions for affixes. When this section is present but not 36 * supported an error message is given. 37 * 38 * <SECTIONS>: <section> ... <sectionend> 39 * 40 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 41 * 42 * <sectionID> 1 byte number from 0 to 254 identifying the section 43 * 44 * <sectionflags> 1 byte SNF_REQUIRED: this section is required for correct 45 * spell checking 46 * 47 * <sectionlen> 4 bytes length of section contents, MSB first 48 * 49 * <sectionend> 1 byte SN_END 50 * 51 * 52 * sectionID == SN_INFO: <infotext> 53 * <infotext> N bytes free format text with spell file info (version, 54 * website, etc) 55 * 56 * sectionID == SN_REGION: <regionname> ... 57 * <regionname> 2 bytes Up to MAXREGIONS region names: ca, au, etc. Lower 58 * case. First <regionname> is region 1. 59 * 60 * sectionID == SN_CHARFLAGS: <charflagslen> <charflags> 61 * <folcharslen> <folchars> 62 * <charflagslen> 1 byte Number of bytes in <charflags> (should be 128). 63 * <charflags> N bytes List of flags (first one is for character 128): 64 * 0x01 word character CF_WORD 65 * 0x02 upper-case character CF_UPPER 66 * <folcharslen> 2 bytes Number of bytes in <folchars>. 67 * <folchars> N bytes Folded characters, first one is for character 128. 68 * 69 * sectionID == SN_MIDWORD: <midword> 70 * <midword> N bytes Characters that are word characters only when used 71 * in the middle of a word. 72 * 73 * sectionID == SN_PREFCOND: <prefcondcnt> <prefcond> ... 74 * <prefcondcnt> 2 bytes Number of <prefcond> items following. 75 * <prefcond> : <condlen> <condstr> 76 * <condlen> 1 byte Length of <condstr>. 77 * <condstr> N bytes Condition for the prefix. 78 * 79 * sectionID == SN_REP: <repcount> <rep> ... 80 * <repcount> 2 bytes number of <rep> items, MSB first. 81 * <rep> : <repfromlen> <repfrom> <reptolen> <repto> 82 * <repfromlen> 1 byte length of <repfrom> 83 * <repfrom> N bytes "from" part of replacement 84 * <reptolen> 1 byte length of <repto> 85 * <repto> N bytes "to" part of replacement 86 * 87 * sectionID == SN_REPSAL: <repcount> <rep> ... 88 * just like SN_REP but for soundfolded words 89 * 90 * sectionID == SN_SAL: <salflags> <salcount> <sal> ... 91 * <salflags> 1 byte flags for soundsalike conversion: 92 * SAL_F0LLOWUP 93 * SAL_COLLAPSE 94 * SAL_REM_ACCENTS 95 * <salcount> 2 bytes number of <sal> items following 96 * <sal> : <salfromlen> <salfrom> <saltolen> <salto> 97 * <salfromlen> 1 byte length of <salfrom> 98 * <salfrom> N bytes "from" part of soundsalike 99 * <saltolen> 1 byte length of <salto> 100 * <salto> N bytes "to" part of soundsalike 101 * 102 * sectionID == SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 103 * <sofofromlen> 2 bytes length of <sofofrom> 104 * <sofofrom> N bytes "from" part of soundfold 105 * <sofotolen> 2 bytes length of <sofoto> 106 * <sofoto> N bytes "to" part of soundfold 107 * 108 * sectionID == SN_SUGFILE: <timestamp> 109 * <timestamp> 8 bytes time in seconds that must match with .sug file 110 * 111 * sectionID == SN_NOSPLITSUGS: nothing 112 * 113 * sectionID == SN_NOCOMPOUNDSUGS: nothing 114 * 115 * sectionID == SN_WORDS: <word> ... 116 * <word> N bytes NUL terminated common word 117 * 118 * sectionID == SN_MAP: <mapstr> 119 * <mapstr> N bytes String with sequences of similar characters, 120 * separated by slashes. 121 * 122 * sectionID == SN_COMPOUND: <compmax> <compminlen> <compsylmax> <compoptions> 123 * <comppatcount> <comppattern> ... <compflags> 124 * <compmax> 1 byte Maximum nr of words in compound word. 125 * <compminlen> 1 byte Minimal word length for compounding. 126 * <compsylmax> 1 byte Maximum nr of syllables in compound word. 127 * <compoptions> 2 bytes COMP_ flags. 128 * <comppatcount> 2 bytes number of <comppattern> following 129 * <compflags> N bytes Flags from COMPOUNDRULE items, separated by 130 * slashes. 131 * 132 * <comppattern>: <comppatlen> <comppattext> 133 * <comppatlen> 1 byte length of <comppattext> 134 * <comppattext> N bytes end or begin chars from CHECKCOMPOUNDPATTERN 135 * 136 * sectionID == SN_NOBREAK: (empty, its presence is what matters) 137 * 138 * sectionID == SN_SYLLABLE: <syllable> 139 * <syllable> N bytes String from SYLLABLE item. 140 * 141 * <LWORDTREE>: <wordtree> 142 * 143 * <KWORDTREE>: <wordtree> 144 * 145 * <PREFIXTREE>: <wordtree> 146 * 147 * 148 * <wordtree>: <nodecount> <nodedata> ... 149 * 150 * <nodecount> 4 bytes Number of nodes following. MSB first. 151 * 152 * <nodedata>: <siblingcount> <sibling> ... 153 * 154 * <siblingcount> 1 byte Number of siblings in this node. The siblings 155 * follow in sorted order. 156 * 157 * <sibling>: <byte> [ <nodeidx> <xbyte> 158 * | <flags> [<flags2>] [<region>] [<affixID>] 159 * | [<pflags>] <affixID> <prefcondnr> ] 160 * 161 * <byte> 1 byte Byte value of the sibling. Special cases: 162 * BY_NOFLAGS: End of word without flags and for all 163 * regions. 164 * For PREFIXTREE <affixID> and 165 * <prefcondnr> follow. 166 * BY_FLAGS: End of word, <flags> follow. 167 * For PREFIXTREE <pflags>, <affixID> 168 * and <prefcondnr> follow. 169 * BY_FLAGS2: End of word, <flags> and <flags2> 170 * follow. Not used in PREFIXTREE. 171 * BY_INDEX: Child of sibling is shared, <nodeidx> 172 * and <xbyte> follow. 173 * 174 * <nodeidx> 3 bytes Index of child for this sibling, MSB first. 175 * 176 * <xbyte> 1 byte byte value of the sibling. 177 * 178 * <flags> 1 byte bitmask of: 179 * WF_ALLCAP word must have only capitals 180 * WF_ONECAP first char of word must be capital 181 * WF_KEEPCAP keep-case word 182 * WF_FIXCAP keep-case word, all caps not allowed 183 * WF_RARE rare word 184 * WF_BANNED bad word 185 * WF_REGION <region> follows 186 * WF_AFX <affixID> follows 187 * 188 * <flags2> 1 byte Bitmask of: 189 * WF_HAS_AFF >> 8 word includes affix 190 * WF_NEEDCOMP >> 8 word only valid in compound 191 * WF_NOSUGGEST >> 8 word not used for suggestions 192 * WF_COMPROOT >> 8 word already a compound 193 * WF_NOCOMPBEF >> 8 no compounding before this word 194 * WF_NOCOMPAFT >> 8 no compounding after this word 195 * 196 * <pflags> 1 byte bitmask of: 197 * WFP_RARE rare prefix 198 * WFP_NC non-combining prefix 199 * WFP_UP letter after prefix made upper case 200 * 201 * <region> 1 byte Bitmask for regions in which word is valid. When 202 * omitted it's valid in all regions. 203 * Lowest bit is for region 1. 204 * 205 * <affixID> 1 byte ID of affix that can be used with this word. In 206 * PREFIXTREE used for the required prefix ID. 207 * 208 * <prefcondnr> 2 bytes Prefix condition number, index in <prefcond> list 209 * from HEADER. 210 * 211 * All text characters are in 'encoding', but stored as single bytes. 212 */ 213 214 /* 215 * Vim .sug file format: <SUGHEADER> 216 * <SUGWORDTREE> 217 * <SUGTABLE> 218 * 219 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 220 * 221 * <fileID> 6 bytes "VIMsug" 222 * <versionnr> 1 byte VIMSUGVERSION 223 * <timestamp> 8 bytes timestamp that must match with .spl file 224 * 225 * 226 * <SUGWORDTREE>: <wordtree> (see above, no flags or region used) 227 * 228 * 229 * <SUGTABLE>: <sugwcount> <sugline> ... 230 * 231 * <sugwcount> 4 bytes number of <sugline> following 232 * 233 * <sugline>: <sugnr> ... NUL 234 * 235 * <sugnr>: X bytes word number that results in this soundfolded word, 236 * stored as an offset to the previous number in as 237 * few bytes as possible, see offset2bytes()) 238 */ 239 240 #include "vim.h" 241 242 #if defined(FEAT_SPELL) || defined(PROTO) 243 244 #ifndef UNIX // it's in os_unix.h for Unix 245 # include <time.h> // for time_t 246 #endif 247 248 #ifndef UNIX // it's in os_unix.h for Unix 249 # include <time.h> // for time_t 250 #endif 251 252 // Special byte values for <byte>. Some are only used in the tree for 253 // postponed prefixes, some only in the other trees. This is a bit messy... 254 #define BY_NOFLAGS 0 // end of word without flags or region; for 255 // postponed prefix: no <pflags> 256 #define BY_INDEX 1 // child is shared, index follows 257 #define BY_FLAGS 2 // end of word, <flags> byte follows; for 258 // postponed prefix: <pflags> follows 259 #define BY_FLAGS2 3 // end of word, <flags> and <flags2> bytes 260 // follow; never used in prefix tree 261 #define BY_SPECIAL BY_FLAGS2 // highest special byte value 262 263 #define ZERO_FLAG 65009 // used when flag is zero: "0" 264 265 // Flags used in .spl file for soundsalike flags. 266 #define SAL_F0LLOWUP 1 267 #define SAL_COLLAPSE 2 268 #define SAL_REM_ACCENTS 4 269 270 #define VIMSPELLMAGIC "VIMspell" // string at start of Vim spell file 271 #define VIMSPELLMAGICL 8 272 #define VIMSPELLVERSION 50 273 274 // Section IDs. Only renumber them when VIMSPELLVERSION changes! 275 #define SN_REGION 0 // <regionname> section 276 #define SN_CHARFLAGS 1 // charflags section 277 #define SN_MIDWORD 2 // <midword> section 278 #define SN_PREFCOND 3 // <prefcond> section 279 #define SN_REP 4 // REP items section 280 #define SN_SAL 5 // SAL items section 281 #define SN_SOFO 6 // soundfolding section 282 #define SN_MAP 7 // MAP items section 283 #define SN_COMPOUND 8 // compound words section 284 #define SN_SYLLABLE 9 // syllable section 285 #define SN_NOBREAK 10 // NOBREAK section 286 #define SN_SUGFILE 11 // timestamp for .sug file 287 #define SN_REPSAL 12 // REPSAL items section 288 #define SN_WORDS 13 // common words 289 #define SN_NOSPLITSUGS 14 // don't split word for suggestions 290 #define SN_INFO 15 // info section 291 #define SN_NOCOMPOUNDSUGS 16 // don't compound for suggestions 292 #define SN_END 255 // end of sections 293 294 #define SNF_REQUIRED 1 // <sectionflags>: required section 295 296 #define CF_WORD 0x01 297 #define CF_UPPER 0x02 298 299 /* 300 * Loop through all the siblings of a node (including the node) 301 */ 302 #define FOR_ALL_NODE_SIBLINGS(node, np) \ 303 for ((np) = (node); (np) != NULL; (np) = (np)->wn_sibling) 304 305 static int set_spell_finish(spelltab_T *new_st); 306 static int write_spell_prefcond(FILE *fd, garray_T *gap); 307 static int read_region_section(FILE *fd, slang_T *slang, int len); 308 static int read_charflags_section(FILE *fd); 309 static int read_prefcond_section(FILE *fd, slang_T *lp); 310 static int read_rep_section(FILE *fd, garray_T *gap, short *first); 311 static int read_sal_section(FILE *fd, slang_T *slang); 312 static int read_words_section(FILE *fd, slang_T *lp, int len); 313 static int read_sofo_section(FILE *fd, slang_T *slang); 314 static int read_compound(FILE *fd, slang_T *slang, int len); 315 static int set_sofo(slang_T *lp, char_u *from, char_u *to); 316 static void set_sal_first(slang_T *lp); 317 static int *mb_str2wide(char_u *s); 318 static int spell_read_tree(FILE *fd, char_u **bytsp, long *bytsp_len, idx_T **idxsp, int prefixtree, int prefixcnt); 319 static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr); 320 static void set_spell_charflags(char_u *flags, int cnt, char_u *upp); 321 static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp); 322 static void set_map_str(slang_T *lp, char_u *map); 323 324 325 static char *e_spell_trunc = N_("E758: Truncated spell file"); 326 static char *e_afftrailing = N_("Trailing text in %s line %d: %s"); 327 static char *e_affname = N_("Affix name too long in %s line %d: %s"); 328 static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP"); 329 static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range"); 330 static char *msg_compressing = N_("Compressing word tree..."); 331 332 /* 333 * Load one spell file and store the info into a slang_T. 334 * 335 * This is invoked in three ways: 336 * - From spell_load_cb() to load a spell file for the first time. "lang" is 337 * the language name, "old_lp" is NULL. Will allocate an slang_T. 338 * - To reload a spell file that was changed. "lang" is NULL and "old_lp" 339 * points to the existing slang_T. 340 * - Just after writing a .spl file; it's read back to produce the .sug file. 341 * "old_lp" is NULL and "lang" is NULL. Will allocate an slang_T. 342 * 343 * Returns the slang_T the spell file was loaded into. NULL for error. 344 */ 345 slang_T * 346 spell_load_file( 347 char_u *fname, 348 char_u *lang, 349 slang_T *old_lp, 350 int silent) // no error if file doesn't exist 351 { 352 FILE *fd; 353 char_u buf[VIMSPELLMAGICL]; 354 char_u *p; 355 int i; 356 int n; 357 int len; 358 slang_T *lp = NULL; 359 int c = 0; 360 int res; 361 int did_estack_push = FALSE; 362 ESTACK_CHECK_DECLARATION 363 364 fd = mch_fopen((char *)fname, "r"); 365 if (fd == NULL) 366 { 367 if (!silent) 368 semsg(_(e_notopen), fname); 369 else if (p_verbose > 2) 370 { 371 verbose_enter(); 372 smsg((const char *)e_notopen, fname); 373 verbose_leave(); 374 } 375 goto endFAIL; 376 } 377 if (p_verbose > 2) 378 { 379 verbose_enter(); 380 smsg(_("Reading spell file \"%s\""), fname); 381 verbose_leave(); 382 } 383 384 if (old_lp == NULL) 385 { 386 lp = slang_alloc(lang); 387 if (lp == NULL) 388 goto endFAIL; 389 390 // Remember the file name, used to reload the file when it's updated. 391 lp->sl_fname = vim_strsave(fname); 392 if (lp->sl_fname == NULL) 393 goto endFAIL; 394 395 // Check for .add.spl (_add.spl for VMS). 396 lp->sl_add = strstr((char *)gettail(fname), SPL_FNAME_ADD) != NULL; 397 } 398 else 399 lp = old_lp; 400 401 // Set sourcing_name, so that error messages mention the file name. 402 estack_push(ETYPE_SPELL, fname, 0); 403 ESTACK_CHECK_SETUP 404 did_estack_push = TRUE; 405 406 /* 407 * <HEADER>: <fileID> 408 */ 409 for (i = 0; i < VIMSPELLMAGICL; ++i) 410 buf[i] = getc(fd); // <fileID> 411 if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0) 412 { 413 emsg(_("E757: This does not look like a spell file")); 414 goto endFAIL; 415 } 416 c = getc(fd); // <versionnr> 417 if (c < VIMSPELLVERSION) 418 { 419 emsg(_("E771: Old spell file, needs to be updated")); 420 goto endFAIL; 421 } 422 else if (c > VIMSPELLVERSION) 423 { 424 emsg(_("E772: Spell file is for newer version of Vim")); 425 goto endFAIL; 426 } 427 428 429 /* 430 * <SECTIONS>: <section> ... <sectionend> 431 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 432 */ 433 for (;;) 434 { 435 n = getc(fd); // <sectionID> or <sectionend> 436 if (n == SN_END) 437 break; 438 c = getc(fd); // <sectionflags> 439 len = get4c(fd); // <sectionlen> 440 if (len < 0) 441 goto truncerr; 442 443 res = 0; 444 switch (n) 445 { 446 case SN_INFO: 447 lp->sl_info = read_string(fd, len); // <infotext> 448 if (lp->sl_info == NULL) 449 goto endFAIL; 450 break; 451 452 case SN_REGION: 453 res = read_region_section(fd, lp, len); 454 break; 455 456 case SN_CHARFLAGS: 457 res = read_charflags_section(fd); 458 break; 459 460 case SN_MIDWORD: 461 lp->sl_midword = read_string(fd, len); // <midword> 462 if (lp->sl_midword == NULL) 463 goto endFAIL; 464 break; 465 466 case SN_PREFCOND: 467 res = read_prefcond_section(fd, lp); 468 break; 469 470 case SN_REP: 471 res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first); 472 break; 473 474 case SN_REPSAL: 475 res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first); 476 break; 477 478 case SN_SAL: 479 res = read_sal_section(fd, lp); 480 break; 481 482 case SN_SOFO: 483 res = read_sofo_section(fd, lp); 484 break; 485 486 case SN_MAP: 487 p = read_string(fd, len); // <mapstr> 488 if (p == NULL) 489 goto endFAIL; 490 set_map_str(lp, p); 491 vim_free(p); 492 break; 493 494 case SN_WORDS: 495 res = read_words_section(fd, lp, len); 496 break; 497 498 case SN_SUGFILE: 499 lp->sl_sugtime = get8ctime(fd); // <timestamp> 500 break; 501 502 case SN_NOSPLITSUGS: 503 lp->sl_nosplitsugs = TRUE; 504 break; 505 506 case SN_NOCOMPOUNDSUGS: 507 lp->sl_nocompoundsugs = TRUE; 508 break; 509 510 case SN_COMPOUND: 511 res = read_compound(fd, lp, len); 512 break; 513 514 case SN_NOBREAK: 515 lp->sl_nobreak = TRUE; 516 break; 517 518 case SN_SYLLABLE: 519 lp->sl_syllable = read_string(fd, len); // <syllable> 520 if (lp->sl_syllable == NULL) 521 goto endFAIL; 522 if (init_syl_tab(lp) != OK) 523 goto endFAIL; 524 break; 525 526 default: 527 // Unsupported section. When it's required give an error 528 // message. When it's not required skip the contents. 529 if (c & SNF_REQUIRED) 530 { 531 emsg(_("E770: Unsupported section in spell file")); 532 goto endFAIL; 533 } 534 while (--len >= 0) 535 if (getc(fd) < 0) 536 goto truncerr; 537 break; 538 } 539 someerror: 540 if (res == SP_FORMERROR) 541 { 542 emsg(_(e_format)); 543 goto endFAIL; 544 } 545 if (res == SP_TRUNCERROR) 546 { 547 truncerr: 548 emsg(_(e_spell_trunc)); 549 goto endFAIL; 550 } 551 if (res == SP_OTHERERROR) 552 goto endFAIL; 553 } 554 555 // <LWORDTREE> 556 res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fbyts_len, 557 &lp->sl_fidxs, FALSE, 0); 558 if (res != 0) 559 goto someerror; 560 561 // <KWORDTREE> 562 res = spell_read_tree(fd, &lp->sl_kbyts, NULL, &lp->sl_kidxs, FALSE, 0); 563 if (res != 0) 564 goto someerror; 565 566 // <PREFIXTREE> 567 res = spell_read_tree(fd, &lp->sl_pbyts, NULL, &lp->sl_pidxs, TRUE, 568 lp->sl_prefixcnt); 569 if (res != 0) 570 goto someerror; 571 572 // For a new file link it in the list of spell files. 573 if (old_lp == NULL && lang != NULL) 574 { 575 lp->sl_next = first_lang; 576 first_lang = lp; 577 } 578 579 goto endOK; 580 581 endFAIL: 582 if (lang != NULL) 583 // truncating the name signals the error to spell_load_lang() 584 *lang = NUL; 585 if (lp != NULL && old_lp == NULL) 586 slang_free(lp); 587 lp = NULL; 588 589 endOK: 590 if (fd != NULL) 591 fclose(fd); 592 if (did_estack_push) 593 { 594 ESTACK_CHECK_NOW 595 estack_pop(); 596 } 597 598 return lp; 599 } 600 601 /* 602 * Fill in the wordcount fields for a trie. 603 * Returns the total number of words. 604 */ 605 static void 606 tree_count_words(char_u *byts, idx_T *idxs) 607 { 608 int depth; 609 idx_T arridx[MAXWLEN]; 610 int curi[MAXWLEN]; 611 int c; 612 idx_T n; 613 int wordcount[MAXWLEN]; 614 615 arridx[0] = 0; 616 curi[0] = 1; 617 wordcount[0] = 0; 618 depth = 0; 619 while (depth >= 0 && !got_int) 620 { 621 if (curi[depth] > byts[arridx[depth]]) 622 { 623 // Done all bytes at this node, go up one level. 624 idxs[arridx[depth]] = wordcount[depth]; 625 if (depth > 0) 626 wordcount[depth - 1] += wordcount[depth]; 627 628 --depth; 629 fast_breakcheck(); 630 } 631 else 632 { 633 // Do one more byte at this node. 634 n = arridx[depth] + curi[depth]; 635 ++curi[depth]; 636 637 c = byts[n]; 638 if (c == 0) 639 { 640 // End of word, count it. 641 ++wordcount[depth]; 642 643 // Skip over any other NUL bytes (same word with different 644 // flags). 645 while (byts[n + 1] == 0) 646 { 647 ++n; 648 ++curi[depth]; 649 } 650 } 651 else 652 { 653 // Normal char, go one level deeper to count the words. 654 ++depth; 655 arridx[depth] = idxs[n]; 656 curi[depth] = 1; 657 wordcount[depth] = 0; 658 } 659 } 660 } 661 } 662 663 /* 664 * Load the .sug files for languages that have one and weren't loaded yet. 665 */ 666 void 667 suggest_load_files(void) 668 { 669 langp_T *lp; 670 int lpi; 671 slang_T *slang; 672 char_u *dotp; 673 FILE *fd; 674 char_u buf[MAXWLEN]; 675 int i; 676 time_t timestamp; 677 int wcount; 678 int wordnr; 679 garray_T ga; 680 int c; 681 682 // Do this for all languages that support sound folding. 683 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 684 { 685 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 686 slang = lp->lp_slang; 687 if (slang->sl_sugtime != 0 && !slang->sl_sugloaded) 688 { 689 // Change ".spl" to ".sug" and open the file. When the file isn't 690 // found silently skip it. Do set "sl_sugloaded" so that we 691 // don't try again and again. 692 slang->sl_sugloaded = TRUE; 693 694 dotp = vim_strrchr(slang->sl_fname, '.'); 695 if (dotp == NULL || fnamecmp(dotp, ".spl") != 0) 696 continue; 697 STRCPY(dotp, ".sug"); 698 fd = mch_fopen((char *)slang->sl_fname, "r"); 699 if (fd == NULL) 700 goto nextone; 701 702 /* 703 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 704 */ 705 for (i = 0; i < VIMSUGMAGICL; ++i) 706 buf[i] = getc(fd); // <fileID> 707 if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0) 708 { 709 semsg(_("E778: This does not look like a .sug file: %s"), 710 slang->sl_fname); 711 goto nextone; 712 } 713 c = getc(fd); // <versionnr> 714 if (c < VIMSUGVERSION) 715 { 716 semsg(_("E779: Old .sug file, needs to be updated: %s"), 717 slang->sl_fname); 718 goto nextone; 719 } 720 else if (c > VIMSUGVERSION) 721 { 722 semsg(_("E780: .sug file is for newer version of Vim: %s"), 723 slang->sl_fname); 724 goto nextone; 725 } 726 727 // Check the timestamp, it must be exactly the same as the one in 728 // the .spl file. Otherwise the word numbers won't match. 729 timestamp = get8ctime(fd); // <timestamp> 730 if (timestamp != slang->sl_sugtime) 731 { 732 semsg(_("E781: .sug file doesn't match .spl file: %s"), 733 slang->sl_fname); 734 goto nextone; 735 } 736 737 /* 738 * <SUGWORDTREE>: <wordtree> 739 * Read the trie with the soundfolded words. 740 */ 741 if (spell_read_tree(fd, &slang->sl_sbyts, NULL, &slang->sl_sidxs, 742 FALSE, 0) != 0) 743 { 744 someerror: 745 semsg(_("E782: error while reading .sug file: %s"), 746 slang->sl_fname); 747 slang_clear_sug(slang); 748 goto nextone; 749 } 750 751 /* 752 * <SUGTABLE>: <sugwcount> <sugline> ... 753 * 754 * Read the table with word numbers. We use a file buffer for 755 * this, because it's so much like a file with lines. Makes it 756 * possible to swap the info and save on memory use. 757 */ 758 slang->sl_sugbuf = open_spellbuf(); 759 if (slang->sl_sugbuf == NULL) 760 goto someerror; 761 // <sugwcount> 762 wcount = get4c(fd); 763 if (wcount < 0) 764 goto someerror; 765 766 // Read all the wordnr lists into the buffer, one NUL terminated 767 // list per line. 768 ga_init2(&ga, 1, 100); 769 for (wordnr = 0; wordnr < wcount; ++wordnr) 770 { 771 ga.ga_len = 0; 772 for (;;) 773 { 774 c = getc(fd); // <sugline> 775 if (c < 0 || ga_grow(&ga, 1) == FAIL) 776 goto someerror; 777 ((char_u *)ga.ga_data)[ga.ga_len++] = c; 778 if (c == NUL) 779 break; 780 } 781 if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr, 782 ga.ga_data, ga.ga_len, TRUE) == FAIL) 783 goto someerror; 784 } 785 ga_clear(&ga); 786 787 /* 788 * Need to put word counts in the word tries, so that we can find 789 * a word by its number. 790 */ 791 tree_count_words(slang->sl_fbyts, slang->sl_fidxs); 792 tree_count_words(slang->sl_sbyts, slang->sl_sidxs); 793 794 nextone: 795 if (fd != NULL) 796 fclose(fd); 797 STRCPY(dotp, ".spl"); 798 } 799 } 800 } 801 802 803 /* 804 * Read a length field from "fd" in "cnt_bytes" bytes. 805 * Allocate memory, read the string into it and add a NUL at the end. 806 * Returns NULL when the count is zero. 807 * Sets "*cntp" to SP_*ERROR when there is an error, length of the result 808 * otherwise. 809 */ 810 static char_u * 811 read_cnt_string(FILE *fd, int cnt_bytes, int *cntp) 812 { 813 int cnt = 0; 814 int i; 815 char_u *str; 816 817 // read the length bytes, MSB first 818 for (i = 0; i < cnt_bytes; ++i) 819 { 820 int c = getc(fd); 821 822 if (c == EOF) 823 { 824 *cntp = SP_TRUNCERROR; 825 return NULL; 826 } 827 cnt = (cnt << 8) + (unsigned)c; 828 } 829 *cntp = cnt; 830 if (cnt == 0) 831 return NULL; // nothing to read, return NULL 832 833 str = read_string(fd, cnt); 834 if (str == NULL) 835 *cntp = SP_OTHERERROR; 836 return str; 837 } 838 839 /* 840 * Read SN_REGION: <regionname> ... 841 * Return SP_*ERROR flags. 842 */ 843 static int 844 read_region_section(FILE *fd, slang_T *lp, int len) 845 { 846 int i; 847 848 if (len > MAXREGIONS * 2) 849 return SP_FORMERROR; 850 for (i = 0; i < len; ++i) 851 lp->sl_regions[i] = getc(fd); // <regionname> 852 lp->sl_regions[len] = NUL; 853 return 0; 854 } 855 856 /* 857 * Read SN_CHARFLAGS section: <charflagslen> <charflags> 858 * <folcharslen> <folchars> 859 * Return SP_*ERROR flags. 860 */ 861 static int 862 read_charflags_section(FILE *fd) 863 { 864 char_u *flags; 865 char_u *fol; 866 int flagslen, follen; 867 868 // <charflagslen> <charflags> 869 flags = read_cnt_string(fd, 1, &flagslen); 870 if (flagslen < 0) 871 return flagslen; 872 873 // <folcharslen> <folchars> 874 fol = read_cnt_string(fd, 2, &follen); 875 if (follen < 0) 876 { 877 vim_free(flags); 878 return follen; 879 } 880 881 // Set the word-char flags and fill SPELL_ISUPPER() table. 882 if (flags != NULL && fol != NULL) 883 set_spell_charflags(flags, flagslen, fol); 884 885 vim_free(flags); 886 vim_free(fol); 887 888 // When <charflagslen> is zero then <fcharlen> must also be zero. 889 if ((flags == NULL) != (fol == NULL)) 890 return SP_FORMERROR; 891 return 0; 892 } 893 894 /* 895 * Read SN_PREFCOND section. 896 * Return SP_*ERROR flags. 897 */ 898 static int 899 read_prefcond_section(FILE *fd, slang_T *lp) 900 { 901 int cnt; 902 int i; 903 int n; 904 char_u *p; 905 char_u buf[MAXWLEN + 1]; 906 907 // <prefcondcnt> <prefcond> ... 908 cnt = get2c(fd); // <prefcondcnt> 909 if (cnt <= 0) 910 return SP_FORMERROR; 911 912 lp->sl_prefprog = ALLOC_CLEAR_MULT(regprog_T *, cnt); 913 if (lp->sl_prefprog == NULL) 914 return SP_OTHERERROR; 915 lp->sl_prefixcnt = cnt; 916 917 for (i = 0; i < cnt; ++i) 918 { 919 // <prefcond> : <condlen> <condstr> 920 n = getc(fd); // <condlen> 921 if (n < 0 || n >= MAXWLEN) 922 return SP_FORMERROR; 923 924 // When <condlen> is zero we have an empty condition. Otherwise 925 // compile the regexp program used to check for the condition. 926 if (n > 0) 927 { 928 buf[0] = '^'; // always match at one position only 929 p = buf + 1; 930 while (n-- > 0) 931 *p++ = getc(fd); // <condstr> 932 *p = NUL; 933 lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING); 934 } 935 } 936 return 0; 937 } 938 939 /* 940 * Read REP or REPSAL items section from "fd": <repcount> <rep> ... 941 * Return SP_*ERROR flags. 942 */ 943 static int 944 read_rep_section(FILE *fd, garray_T *gap, short *first) 945 { 946 int cnt; 947 fromto_T *ftp; 948 int i; 949 950 cnt = get2c(fd); // <repcount> 951 if (cnt < 0) 952 return SP_TRUNCERROR; 953 954 if (ga_grow(gap, cnt) == FAIL) 955 return SP_OTHERERROR; 956 957 // <rep> : <repfromlen> <repfrom> <reptolen> <repto> 958 for (; gap->ga_len < cnt; ++gap->ga_len) 959 { 960 ftp = &((fromto_T *)gap->ga_data)[gap->ga_len]; 961 ftp->ft_from = read_cnt_string(fd, 1, &i); 962 if (i < 0) 963 return i; 964 if (i == 0) 965 return SP_FORMERROR; 966 ftp->ft_to = read_cnt_string(fd, 1, &i); 967 if (i <= 0) 968 { 969 vim_free(ftp->ft_from); 970 if (i < 0) 971 return i; 972 return SP_FORMERROR; 973 } 974 } 975 976 // Fill the first-index table. 977 for (i = 0; i < 256; ++i) 978 first[i] = -1; 979 for (i = 0; i < gap->ga_len; ++i) 980 { 981 ftp = &((fromto_T *)gap->ga_data)[i]; 982 if (first[*ftp->ft_from] == -1) 983 first[*ftp->ft_from] = i; 984 } 985 return 0; 986 } 987 988 /* 989 * Read SN_SAL section: <salflags> <salcount> <sal> ... 990 * Return SP_*ERROR flags. 991 */ 992 static int 993 read_sal_section(FILE *fd, slang_T *slang) 994 { 995 int i; 996 int cnt; 997 garray_T *gap; 998 salitem_T *smp; 999 int ccnt; 1000 char_u *p; 1001 1002 slang->sl_sofo = FALSE; 1003 1004 i = getc(fd); // <salflags> 1005 if (i & SAL_F0LLOWUP) 1006 slang->sl_followup = TRUE; 1007 if (i & SAL_COLLAPSE) 1008 slang->sl_collapse = TRUE; 1009 if (i & SAL_REM_ACCENTS) 1010 slang->sl_rem_accents = TRUE; 1011 1012 cnt = get2c(fd); // <salcount> 1013 if (cnt < 0) 1014 return SP_TRUNCERROR; 1015 1016 gap = &slang->sl_sal; 1017 ga_init2(gap, sizeof(salitem_T), 10); 1018 if (ga_grow(gap, cnt + 1) == FAIL) 1019 return SP_OTHERERROR; 1020 1021 // <sal> : <salfromlen> <salfrom> <saltolen> <salto> 1022 for (; gap->ga_len < cnt; ++gap->ga_len) 1023 { 1024 int c = NUL; 1025 1026 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 1027 ccnt = getc(fd); // <salfromlen> 1028 if (ccnt < 0) 1029 return SP_TRUNCERROR; 1030 if ((p = alloc(ccnt + 2)) == NULL) 1031 return SP_OTHERERROR; 1032 smp->sm_lead = p; 1033 1034 // Read up to the first special char into sm_lead. 1035 for (i = 0; i < ccnt; ++i) 1036 { 1037 c = getc(fd); // <salfrom> 1038 if (vim_strchr((char_u *)"0123456789(-<^$", c) != NULL) 1039 break; 1040 *p++ = c; 1041 } 1042 smp->sm_leadlen = (int)(p - smp->sm_lead); 1043 *p++ = NUL; 1044 1045 // Put (abc) chars in sm_oneof, if any. 1046 if (c == '(') 1047 { 1048 smp->sm_oneof = p; 1049 for (++i; i < ccnt; ++i) 1050 { 1051 c = getc(fd); // <salfrom> 1052 if (c == ')') 1053 break; 1054 *p++ = c; 1055 } 1056 *p++ = NUL; 1057 if (++i < ccnt) 1058 c = getc(fd); 1059 } 1060 else 1061 smp->sm_oneof = NULL; 1062 1063 // Any following chars go in sm_rules. 1064 smp->sm_rules = p; 1065 if (i < ccnt) 1066 // store the char we got while checking for end of sm_lead 1067 *p++ = c; 1068 for (++i; i < ccnt; ++i) 1069 *p++ = getc(fd); // <salfrom> 1070 *p++ = NUL; 1071 1072 // <saltolen> <salto> 1073 smp->sm_to = read_cnt_string(fd, 1, &ccnt); 1074 if (ccnt < 0) 1075 { 1076 vim_free(smp->sm_lead); 1077 return ccnt; 1078 } 1079 1080 if (has_mbyte) 1081 { 1082 // convert the multi-byte strings to wide char strings 1083 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 1084 smp->sm_leadlen = mb_charlen(smp->sm_lead); 1085 if (smp->sm_oneof == NULL) 1086 smp->sm_oneof_w = NULL; 1087 else 1088 smp->sm_oneof_w = mb_str2wide(smp->sm_oneof); 1089 if (smp->sm_to == NULL) 1090 smp->sm_to_w = NULL; 1091 else 1092 smp->sm_to_w = mb_str2wide(smp->sm_to); 1093 if (smp->sm_lead_w == NULL 1094 || (smp->sm_oneof_w == NULL && smp->sm_oneof != NULL) 1095 || (smp->sm_to_w == NULL && smp->sm_to != NULL)) 1096 { 1097 vim_free(smp->sm_lead); 1098 vim_free(smp->sm_to); 1099 vim_free(smp->sm_lead_w); 1100 vim_free(smp->sm_oneof_w); 1101 vim_free(smp->sm_to_w); 1102 return SP_OTHERERROR; 1103 } 1104 } 1105 } 1106 1107 if (gap->ga_len > 0) 1108 { 1109 // Add one extra entry to mark the end with an empty sm_lead. Avoids 1110 // that we need to check the index every time. 1111 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 1112 if ((p = alloc(1)) == NULL) 1113 return SP_OTHERERROR; 1114 p[0] = NUL; 1115 smp->sm_lead = p; 1116 smp->sm_leadlen = 0; 1117 smp->sm_oneof = NULL; 1118 smp->sm_rules = p; 1119 smp->sm_to = NULL; 1120 if (has_mbyte) 1121 { 1122 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 1123 smp->sm_leadlen = 0; 1124 smp->sm_oneof_w = NULL; 1125 smp->sm_to_w = NULL; 1126 } 1127 ++gap->ga_len; 1128 } 1129 1130 // Fill the first-index table. 1131 set_sal_first(slang); 1132 1133 return 0; 1134 } 1135 1136 /* 1137 * Read SN_WORDS: <word> ... 1138 * Return SP_*ERROR flags. 1139 */ 1140 static int 1141 read_words_section(FILE *fd, slang_T *lp, int len) 1142 { 1143 int done = 0; 1144 int i; 1145 int c; 1146 char_u word[MAXWLEN]; 1147 1148 while (done < len) 1149 { 1150 // Read one word at a time. 1151 for (i = 0; ; ++i) 1152 { 1153 c = getc(fd); 1154 if (c == EOF) 1155 return SP_TRUNCERROR; 1156 word[i] = c; 1157 if (word[i] == NUL) 1158 break; 1159 if (i == MAXWLEN - 1) 1160 return SP_FORMERROR; 1161 } 1162 1163 // Init the count to 10. 1164 count_common_word(lp, word, -1, 10); 1165 done += i + 1; 1166 } 1167 return 0; 1168 } 1169 1170 /* 1171 * SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 1172 * Return SP_*ERROR flags. 1173 */ 1174 static int 1175 read_sofo_section(FILE *fd, slang_T *slang) 1176 { 1177 int cnt; 1178 char_u *from, *to; 1179 int res; 1180 1181 slang->sl_sofo = TRUE; 1182 1183 // <sofofromlen> <sofofrom> 1184 from = read_cnt_string(fd, 2, &cnt); 1185 if (cnt < 0) 1186 return cnt; 1187 1188 // <sofotolen> <sofoto> 1189 to = read_cnt_string(fd, 2, &cnt); 1190 if (cnt < 0) 1191 { 1192 vim_free(from); 1193 return cnt; 1194 } 1195 1196 // Store the info in slang->sl_sal and/or slang->sl_sal_first. 1197 if (from != NULL && to != NULL) 1198 res = set_sofo(slang, from, to); 1199 else if (from != NULL || to != NULL) 1200 res = SP_FORMERROR; // only one of two strings is an error 1201 else 1202 res = 0; 1203 1204 vim_free(from); 1205 vim_free(to); 1206 return res; 1207 } 1208 1209 /* 1210 * Read the compound section from the .spl file: 1211 * <compmax> <compminlen> <compsylmax> <compoptions> <compflags> 1212 * Returns SP_*ERROR flags. 1213 */ 1214 static int 1215 read_compound(FILE *fd, slang_T *slang, int len) 1216 { 1217 int todo = len; 1218 int c; 1219 int atstart; 1220 char_u *pat; 1221 char_u *pp; 1222 char_u *cp; 1223 char_u *ap; 1224 char_u *crp; 1225 int cnt; 1226 garray_T *gap; 1227 1228 if (todo < 2) 1229 return SP_FORMERROR; // need at least two bytes 1230 1231 --todo; 1232 c = getc(fd); // <compmax> 1233 if (c < 2) 1234 c = MAXWLEN; 1235 slang->sl_compmax = c; 1236 1237 --todo; 1238 c = getc(fd); // <compminlen> 1239 if (c < 1) 1240 c = 0; 1241 slang->sl_compminlen = c; 1242 1243 --todo; 1244 c = getc(fd); // <compsylmax> 1245 if (c < 1) 1246 c = MAXWLEN; 1247 slang->sl_compsylmax = c; 1248 1249 c = getc(fd); // <compoptions> 1250 if (c != 0) 1251 ungetc(c, fd); // be backwards compatible with Vim 7.0b 1252 else 1253 { 1254 --todo; 1255 c = getc(fd); // only use the lower byte for now 1256 --todo; 1257 slang->sl_compoptions = c; 1258 1259 gap = &slang->sl_comppat; 1260 c = get2c(fd); // <comppatcount> 1261 todo -= 2; 1262 ga_init2(gap, sizeof(char_u *), c); 1263 if (ga_grow(gap, c) == OK) 1264 while (--c >= 0) 1265 { 1266 ((char_u **)(gap->ga_data))[gap->ga_len++] = 1267 read_cnt_string(fd, 1, &cnt); 1268 // <comppatlen> <comppattext> 1269 if (cnt < 0) 1270 return cnt; 1271 todo -= cnt + 1; 1272 } 1273 } 1274 if (todo < 0) 1275 return SP_FORMERROR; 1276 1277 // Turn the COMPOUNDRULE items into a regexp pattern: 1278 // "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$". 1279 // Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes. 1280 // Conversion to utf-8 may double the size. 1281 c = todo * 2 + 7; 1282 if (enc_utf8) 1283 c += todo * 2; 1284 pat = alloc(c); 1285 if (pat == NULL) 1286 return SP_OTHERERROR; 1287 1288 // We also need a list of all flags that can appear at the start and one 1289 // for all flags. 1290 cp = alloc(todo + 1); 1291 if (cp == NULL) 1292 { 1293 vim_free(pat); 1294 return SP_OTHERERROR; 1295 } 1296 slang->sl_compstartflags = cp; 1297 *cp = NUL; 1298 1299 ap = alloc(todo + 1); 1300 if (ap == NULL) 1301 { 1302 vim_free(pat); 1303 return SP_OTHERERROR; 1304 } 1305 slang->sl_compallflags = ap; 1306 *ap = NUL; 1307 1308 // And a list of all patterns in their original form, for checking whether 1309 // compounding may work in match_compoundrule(). This is freed when we 1310 // encounter a wildcard, the check doesn't work then. 1311 crp = alloc(todo + 1); 1312 slang->sl_comprules = crp; 1313 1314 pp = pat; 1315 *pp++ = '^'; 1316 *pp++ = '\\'; 1317 *pp++ = '('; 1318 1319 atstart = 1; 1320 while (todo-- > 0) 1321 { 1322 c = getc(fd); // <compflags> 1323 if (c == EOF) 1324 { 1325 vim_free(pat); 1326 return SP_TRUNCERROR; 1327 } 1328 1329 // Add all flags to "sl_compallflags". 1330 if (vim_strchr((char_u *)"?*+[]/", c) == NULL 1331 && !byte_in_str(slang->sl_compallflags, c)) 1332 { 1333 *ap++ = c; 1334 *ap = NUL; 1335 } 1336 1337 if (atstart != 0) 1338 { 1339 // At start of item: copy flags to "sl_compstartflags". For a 1340 // [abc] item set "atstart" to 2 and copy up to the ']'. 1341 if (c == '[') 1342 atstart = 2; 1343 else if (c == ']') 1344 atstart = 0; 1345 else 1346 { 1347 if (!byte_in_str(slang->sl_compstartflags, c)) 1348 { 1349 *cp++ = c; 1350 *cp = NUL; 1351 } 1352 if (atstart == 1) 1353 atstart = 0; 1354 } 1355 } 1356 1357 // Copy flag to "sl_comprules", unless we run into a wildcard. 1358 if (crp != NULL) 1359 { 1360 if (c == '?' || c == '+' || c == '*') 1361 { 1362 VIM_CLEAR(slang->sl_comprules); 1363 crp = NULL; 1364 } 1365 else 1366 *crp++ = c; 1367 } 1368 1369 if (c == '/') // slash separates two items 1370 { 1371 *pp++ = '\\'; 1372 *pp++ = '|'; 1373 atstart = 1; 1374 } 1375 else // normal char, "[abc]" and '*' are copied as-is 1376 { 1377 if (c == '?' || c == '+' || c == '~') 1378 *pp++ = '\\'; // "a?" becomes "a\?", "a+" becomes "a\+" 1379 if (enc_utf8) 1380 pp += mb_char2bytes(c, pp); 1381 else 1382 *pp++ = c; 1383 } 1384 } 1385 1386 *pp++ = '\\'; 1387 *pp++ = ')'; 1388 *pp++ = '$'; 1389 *pp = NUL; 1390 1391 if (crp != NULL) 1392 *crp = NUL; 1393 1394 slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT); 1395 vim_free(pat); 1396 if (slang->sl_compprog == NULL) 1397 return SP_FORMERROR; 1398 1399 return 0; 1400 } 1401 1402 /* 1403 * Set the SOFOFROM and SOFOTO items in language "lp". 1404 * Returns SP_*ERROR flags when there is something wrong. 1405 */ 1406 static int 1407 set_sofo(slang_T *lp, char_u *from, char_u *to) 1408 { 1409 int i; 1410 1411 garray_T *gap; 1412 char_u *s; 1413 char_u *p; 1414 int c; 1415 int *inp; 1416 1417 if (has_mbyte) 1418 { 1419 // Use "sl_sal" as an array with 256 pointers to a list of wide 1420 // characters. The index is the low byte of the character. 1421 // The list contains from-to pairs with a terminating NUL. 1422 // sl_sal_first[] is used for latin1 "from" characters. 1423 gap = &lp->sl_sal; 1424 ga_init2(gap, sizeof(int *), 1); 1425 if (ga_grow(gap, 256) == FAIL) 1426 return SP_OTHERERROR; 1427 vim_memset(gap->ga_data, 0, sizeof(int *) * 256); 1428 gap->ga_len = 256; 1429 1430 // First count the number of items for each list. Temporarily use 1431 // sl_sal_first[] for this. 1432 for (p = from, s = to; *p != NUL && *s != NUL; ) 1433 { 1434 c = mb_cptr2char_adv(&p); 1435 MB_CPTR_ADV(s); 1436 if (c >= 256) 1437 ++lp->sl_sal_first[c & 0xff]; 1438 } 1439 if (*p != NUL || *s != NUL) // lengths differ 1440 return SP_FORMERROR; 1441 1442 // Allocate the lists. 1443 for (i = 0; i < 256; ++i) 1444 if (lp->sl_sal_first[i] > 0) 1445 { 1446 p = alloc(sizeof(int) * (lp->sl_sal_first[i] * 2 + 1)); 1447 if (p == NULL) 1448 return SP_OTHERERROR; 1449 ((int **)gap->ga_data)[i] = (int *)p; 1450 *(int *)p = 0; 1451 } 1452 1453 // Put the characters up to 255 in sl_sal_first[] the rest in a sl_sal 1454 // list. 1455 vim_memset(lp->sl_sal_first, 0, sizeof(salfirst_T) * 256); 1456 for (p = from, s = to; *p != NUL && *s != NUL; ) 1457 { 1458 c = mb_cptr2char_adv(&p); 1459 i = mb_cptr2char_adv(&s); 1460 if (c >= 256) 1461 { 1462 // Append the from-to chars at the end of the list with 1463 // the low byte. 1464 inp = ((int **)gap->ga_data)[c & 0xff]; 1465 while (*inp != 0) 1466 ++inp; 1467 *inp++ = c; // from char 1468 *inp++ = i; // to char 1469 *inp++ = NUL; // NUL at the end 1470 } 1471 else 1472 // mapping byte to char is done in sl_sal_first[] 1473 lp->sl_sal_first[c] = i; 1474 } 1475 } 1476 else 1477 { 1478 // mapping bytes to bytes is done in sl_sal_first[] 1479 if (STRLEN(from) != STRLEN(to)) 1480 return SP_FORMERROR; 1481 1482 for (i = 0; to[i] != NUL; ++i) 1483 lp->sl_sal_first[from[i]] = to[i]; 1484 lp->sl_sal.ga_len = 1; // indicates we have soundfolding 1485 } 1486 1487 return 0; 1488 } 1489 1490 /* 1491 * Fill the first-index table for "lp". 1492 */ 1493 static void 1494 set_sal_first(slang_T *lp) 1495 { 1496 salfirst_T *sfirst; 1497 int i; 1498 salitem_T *smp; 1499 int c; 1500 garray_T *gap = &lp->sl_sal; 1501 1502 sfirst = lp->sl_sal_first; 1503 for (i = 0; i < 256; ++i) 1504 sfirst[i] = -1; 1505 smp = (salitem_T *)gap->ga_data; 1506 for (i = 0; i < gap->ga_len; ++i) 1507 { 1508 if (has_mbyte) 1509 // Use the lowest byte of the first character. For latin1 it's 1510 // the character, for other encodings it should differ for most 1511 // characters. 1512 c = *smp[i].sm_lead_w & 0xff; 1513 else 1514 c = *smp[i].sm_lead; 1515 if (sfirst[c] == -1) 1516 { 1517 sfirst[c] = i; 1518 if (has_mbyte) 1519 { 1520 int n; 1521 1522 // Make sure all entries with this byte are following each 1523 // other. Move the ones that are in the wrong position. Do 1524 // keep the same ordering! 1525 while (i + 1 < gap->ga_len 1526 && (*smp[i + 1].sm_lead_w & 0xff) == c) 1527 // Skip over entry with same index byte. 1528 ++i; 1529 1530 for (n = 1; i + n < gap->ga_len; ++n) 1531 if ((*smp[i + n].sm_lead_w & 0xff) == c) 1532 { 1533 salitem_T tsal; 1534 1535 // Move entry with same index byte after the entries 1536 // we already found. 1537 ++i; 1538 --n; 1539 tsal = smp[i + n]; 1540 mch_memmove(smp + i + 1, smp + i, 1541 sizeof(salitem_T) * n); 1542 smp[i] = tsal; 1543 } 1544 } 1545 } 1546 } 1547 } 1548 1549 /* 1550 * Turn a multi-byte string into a wide character string. 1551 * Return it in allocated memory (NULL for out-of-memory) 1552 */ 1553 static int * 1554 mb_str2wide(char_u *s) 1555 { 1556 int *res; 1557 char_u *p; 1558 int i = 0; 1559 1560 res = ALLOC_MULT(int, mb_charlen(s) + 1); 1561 if (res != NULL) 1562 { 1563 for (p = s; *p != NUL; ) 1564 res[i++] = mb_ptr2char_adv(&p); 1565 res[i] = NUL; 1566 } 1567 return res; 1568 } 1569 1570 /* 1571 * Read a tree from the .spl or .sug file. 1572 * Allocates the memory and stores pointers in "bytsp" and "idxsp". 1573 * This is skipped when the tree has zero length. 1574 * Returns zero when OK, SP_ value for an error. 1575 */ 1576 static int 1577 spell_read_tree( 1578 FILE *fd, 1579 char_u **bytsp, 1580 long *bytsp_len, 1581 idx_T **idxsp, 1582 int prefixtree, // TRUE for the prefix tree 1583 int prefixcnt) // when "prefixtree" is TRUE: prefix count 1584 { 1585 long len; 1586 int idx; 1587 char_u *bp; 1588 idx_T *ip; 1589 1590 // The tree size was computed when writing the file, so that we can 1591 // allocate it as one long block. <nodecount> 1592 len = get4c(fd); 1593 if (len < 0) 1594 return SP_TRUNCERROR; 1595 if (len >= LONG_MAX / (long)sizeof(int)) 1596 // Invalid length, multiply with sizeof(int) would overflow. 1597 return SP_FORMERROR; 1598 if (len > 0) 1599 { 1600 // Allocate the byte array. 1601 bp = alloc(len); 1602 if (bp == NULL) 1603 return SP_OTHERERROR; 1604 *bytsp = bp; 1605 if (bytsp_len != NULL) 1606 *bytsp_len = len; 1607 1608 // Allocate the index array. 1609 ip = lalloc_clear(len * sizeof(int), TRUE); 1610 if (ip == NULL) 1611 return SP_OTHERERROR; 1612 *idxsp = ip; 1613 1614 // Recursively read the tree and store it in the array. 1615 idx = read_tree_node(fd, bp, ip, len, 0, prefixtree, prefixcnt); 1616 if (idx < 0) 1617 return idx; 1618 } 1619 return 0; 1620 } 1621 1622 /* 1623 * Read one row of siblings from the spell file and store it in the byte array 1624 * "byts" and index array "idxs". Recursively read the children. 1625 * 1626 * NOTE: The code here must match put_node()! 1627 * 1628 * Returns the index (>= 0) following the siblings. 1629 * Returns SP_TRUNCERROR if the file is shorter than expected. 1630 * Returns SP_FORMERROR if there is a format error. 1631 */ 1632 static idx_T 1633 read_tree_node( 1634 FILE *fd, 1635 char_u *byts, 1636 idx_T *idxs, 1637 int maxidx, // size of arrays 1638 idx_T startidx, // current index in "byts" and "idxs" 1639 int prefixtree, // TRUE for reading PREFIXTREE 1640 int maxprefcondnr) // maximum for <prefcondnr> 1641 { 1642 int len; 1643 int i; 1644 int n; 1645 idx_T idx = startidx; 1646 int c; 1647 int c2; 1648 #define SHARED_MASK 0x8000000 1649 1650 len = getc(fd); // <siblingcount> 1651 if (len <= 0) 1652 return SP_TRUNCERROR; 1653 1654 if (startidx + len >= maxidx) 1655 return SP_FORMERROR; 1656 byts[idx++] = len; 1657 1658 // Read the byte values, flag/region bytes and shared indexes. 1659 for (i = 1; i <= len; ++i) 1660 { 1661 c = getc(fd); // <byte> 1662 if (c < 0) 1663 return SP_TRUNCERROR; 1664 if (c <= BY_SPECIAL) 1665 { 1666 if (c == BY_NOFLAGS && !prefixtree) 1667 { 1668 // No flags, all regions. 1669 idxs[idx] = 0; 1670 c = 0; 1671 } 1672 else if (c != BY_INDEX) 1673 { 1674 if (prefixtree) 1675 { 1676 // Read the optional pflags byte, the prefix ID and the 1677 // condition nr. In idxs[] store the prefix ID in the low 1678 // byte, the condition index shifted up 8 bits, the flags 1679 // shifted up 24 bits. 1680 if (c == BY_FLAGS) 1681 c = getc(fd) << 24; // <pflags> 1682 else 1683 c = 0; 1684 1685 c |= getc(fd); // <affixID> 1686 1687 n = get2c(fd); // <prefcondnr> 1688 if (n >= maxprefcondnr) 1689 return SP_FORMERROR; 1690 c |= (n << 8); 1691 } 1692 else // c must be BY_FLAGS or BY_FLAGS2 1693 { 1694 // Read flags and optional region and prefix ID. In 1695 // idxs[] the flags go in the low two bytes, region above 1696 // that and prefix ID above the region. 1697 c2 = c; 1698 c = getc(fd); // <flags> 1699 if (c2 == BY_FLAGS2) 1700 c = (getc(fd) << 8) + c; // <flags2> 1701 if (c & WF_REGION) 1702 c = (getc(fd) << 16) + c; // <region> 1703 if (c & WF_AFX) 1704 c = (getc(fd) << 24) + c; // <affixID> 1705 } 1706 1707 idxs[idx] = c; 1708 c = 0; 1709 } 1710 else // c == BY_INDEX 1711 { 1712 // <nodeidx> 1713 n = get3c(fd); 1714 if (n < 0 || n >= maxidx) 1715 return SP_FORMERROR; 1716 idxs[idx] = n + SHARED_MASK; 1717 c = getc(fd); // <xbyte> 1718 } 1719 } 1720 byts[idx++] = c; 1721 } 1722 1723 // Recursively read the children for non-shared siblings. 1724 // Skip the end-of-word ones (zero byte value) and the shared ones (and 1725 // remove SHARED_MASK) 1726 for (i = 1; i <= len; ++i) 1727 if (byts[startidx + i] != 0) 1728 { 1729 if (idxs[startidx + i] & SHARED_MASK) 1730 idxs[startidx + i] &= ~SHARED_MASK; 1731 else 1732 { 1733 idxs[startidx + i] = idx; 1734 idx = read_tree_node(fd, byts, idxs, maxidx, idx, 1735 prefixtree, maxprefcondnr); 1736 if (idx < 0) 1737 break; 1738 } 1739 } 1740 1741 return idx; 1742 } 1743 1744 /* 1745 * Reload the spell file "fname" if it's loaded. 1746 */ 1747 static void 1748 spell_reload_one( 1749 char_u *fname, 1750 int added_word) // invoked through "zg" 1751 { 1752 slang_T *slang; 1753 int didit = FALSE; 1754 1755 FOR_ALL_SPELL_LANGS(slang) 1756 { 1757 if (fullpathcmp(fname, slang->sl_fname, FALSE, TRUE) == FPC_SAME) 1758 { 1759 slang_clear(slang); 1760 if (spell_load_file(fname, NULL, slang, FALSE) == NULL) 1761 // reloading failed, clear the language 1762 slang_clear(slang); 1763 redraw_all_later(SOME_VALID); 1764 didit = TRUE; 1765 } 1766 } 1767 1768 // When "zg" was used and the file wasn't loaded yet, should redo 1769 // 'spelllang' to load it now. 1770 if (added_word && !didit) 1771 did_set_spelllang(curwin); 1772 } 1773 1774 1775 /* 1776 * Functions for ":mkspell". 1777 */ 1778 1779 #define MAXLINELEN 500 // Maximum length in bytes of a line in a .aff 1780 // and .dic file. 1781 /* 1782 * Main structure to store the contents of a ".aff" file. 1783 */ 1784 typedef struct afffile_S 1785 { 1786 char_u *af_enc; // "SET", normalized, alloc'ed string or NULL 1787 int af_flagtype; // AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG 1788 unsigned af_rare; // RARE ID for rare word 1789 unsigned af_keepcase; // KEEPCASE ID for keep-case word 1790 unsigned af_bad; // BAD ID for banned word 1791 unsigned af_needaffix; // NEEDAFFIX ID 1792 unsigned af_circumfix; // CIRCUMFIX ID 1793 unsigned af_needcomp; // NEEDCOMPOUND ID 1794 unsigned af_comproot; // COMPOUNDROOT ID 1795 unsigned af_compforbid; // COMPOUNDFORBIDFLAG ID 1796 unsigned af_comppermit; // COMPOUNDPERMITFLAG ID 1797 unsigned af_nosuggest; // NOSUGGEST ID 1798 int af_pfxpostpone; // postpone prefixes without chop string and 1799 // without flags 1800 int af_ignoreextra; // IGNOREEXTRA present 1801 hashtab_T af_pref; // hashtable for prefixes, affheader_T 1802 hashtab_T af_suff; // hashtable for suffixes, affheader_T 1803 hashtab_T af_comp; // hashtable for compound flags, compitem_T 1804 } afffile_T; 1805 1806 #define AFT_CHAR 0 // flags are one character 1807 #define AFT_LONG 1 // flags are two characters 1808 #define AFT_CAPLONG 2 // flags are one or two characters 1809 #define AFT_NUM 3 // flags are numbers, comma separated 1810 1811 typedef struct affentry_S affentry_T; 1812 // Affix entry from ".aff" file. Used for prefixes and suffixes. 1813 struct affentry_S 1814 { 1815 affentry_T *ae_next; // next affix with same name/number 1816 char_u *ae_chop; // text to chop off basic word (can be NULL) 1817 char_u *ae_add; // text to add to basic word (can be NULL) 1818 char_u *ae_flags; // flags on the affix (can be NULL) 1819 char_u *ae_cond; // condition (NULL for ".") 1820 regprog_T *ae_prog; // regexp program for ae_cond or NULL 1821 char ae_compforbid; // COMPOUNDFORBIDFLAG found 1822 char ae_comppermit; // COMPOUNDPERMITFLAG found 1823 }; 1824 1825 #define AH_KEY_LEN 17 // 2 x 8 bytes + NUL 1826 1827 // Affix header from ".aff" file. Used for af_pref and af_suff. 1828 typedef struct affheader_S 1829 { 1830 char_u ah_key[AH_KEY_LEN]; // key for hashtab == name of affix 1831 unsigned ah_flag; // affix name as number, uses "af_flagtype" 1832 int ah_newID; // prefix ID after renumbering; 0 if not used 1833 int ah_combine; // suffix may combine with prefix 1834 int ah_follows; // another affix block should be following 1835 affentry_T *ah_first; // first affix entry 1836 } affheader_T; 1837 1838 #define HI2AH(hi) ((affheader_T *)(hi)->hi_key) 1839 1840 // Flag used in compound items. 1841 typedef struct compitem_S 1842 { 1843 char_u ci_key[AH_KEY_LEN]; // key for hashtab == name of compound 1844 unsigned ci_flag; // affix name as number, uses "af_flagtype" 1845 int ci_newID; // affix ID after renumbering. 1846 } compitem_T; 1847 1848 #define HI2CI(hi) ((compitem_T *)(hi)->hi_key) 1849 1850 /* 1851 * Structure that is used to store the items in the word tree. This avoids 1852 * the need to keep track of each allocated thing, everything is freed all at 1853 * once after ":mkspell" is done. 1854 * Note: "sb_next" must be just before "sb_data" to make sure the alignment of 1855 * "sb_data" is correct for systems where pointers must be aligned on 1856 * pointer-size boundaries and sizeof(pointer) > sizeof(int) (e.g., Sparc). 1857 */ 1858 #define SBLOCKSIZE 16000 // size of sb_data 1859 typedef struct sblock_S sblock_T; 1860 struct sblock_S 1861 { 1862 int sb_used; // nr of bytes already in use 1863 sblock_T *sb_next; // next block in list 1864 char_u sb_data[1]; // data, actually longer 1865 }; 1866 1867 /* 1868 * A node in the tree. 1869 */ 1870 typedef struct wordnode_S wordnode_T; 1871 struct wordnode_S 1872 { 1873 union // shared to save space 1874 { 1875 char_u hashkey[6]; // the hash key, only used while compressing 1876 int index; // index in written nodes (valid after first 1877 // round) 1878 } wn_u1; 1879 union // shared to save space 1880 { 1881 wordnode_T *next; // next node with same hash key 1882 wordnode_T *wnode; // parent node that will write this node 1883 } wn_u2; 1884 wordnode_T *wn_child; // child (next byte in word) 1885 wordnode_T *wn_sibling; // next sibling (alternate byte in word, 1886 // always sorted) 1887 int wn_refs; // Nr. of references to this node. Only 1888 // relevant for first node in a list of 1889 // siblings, in following siblings it is 1890 // always one. 1891 char_u wn_byte; // Byte for this node. NUL for word end 1892 1893 // Info for when "wn_byte" is NUL. 1894 // In PREFIXTREE "wn_region" is used for the prefcondnr. 1895 // In the soundfolded word tree "wn_flags" has the MSW of the wordnr and 1896 // "wn_region" the LSW of the wordnr. 1897 char_u wn_affixID; // supported/required prefix ID or 0 1898 short_u wn_flags; // WF_ flags 1899 short wn_region; // region mask 1900 1901 #ifdef SPELL_PRINTTREE 1902 int wn_nr; // sequence nr for printing 1903 #endif 1904 }; 1905 1906 #define WN_MASK 0xffff // mask relevant bits of "wn_flags" 1907 1908 #define HI2WN(hi) (wordnode_T *)((hi)->hi_key) 1909 1910 /* 1911 * Info used while reading the spell files. 1912 */ 1913 typedef struct spellinfo_S 1914 { 1915 wordnode_T *si_foldroot; // tree with case-folded words 1916 long si_foldwcount; // nr of words in si_foldroot 1917 1918 wordnode_T *si_keeproot; // tree with keep-case words 1919 long si_keepwcount; // nr of words in si_keeproot 1920 1921 wordnode_T *si_prefroot; // tree with postponed prefixes 1922 1923 long si_sugtree; // creating the soundfolding trie 1924 1925 sblock_T *si_blocks; // memory blocks used 1926 long si_blocks_cnt; // memory blocks allocated 1927 int si_did_emsg; // TRUE when ran out of memory 1928 1929 long si_compress_cnt; // words to add before lowering 1930 // compression limit 1931 wordnode_T *si_first_free; // List of nodes that have been freed during 1932 // compression, linked by "wn_child" field. 1933 long si_free_count; // number of nodes in si_first_free 1934 #ifdef SPELL_PRINTTREE 1935 int si_wordnode_nr; // sequence nr for nodes 1936 #endif 1937 buf_T *si_spellbuf; // buffer used to store soundfold word table 1938 1939 int si_ascii; // handling only ASCII words 1940 int si_add; // addition file 1941 int si_clear_chartab; // when TRUE clear char tables 1942 int si_region; // region mask 1943 vimconv_T si_conv; // for conversion to 'encoding' 1944 int si_memtot; // runtime memory used 1945 int si_verbose; // verbose messages 1946 int si_msg_count; // number of words added since last message 1947 char_u *si_info; // info text chars or NULL 1948 int si_region_count; // number of regions supported (1 when there 1949 // are no regions) 1950 char_u si_region_name[MAXREGIONS * 2 + 1]; 1951 // region names; used only if 1952 // si_region_count > 1) 1953 1954 garray_T si_rep; // list of fromto_T entries from REP lines 1955 garray_T si_repsal; // list of fromto_T entries from REPSAL lines 1956 garray_T si_sal; // list of fromto_T entries from SAL lines 1957 char_u *si_sofofr; // SOFOFROM text 1958 char_u *si_sofoto; // SOFOTO text 1959 int si_nosugfile; // NOSUGFILE item found 1960 int si_nosplitsugs; // NOSPLITSUGS item found 1961 int si_nocompoundsugs; // NOCOMPOUNDSUGS item found 1962 int si_followup; // soundsalike: ? 1963 int si_collapse; // soundsalike: ? 1964 hashtab_T si_commonwords; // hashtable for common words 1965 time_t si_sugtime; // timestamp for .sug file 1966 int si_rem_accents; // soundsalike: remove accents 1967 garray_T si_map; // MAP info concatenated 1968 char_u *si_midword; // MIDWORD chars or NULL 1969 int si_compmax; // max nr of words for compounding 1970 int si_compminlen; // minimal length for compounding 1971 int si_compsylmax; // max nr of syllables for compounding 1972 int si_compoptions; // COMP_ flags 1973 garray_T si_comppat; // CHECKCOMPOUNDPATTERN items, each stored as 1974 // a string 1975 char_u *si_compflags; // flags used for compounding 1976 char_u si_nobreak; // NOBREAK 1977 char_u *si_syllable; // syllable string 1978 garray_T si_prefcond; // table with conditions for postponed 1979 // prefixes, each stored as a string 1980 int si_newprefID; // current value for ah_newID 1981 int si_newcompID; // current value for compound ID 1982 } spellinfo_T; 1983 1984 static int is_aff_rule(char_u **items, int itemcnt, char *rulename, int mincount); 1985 static void aff_process_flags(afffile_T *affile, affentry_T *entry); 1986 static int spell_info_item(char_u *s); 1987 static unsigned affitem2flag(int flagtype, char_u *item, char_u *fname, int lnum); 1988 static unsigned get_affitem(int flagtype, char_u **pp); 1989 static void process_compflags(spellinfo_T *spin, afffile_T *aff, char_u *compflags); 1990 static void check_renumber(spellinfo_T *spin); 1991 static void aff_check_number(int spinval, int affval, char *name); 1992 static void aff_check_string(char_u *spinval, char_u *affval, char *name); 1993 static int str_equal(char_u *s1, char_u *s2); 1994 static void add_fromto(spellinfo_T *spin, garray_T *gap, char_u *from, char_u *to); 1995 static int sal_to_bool(char_u *s); 1996 static int get_affix_flags(afffile_T *affile, char_u *afflist); 1997 static int get_pfxlist(afffile_T *affile, char_u *afflist, char_u *store_afflist); 1998 static void get_compflags(afffile_T *affile, char_u *afflist, char_u *store_afflist); 1999 static int store_aff_word(spellinfo_T *spin, char_u *word, char_u *afflist, afffile_T *affile, hashtab_T *ht, hashtab_T *xht, int condit, int flags, char_u *pfxlist, int pfxlen); 2000 static void *getroom(spellinfo_T *spin, size_t len, int align); 2001 static char_u *getroom_save(spellinfo_T *spin, char_u *s); 2002 static int store_word(spellinfo_T *spin, char_u *word, int flags, int region, char_u *pfxlist, int need_affix); 2003 static int tree_add_word(spellinfo_T *spin, char_u *word, wordnode_T *tree, int flags, int region, int affixID); 2004 static wordnode_T *get_wordnode(spellinfo_T *spin); 2005 static void free_wordnode(spellinfo_T *spin, wordnode_T *n); 2006 static void wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name); 2007 static long node_compress(spellinfo_T *spin, wordnode_T *node, hashtab_T *ht, long *tot); 2008 static int node_equal(wordnode_T *n1, wordnode_T *n2); 2009 static void clear_node(wordnode_T *node); 2010 static int put_node(FILE *fd, wordnode_T *node, int idx, int regionmask, int prefixtree); 2011 static int sug_filltree(spellinfo_T *spin, slang_T *slang); 2012 static int sug_maketable(spellinfo_T *spin); 2013 static int sug_filltable(spellinfo_T *spin, wordnode_T *node, int startwordnr, garray_T *gap); 2014 static int offset2bytes(int nr, char_u *buf); 2015 static void sug_write(spellinfo_T *spin, char_u *fname); 2016 static void spell_message(spellinfo_T *spin, char_u *str); 2017 static void init_spellfile(void); 2018 2019 // In the postponed prefixes tree wn_flags is used to store the WFP_ flags, 2020 // but it must be negative to indicate the prefix tree to tree_add_word(). 2021 // Use a negative number with the lower 8 bits zero. 2022 #define PFX_FLAGS -256 2023 2024 // flags for "condit" argument of store_aff_word() 2025 #define CONDIT_COMB 1 // affix must combine 2026 #define CONDIT_CFIX 2 // affix must have CIRCUMFIX flag 2027 #define CONDIT_SUF 4 // add a suffix for matching flags 2028 #define CONDIT_AFF 8 // word already has an affix 2029 2030 /* 2031 * Tunable parameters for when the tree is compressed. Filled from the 2032 * 'mkspellmem' option. 2033 */ 2034 static long compress_start = 30000; // memory / SBLOCKSIZE 2035 static long compress_inc = 100; // memory / SBLOCKSIZE 2036 static long compress_added = 500000; // word count 2037 2038 /* 2039 * Check the 'mkspellmem' option. Return FAIL if it's wrong. 2040 * Sets "sps_flags". 2041 */ 2042 int 2043 spell_check_msm(void) 2044 { 2045 char_u *p = p_msm; 2046 long start = 0; 2047 long incr = 0; 2048 long added = 0; 2049 2050 if (!VIM_ISDIGIT(*p)) 2051 return FAIL; 2052 // block count = (value * 1024) / SBLOCKSIZE (but avoid overflow) 2053 start = (getdigits(&p) * 10) / (SBLOCKSIZE / 102); 2054 if (*p != ',') 2055 return FAIL; 2056 ++p; 2057 if (!VIM_ISDIGIT(*p)) 2058 return FAIL; 2059 incr = (getdigits(&p) * 102) / (SBLOCKSIZE / 10); 2060 if (*p != ',') 2061 return FAIL; 2062 ++p; 2063 if (!VIM_ISDIGIT(*p)) 2064 return FAIL; 2065 added = getdigits(&p) * 1024; 2066 if (*p != NUL) 2067 return FAIL; 2068 2069 if (start == 0 || incr == 0 || added == 0 || incr > start) 2070 return FAIL; 2071 2072 compress_start = start; 2073 compress_inc = incr; 2074 compress_added = added; 2075 return OK; 2076 } 2077 2078 #ifdef SPELL_PRINTTREE 2079 /* 2080 * For debugging the tree code: print the current tree in a (more or less) 2081 * readable format, so that we can see what happens when adding a word and/or 2082 * compressing the tree. 2083 * Based on code from Olaf Seibert. 2084 */ 2085 #define PRINTLINESIZE 1000 2086 #define PRINTWIDTH 6 2087 2088 #define PRINTSOME(l, depth, fmt, a1, a2) vim_snprintf(l + depth * PRINTWIDTH, \ 2089 PRINTLINESIZE - PRINTWIDTH * depth, fmt, a1, a2) 2090 2091 static char line1[PRINTLINESIZE]; 2092 static char line2[PRINTLINESIZE]; 2093 static char line3[PRINTLINESIZE]; 2094 2095 static void 2096 spell_clear_flags(wordnode_T *node) 2097 { 2098 wordnode_T *np; 2099 2100 FOR_ALL_NODE_SIBLINGS(node, np) 2101 { 2102 np->wn_u1.index = FALSE; 2103 spell_clear_flags(np->wn_child); 2104 } 2105 } 2106 2107 static void 2108 spell_print_node(wordnode_T *node, int depth) 2109 { 2110 if (node->wn_u1.index) 2111 { 2112 // Done this node before, print the reference. 2113 PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0); 2114 PRINTSOME(line2, depth, " ", 0, 0); 2115 PRINTSOME(line3, depth, " ", 0, 0); 2116 msg(line1); 2117 msg(line2); 2118 msg(line3); 2119 } 2120 else 2121 { 2122 node->wn_u1.index = TRUE; 2123 2124 if (node->wn_byte != NUL) 2125 { 2126 if (node->wn_child != NULL) 2127 PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0); 2128 else 2129 // Cannot happen? 2130 PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0); 2131 } 2132 else 2133 PRINTSOME(line1, depth, " $ ", 0, 0); 2134 2135 PRINTSOME(line2, depth, "%d/%d ", node->wn_nr, node->wn_refs); 2136 2137 if (node->wn_sibling != NULL) 2138 PRINTSOME(line3, depth, " | ", 0, 0); 2139 else 2140 PRINTSOME(line3, depth, " ", 0, 0); 2141 2142 if (node->wn_byte == NUL) 2143 { 2144 msg(line1); 2145 msg(line2); 2146 msg(line3); 2147 } 2148 2149 // do the children 2150 if (node->wn_byte != NUL && node->wn_child != NULL) 2151 spell_print_node(node->wn_child, depth + 1); 2152 2153 // do the siblings 2154 if (node->wn_sibling != NULL) 2155 { 2156 // get rid of all parent details except | 2157 STRCPY(line1, line3); 2158 STRCPY(line2, line3); 2159 spell_print_node(node->wn_sibling, depth); 2160 } 2161 } 2162 } 2163 2164 static void 2165 spell_print_tree(wordnode_T *root) 2166 { 2167 if (root != NULL) 2168 { 2169 // Clear the "wn_u1.index" fields, used to remember what has been 2170 // done. 2171 spell_clear_flags(root); 2172 2173 // Recursively print the tree. 2174 spell_print_node(root, 0); 2175 } 2176 } 2177 #endif // SPELL_PRINTTREE 2178 2179 /* 2180 * Read the affix file "fname". 2181 * Returns an afffile_T, NULL for complete failure. 2182 */ 2183 static afffile_T * 2184 spell_read_aff(spellinfo_T *spin, char_u *fname) 2185 { 2186 FILE *fd; 2187 afffile_T *aff; 2188 char_u rline[MAXLINELEN]; 2189 char_u *line; 2190 char_u *pc = NULL; 2191 #define MAXITEMCNT 30 2192 char_u *(items[MAXITEMCNT]); 2193 int itemcnt; 2194 char_u *p; 2195 int lnum = 0; 2196 affheader_T *cur_aff = NULL; 2197 int did_postpone_prefix = FALSE; 2198 int aff_todo = 0; 2199 hashtab_T *tp; 2200 char_u *low = NULL; 2201 char_u *fol = NULL; 2202 char_u *upp = NULL; 2203 int do_rep; 2204 int do_repsal; 2205 int do_sal; 2206 int do_mapline; 2207 int found_map = FALSE; 2208 hashitem_T *hi; 2209 int l; 2210 int compminlen = 0; // COMPOUNDMIN value 2211 int compsylmax = 0; // COMPOUNDSYLMAX value 2212 int compoptions = 0; // COMP_ flags 2213 int compmax = 0; // COMPOUNDWORDMAX value 2214 char_u *compflags = NULL; // COMPOUNDFLAG and COMPOUNDRULE 2215 // concatenated 2216 char_u *midword = NULL; // MIDWORD value 2217 char_u *syllable = NULL; // SYLLABLE value 2218 char_u *sofofrom = NULL; // SOFOFROM value 2219 char_u *sofoto = NULL; // SOFOTO value 2220 2221 /* 2222 * Open the file. 2223 */ 2224 fd = mch_fopen((char *)fname, "r"); 2225 if (fd == NULL) 2226 { 2227 semsg(_(e_notopen), fname); 2228 return NULL; 2229 } 2230 2231 vim_snprintf((char *)IObuff, IOSIZE, _("Reading affix file %s..."), fname); 2232 spell_message(spin, IObuff); 2233 2234 // Only do REP lines when not done in another .aff file already. 2235 do_rep = spin->si_rep.ga_len == 0; 2236 2237 // Only do REPSAL lines when not done in another .aff file already. 2238 do_repsal = spin->si_repsal.ga_len == 0; 2239 2240 // Only do SAL lines when not done in another .aff file already. 2241 do_sal = spin->si_sal.ga_len == 0; 2242 2243 // Only do MAP lines when not done in another .aff file already. 2244 do_mapline = spin->si_map.ga_len == 0; 2245 2246 /* 2247 * Allocate and init the afffile_T structure. 2248 */ 2249 aff = (afffile_T *)getroom(spin, sizeof(afffile_T), TRUE); 2250 if (aff == NULL) 2251 { 2252 fclose(fd); 2253 return NULL; 2254 } 2255 hash_init(&aff->af_pref); 2256 hash_init(&aff->af_suff); 2257 hash_init(&aff->af_comp); 2258 2259 /* 2260 * Read all the lines in the file one by one. 2261 */ 2262 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 2263 { 2264 line_breakcheck(); 2265 ++lnum; 2266 2267 // Skip comment lines. 2268 if (*rline == '#') 2269 continue; 2270 2271 // Convert from "SET" to 'encoding' when needed. 2272 vim_free(pc); 2273 if (spin->si_conv.vc_type != CONV_NONE) 2274 { 2275 pc = string_convert(&spin->si_conv, rline, NULL); 2276 if (pc == NULL) 2277 { 2278 smsg(_("Conversion failure for word in %s line %d: %s"), 2279 fname, lnum, rline); 2280 continue; 2281 } 2282 line = pc; 2283 } 2284 else 2285 { 2286 pc = NULL; 2287 line = rline; 2288 } 2289 2290 // Split the line up in white separated items. Put a NUL after each 2291 // item. 2292 itemcnt = 0; 2293 for (p = line; ; ) 2294 { 2295 while (*p != NUL && *p <= ' ') // skip white space and CR/NL 2296 ++p; 2297 if (*p == NUL) 2298 break; 2299 if (itemcnt == MAXITEMCNT) // too many items 2300 break; 2301 items[itemcnt++] = p; 2302 // A few items have arbitrary text argument, don't split them. 2303 if (itemcnt == 2 && spell_info_item(items[0])) 2304 while (*p >= ' ' || *p == TAB) // skip until CR/NL 2305 ++p; 2306 else 2307 while (*p > ' ') // skip until white space or CR/NL 2308 ++p; 2309 if (*p == NUL) 2310 break; 2311 *p++ = NUL; 2312 } 2313 2314 // Handle non-empty lines. 2315 if (itemcnt > 0) 2316 { 2317 if (is_aff_rule(items, itemcnt, "SET", 2) && aff->af_enc == NULL) 2318 { 2319 // Setup for conversion from "ENC" to 'encoding'. 2320 aff->af_enc = enc_canonize(items[1]); 2321 if (aff->af_enc != NULL && !spin->si_ascii 2322 && convert_setup(&spin->si_conv, aff->af_enc, 2323 p_enc) == FAIL) 2324 smsg(_("Conversion in %s not supported: from %s to %s"), 2325 fname, aff->af_enc, p_enc); 2326 spin->si_conv.vc_fail = TRUE; 2327 } 2328 else if (is_aff_rule(items, itemcnt, "FLAG", 2) 2329 && aff->af_flagtype == AFT_CHAR) 2330 { 2331 if (STRCMP(items[1], "long") == 0) 2332 aff->af_flagtype = AFT_LONG; 2333 else if (STRCMP(items[1], "num") == 0) 2334 aff->af_flagtype = AFT_NUM; 2335 else if (STRCMP(items[1], "caplong") == 0) 2336 aff->af_flagtype = AFT_CAPLONG; 2337 else 2338 smsg(_("Invalid value for FLAG in %s line %d: %s"), 2339 fname, lnum, items[1]); 2340 if (aff->af_rare != 0 2341 || aff->af_keepcase != 0 2342 || aff->af_bad != 0 2343 || aff->af_needaffix != 0 2344 || aff->af_circumfix != 0 2345 || aff->af_needcomp != 0 2346 || aff->af_comproot != 0 2347 || aff->af_nosuggest != 0 2348 || compflags != NULL 2349 || aff->af_suff.ht_used > 0 2350 || aff->af_pref.ht_used > 0) 2351 smsg(_("FLAG after using flags in %s line %d: %s"), 2352 fname, lnum, items[1]); 2353 } 2354 else if (spell_info_item(items[0])) 2355 { 2356 p = (char_u *)getroom(spin, 2357 (spin->si_info == NULL ? 0 : STRLEN(spin->si_info)) 2358 + STRLEN(items[0]) 2359 + STRLEN(items[1]) + 3, FALSE); 2360 if (p != NULL) 2361 { 2362 if (spin->si_info != NULL) 2363 { 2364 STRCPY(p, spin->si_info); 2365 STRCAT(p, "\n"); 2366 } 2367 STRCAT(p, items[0]); 2368 STRCAT(p, " "); 2369 STRCAT(p, items[1]); 2370 spin->si_info = p; 2371 } 2372 } 2373 else if (is_aff_rule(items, itemcnt, "MIDWORD", 2) 2374 && midword == NULL) 2375 { 2376 midword = getroom_save(spin, items[1]); 2377 } 2378 else if (is_aff_rule(items, itemcnt, "TRY", 2)) 2379 { 2380 // ignored, we look in the tree for what chars may appear 2381 } 2382 // TODO: remove "RAR" later 2383 else if ((is_aff_rule(items, itemcnt, "RAR", 2) 2384 || is_aff_rule(items, itemcnt, "RARE", 2)) 2385 && aff->af_rare == 0) 2386 { 2387 aff->af_rare = affitem2flag(aff->af_flagtype, items[1], 2388 fname, lnum); 2389 } 2390 // TODO: remove "KEP" later 2391 else if ((is_aff_rule(items, itemcnt, "KEP", 2) 2392 || is_aff_rule(items, itemcnt, "KEEPCASE", 2)) 2393 && aff->af_keepcase == 0) 2394 { 2395 aff->af_keepcase = affitem2flag(aff->af_flagtype, items[1], 2396 fname, lnum); 2397 } 2398 else if ((is_aff_rule(items, itemcnt, "BAD", 2) 2399 || is_aff_rule(items, itemcnt, "FORBIDDENWORD", 2)) 2400 && aff->af_bad == 0) 2401 { 2402 aff->af_bad = affitem2flag(aff->af_flagtype, items[1], 2403 fname, lnum); 2404 } 2405 else if (is_aff_rule(items, itemcnt, "NEEDAFFIX", 2) 2406 && aff->af_needaffix == 0) 2407 { 2408 aff->af_needaffix = affitem2flag(aff->af_flagtype, items[1], 2409 fname, lnum); 2410 } 2411 else if (is_aff_rule(items, itemcnt, "CIRCUMFIX", 2) 2412 && aff->af_circumfix == 0) 2413 { 2414 aff->af_circumfix = affitem2flag(aff->af_flagtype, items[1], 2415 fname, lnum); 2416 } 2417 else if (is_aff_rule(items, itemcnt, "NOSUGGEST", 2) 2418 && aff->af_nosuggest == 0) 2419 { 2420 aff->af_nosuggest = affitem2flag(aff->af_flagtype, items[1], 2421 fname, lnum); 2422 } 2423 else if ((is_aff_rule(items, itemcnt, "NEEDCOMPOUND", 2) 2424 || is_aff_rule(items, itemcnt, "ONLYINCOMPOUND", 2)) 2425 && aff->af_needcomp == 0) 2426 { 2427 aff->af_needcomp = affitem2flag(aff->af_flagtype, items[1], 2428 fname, lnum); 2429 } 2430 else if (is_aff_rule(items, itemcnt, "COMPOUNDROOT", 2) 2431 && aff->af_comproot == 0) 2432 { 2433 aff->af_comproot = affitem2flag(aff->af_flagtype, items[1], 2434 fname, lnum); 2435 } 2436 else if (is_aff_rule(items, itemcnt, "COMPOUNDFORBIDFLAG", 2) 2437 && aff->af_compforbid == 0) 2438 { 2439 aff->af_compforbid = affitem2flag(aff->af_flagtype, items[1], 2440 fname, lnum); 2441 if (aff->af_pref.ht_used > 0) 2442 smsg(_("Defining COMPOUNDFORBIDFLAG after PFX item may give wrong results in %s line %d"), 2443 fname, lnum); 2444 } 2445 else if (is_aff_rule(items, itemcnt, "COMPOUNDPERMITFLAG", 2) 2446 && aff->af_comppermit == 0) 2447 { 2448 aff->af_comppermit = affitem2flag(aff->af_flagtype, items[1], 2449 fname, lnum); 2450 if (aff->af_pref.ht_used > 0) 2451 smsg(_("Defining COMPOUNDPERMITFLAG after PFX item may give wrong results in %s line %d"), 2452 fname, lnum); 2453 } 2454 else if (is_aff_rule(items, itemcnt, "COMPOUNDFLAG", 2) 2455 && compflags == NULL) 2456 { 2457 // Turn flag "c" into COMPOUNDRULE compatible string "c+", 2458 // "Na" into "Na+", "1234" into "1234+". 2459 p = getroom(spin, STRLEN(items[1]) + 2, FALSE); 2460 if (p != NULL) 2461 { 2462 STRCPY(p, items[1]); 2463 STRCAT(p, "+"); 2464 compflags = p; 2465 } 2466 } 2467 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULES", 2)) 2468 { 2469 // We don't use the count, but do check that it's a number and 2470 // not COMPOUNDRULE mistyped. 2471 if (atoi((char *)items[1]) == 0) 2472 smsg(_("Wrong COMPOUNDRULES value in %s line %d: %s"), 2473 fname, lnum, items[1]); 2474 } 2475 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULE", 2)) 2476 { 2477 // Don't use the first rule if it is a number. 2478 if (compflags != NULL || *skipdigits(items[1]) != NUL) 2479 { 2480 // Concatenate this string to previously defined ones, 2481 // using a slash to separate them. 2482 l = (int)STRLEN(items[1]) + 1; 2483 if (compflags != NULL) 2484 l += (int)STRLEN(compflags) + 1; 2485 p = getroom(spin, l, FALSE); 2486 if (p != NULL) 2487 { 2488 if (compflags != NULL) 2489 { 2490 STRCPY(p, compflags); 2491 STRCAT(p, "/"); 2492 } 2493 STRCAT(p, items[1]); 2494 compflags = p; 2495 } 2496 } 2497 } 2498 else if (is_aff_rule(items, itemcnt, "COMPOUNDWORDMAX", 2) 2499 && compmax == 0) 2500 { 2501 compmax = atoi((char *)items[1]); 2502 if (compmax == 0) 2503 smsg(_("Wrong COMPOUNDWORDMAX value in %s line %d: %s"), 2504 fname, lnum, items[1]); 2505 } 2506 else if (is_aff_rule(items, itemcnt, "COMPOUNDMIN", 2) 2507 && compminlen == 0) 2508 { 2509 compminlen = atoi((char *)items[1]); 2510 if (compminlen == 0) 2511 smsg(_("Wrong COMPOUNDMIN value in %s line %d: %s"), 2512 fname, lnum, items[1]); 2513 } 2514 else if (is_aff_rule(items, itemcnt, "COMPOUNDSYLMAX", 2) 2515 && compsylmax == 0) 2516 { 2517 compsylmax = atoi((char *)items[1]); 2518 if (compsylmax == 0) 2519 smsg(_("Wrong COMPOUNDSYLMAX value in %s line %d: %s"), 2520 fname, lnum, items[1]); 2521 } 2522 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDDUP", 1)) 2523 { 2524 compoptions |= COMP_CHECKDUP; 2525 } 2526 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDREP", 1)) 2527 { 2528 compoptions |= COMP_CHECKREP; 2529 } 2530 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDCASE", 1)) 2531 { 2532 compoptions |= COMP_CHECKCASE; 2533 } 2534 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDTRIPLE", 1)) 2535 { 2536 compoptions |= COMP_CHECKTRIPLE; 2537 } 2538 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 2)) 2539 { 2540 if (atoi((char *)items[1]) == 0) 2541 smsg(_("Wrong CHECKCOMPOUNDPATTERN value in %s line %d: %s"), 2542 fname, lnum, items[1]); 2543 } 2544 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 3)) 2545 { 2546 garray_T *gap = &spin->si_comppat; 2547 int i; 2548 2549 // Only add the couple if it isn't already there. 2550 for (i = 0; i < gap->ga_len - 1; i += 2) 2551 if (STRCMP(((char_u **)(gap->ga_data))[i], items[1]) == 0 2552 && STRCMP(((char_u **)(gap->ga_data))[i + 1], 2553 items[2]) == 0) 2554 break; 2555 if (i >= gap->ga_len && ga_grow(gap, 2) == OK) 2556 { 2557 ((char_u **)(gap->ga_data))[gap->ga_len++] 2558 = getroom_save(spin, items[1]); 2559 ((char_u **)(gap->ga_data))[gap->ga_len++] 2560 = getroom_save(spin, items[2]); 2561 } 2562 } 2563 else if (is_aff_rule(items, itemcnt, "SYLLABLE", 2) 2564 && syllable == NULL) 2565 { 2566 syllable = getroom_save(spin, items[1]); 2567 } 2568 else if (is_aff_rule(items, itemcnt, "NOBREAK", 1)) 2569 { 2570 spin->si_nobreak = TRUE; 2571 } 2572 else if (is_aff_rule(items, itemcnt, "NOSPLITSUGS", 1)) 2573 { 2574 spin->si_nosplitsugs = TRUE; 2575 } 2576 else if (is_aff_rule(items, itemcnt, "NOCOMPOUNDSUGS", 1)) 2577 { 2578 spin->si_nocompoundsugs = TRUE; 2579 } 2580 else if (is_aff_rule(items, itemcnt, "NOSUGFILE", 1)) 2581 { 2582 spin->si_nosugfile = TRUE; 2583 } 2584 else if (is_aff_rule(items, itemcnt, "PFXPOSTPONE", 1)) 2585 { 2586 aff->af_pfxpostpone = TRUE; 2587 } 2588 else if (is_aff_rule(items, itemcnt, "IGNOREEXTRA", 1)) 2589 { 2590 aff->af_ignoreextra = TRUE; 2591 } 2592 else if ((STRCMP(items[0], "PFX") == 0 2593 || STRCMP(items[0], "SFX") == 0) 2594 && aff_todo == 0 2595 && itemcnt >= 4) 2596 { 2597 int lasti = 4; 2598 char_u key[AH_KEY_LEN]; 2599 2600 if (*items[0] == 'P') 2601 tp = &aff->af_pref; 2602 else 2603 tp = &aff->af_suff; 2604 2605 // Myspell allows the same affix name to be used multiple 2606 // times. The affix files that do this have an undocumented 2607 // "S" flag on all but the last block, thus we check for that 2608 // and store it in ah_follows. 2609 vim_strncpy(key, items[1], AH_KEY_LEN - 1); 2610 hi = hash_find(tp, key); 2611 if (!HASHITEM_EMPTY(hi)) 2612 { 2613 cur_aff = HI2AH(hi); 2614 if (cur_aff->ah_combine != (*items[2] == 'Y')) 2615 smsg(_("Different combining flag in continued affix block in %s line %d: %s"), 2616 fname, lnum, items[1]); 2617 if (!cur_aff->ah_follows) 2618 smsg(_("Duplicate affix in %s line %d: %s"), 2619 fname, lnum, items[1]); 2620 } 2621 else 2622 { 2623 // New affix letter. 2624 cur_aff = (affheader_T *)getroom(spin, 2625 sizeof(affheader_T), TRUE); 2626 if (cur_aff == NULL) 2627 break; 2628 cur_aff->ah_flag = affitem2flag(aff->af_flagtype, items[1], 2629 fname, lnum); 2630 if (cur_aff->ah_flag == 0 || STRLEN(items[1]) >= AH_KEY_LEN) 2631 break; 2632 if (cur_aff->ah_flag == aff->af_bad 2633 || cur_aff->ah_flag == aff->af_rare 2634 || cur_aff->ah_flag == aff->af_keepcase 2635 || cur_aff->ah_flag == aff->af_needaffix 2636 || cur_aff->ah_flag == aff->af_circumfix 2637 || cur_aff->ah_flag == aff->af_nosuggest 2638 || cur_aff->ah_flag == aff->af_needcomp 2639 || cur_aff->ah_flag == aff->af_comproot) 2640 smsg(_("Affix also used for BAD/RARE/KEEPCASE/NEEDAFFIX/NEEDCOMPOUND/NOSUGGEST in %s line %d: %s"), 2641 fname, lnum, items[1]); 2642 STRCPY(cur_aff->ah_key, items[1]); 2643 hash_add(tp, cur_aff->ah_key); 2644 2645 cur_aff->ah_combine = (*items[2] == 'Y'); 2646 } 2647 2648 // Check for the "S" flag, which apparently means that another 2649 // block with the same affix name is following. 2650 if (itemcnt > lasti && STRCMP(items[lasti], "S") == 0) 2651 { 2652 ++lasti; 2653 cur_aff->ah_follows = TRUE; 2654 } 2655 else 2656 cur_aff->ah_follows = FALSE; 2657 2658 // Myspell allows extra text after the item, but that might 2659 // mean mistakes go unnoticed. Require a comment-starter. 2660 if (itemcnt > lasti && *items[lasti] != '#') 2661 smsg(_(e_afftrailing), fname, lnum, items[lasti]); 2662 2663 if (STRCMP(items[2], "Y") != 0 && STRCMP(items[2], "N") != 0) 2664 smsg(_("Expected Y or N in %s line %d: %s"), 2665 fname, lnum, items[2]); 2666 2667 if (*items[0] == 'P' && aff->af_pfxpostpone) 2668 { 2669 if (cur_aff->ah_newID == 0) 2670 { 2671 // Use a new number in the .spl file later, to be able 2672 // to handle multiple .aff files. 2673 check_renumber(spin); 2674 cur_aff->ah_newID = ++spin->si_newprefID; 2675 2676 // We only really use ah_newID if the prefix is 2677 // postponed. We know that only after handling all 2678 // the items. 2679 did_postpone_prefix = FALSE; 2680 } 2681 else 2682 // Did use the ID in a previous block. 2683 did_postpone_prefix = TRUE; 2684 } 2685 2686 aff_todo = atoi((char *)items[3]); 2687 } 2688 else if ((STRCMP(items[0], "PFX") == 0 2689 || STRCMP(items[0], "SFX") == 0) 2690 && aff_todo > 0 2691 && STRCMP(cur_aff->ah_key, items[1]) == 0 2692 && itemcnt >= 5) 2693 { 2694 affentry_T *aff_entry; 2695 int upper = FALSE; 2696 int lasti = 5; 2697 2698 // Myspell allows extra text after the item, but that might 2699 // mean mistakes go unnoticed. Require a comment-starter, 2700 // unless IGNOREEXTRA is used. Hunspell uses a "-" item. 2701 if (itemcnt > lasti 2702 && !aff->af_ignoreextra 2703 && *items[lasti] != '#' 2704 && (STRCMP(items[lasti], "-") != 0 2705 || itemcnt != lasti + 1)) 2706 smsg(_(e_afftrailing), fname, lnum, items[lasti]); 2707 2708 // New item for an affix letter. 2709 --aff_todo; 2710 aff_entry = (affentry_T *)getroom(spin, 2711 sizeof(affentry_T), TRUE); 2712 if (aff_entry == NULL) 2713 break; 2714 2715 if (STRCMP(items[2], "0") != 0) 2716 aff_entry->ae_chop = getroom_save(spin, items[2]); 2717 if (STRCMP(items[3], "0") != 0) 2718 { 2719 aff_entry->ae_add = getroom_save(spin, items[3]); 2720 2721 // Recognize flags on the affix: abcd/XYZ 2722 aff_entry->ae_flags = vim_strchr(aff_entry->ae_add, '/'); 2723 if (aff_entry->ae_flags != NULL) 2724 { 2725 *aff_entry->ae_flags++ = NUL; 2726 aff_process_flags(aff, aff_entry); 2727 } 2728 } 2729 2730 // Don't use an affix entry with non-ASCII characters when 2731 // "spin->si_ascii" is TRUE. 2732 if (!spin->si_ascii || !(has_non_ascii(aff_entry->ae_chop) 2733 || has_non_ascii(aff_entry->ae_add))) 2734 { 2735 aff_entry->ae_next = cur_aff->ah_first; 2736 cur_aff->ah_first = aff_entry; 2737 2738 if (STRCMP(items[4], ".") != 0) 2739 { 2740 char_u buf[MAXLINELEN]; 2741 2742 aff_entry->ae_cond = getroom_save(spin, items[4]); 2743 if (*items[0] == 'P') 2744 sprintf((char *)buf, "^%s", items[4]); 2745 else 2746 sprintf((char *)buf, "%s$", items[4]); 2747 aff_entry->ae_prog = vim_regcomp(buf, 2748 RE_MAGIC + RE_STRING + RE_STRICT); 2749 if (aff_entry->ae_prog == NULL) 2750 smsg(_("Broken condition in %s line %d: %s"), 2751 fname, lnum, items[4]); 2752 } 2753 2754 // For postponed prefixes we need an entry in si_prefcond 2755 // for the condition. Use an existing one if possible. 2756 // Can't be done for an affix with flags, ignoring 2757 // COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG. 2758 if (*items[0] == 'P' && aff->af_pfxpostpone 2759 && aff_entry->ae_flags == NULL) 2760 { 2761 // When the chop string is one lower-case letter and 2762 // the add string ends in the upper-case letter we set 2763 // the "upper" flag, clear "ae_chop" and remove the 2764 // letters from "ae_add". The condition must either 2765 // be empty or start with the same letter. 2766 if (aff_entry->ae_chop != NULL 2767 && aff_entry->ae_add != NULL 2768 && aff_entry->ae_chop[(*mb_ptr2len)( 2769 aff_entry->ae_chop)] == NUL) 2770 { 2771 int c, c_up; 2772 2773 c = PTR2CHAR(aff_entry->ae_chop); 2774 c_up = SPELL_TOUPPER(c); 2775 if (c_up != c 2776 && (aff_entry->ae_cond == NULL 2777 || PTR2CHAR(aff_entry->ae_cond) == c)) 2778 { 2779 p = aff_entry->ae_add 2780 + STRLEN(aff_entry->ae_add); 2781 MB_PTR_BACK(aff_entry->ae_add, p); 2782 if (PTR2CHAR(p) == c_up) 2783 { 2784 upper = TRUE; 2785 aff_entry->ae_chop = NULL; 2786 *p = NUL; 2787 2788 // The condition is matched with the 2789 // actual word, thus must check for the 2790 // upper-case letter. 2791 if (aff_entry->ae_cond != NULL) 2792 { 2793 char_u buf[MAXLINELEN]; 2794 2795 if (has_mbyte) 2796 { 2797 onecap_copy(items[4], buf, TRUE); 2798 aff_entry->ae_cond = getroom_save( 2799 spin, buf); 2800 } 2801 else 2802 *aff_entry->ae_cond = c_up; 2803 if (aff_entry->ae_cond != NULL) 2804 { 2805 sprintf((char *)buf, "^%s", 2806 aff_entry->ae_cond); 2807 vim_regfree(aff_entry->ae_prog); 2808 aff_entry->ae_prog = vim_regcomp( 2809 buf, RE_MAGIC + RE_STRING); 2810 } 2811 } 2812 } 2813 } 2814 } 2815 2816 if (aff_entry->ae_chop == NULL 2817 && aff_entry->ae_flags == NULL) 2818 { 2819 int idx; 2820 char_u **pp; 2821 int n; 2822 2823 // Find a previously used condition. 2824 for (idx = spin->si_prefcond.ga_len - 1; idx >= 0; 2825 --idx) 2826 { 2827 p = ((char_u **)spin->si_prefcond.ga_data)[idx]; 2828 if (str_equal(p, aff_entry->ae_cond)) 2829 break; 2830 } 2831 if (idx < 0 && ga_grow(&spin->si_prefcond, 1) == OK) 2832 { 2833 // Not found, add a new condition. 2834 idx = spin->si_prefcond.ga_len++; 2835 pp = ((char_u **)spin->si_prefcond.ga_data) 2836 + idx; 2837 if (aff_entry->ae_cond == NULL) 2838 *pp = NULL; 2839 else 2840 *pp = getroom_save(spin, 2841 aff_entry->ae_cond); 2842 } 2843 2844 // Add the prefix to the prefix tree. 2845 if (aff_entry->ae_add == NULL) 2846 p = (char_u *)""; 2847 else 2848 p = aff_entry->ae_add; 2849 2850 // PFX_FLAGS is a negative number, so that 2851 // tree_add_word() knows this is the prefix tree. 2852 n = PFX_FLAGS; 2853 if (!cur_aff->ah_combine) 2854 n |= WFP_NC; 2855 if (upper) 2856 n |= WFP_UP; 2857 if (aff_entry->ae_comppermit) 2858 n |= WFP_COMPPERMIT; 2859 if (aff_entry->ae_compforbid) 2860 n |= WFP_COMPFORBID; 2861 tree_add_word(spin, p, spin->si_prefroot, n, 2862 idx, cur_aff->ah_newID); 2863 did_postpone_prefix = TRUE; 2864 } 2865 2866 // Didn't actually use ah_newID, backup si_newprefID. 2867 if (aff_todo == 0 && !did_postpone_prefix) 2868 { 2869 --spin->si_newprefID; 2870 cur_aff->ah_newID = 0; 2871 } 2872 } 2873 } 2874 } 2875 else if (is_aff_rule(items, itemcnt, "FOL", 2) && fol == NULL) 2876 { 2877 fol = vim_strsave(items[1]); 2878 } 2879 else if (is_aff_rule(items, itemcnt, "LOW", 2) && low == NULL) 2880 { 2881 low = vim_strsave(items[1]); 2882 } 2883 else if (is_aff_rule(items, itemcnt, "UPP", 2) && upp == NULL) 2884 { 2885 upp = vim_strsave(items[1]); 2886 } 2887 else if (is_aff_rule(items, itemcnt, "REP", 2) 2888 || is_aff_rule(items, itemcnt, "REPSAL", 2)) 2889 { 2890 // Ignore REP/REPSAL count 2891 if (!isdigit(*items[1])) 2892 smsg(_("Expected REP(SAL) count in %s line %d"), 2893 fname, lnum); 2894 } 2895 else if ((STRCMP(items[0], "REP") == 0 2896 || STRCMP(items[0], "REPSAL") == 0) 2897 && itemcnt >= 3) 2898 { 2899 // REP/REPSAL item 2900 // Myspell ignores extra arguments, we require it starts with 2901 // # to detect mistakes. 2902 if (itemcnt > 3 && items[3][0] != '#') 2903 smsg(_(e_afftrailing), fname, lnum, items[3]); 2904 if (items[0][3] == 'S' ? do_repsal : do_rep) 2905 { 2906 // Replace underscore with space (can't include a space 2907 // directly). 2908 for (p = items[1]; *p != NUL; MB_PTR_ADV(p)) 2909 if (*p == '_') 2910 *p = ' '; 2911 for (p = items[2]; *p != NUL; MB_PTR_ADV(p)) 2912 if (*p == '_') 2913 *p = ' '; 2914 add_fromto(spin, items[0][3] == 'S' 2915 ? &spin->si_repsal 2916 : &spin->si_rep, items[1], items[2]); 2917 } 2918 } 2919 else if (is_aff_rule(items, itemcnt, "MAP", 2)) 2920 { 2921 // MAP item or count 2922 if (!found_map) 2923 { 2924 // First line contains the count. 2925 found_map = TRUE; 2926 if (!isdigit(*items[1])) 2927 smsg(_("Expected MAP count in %s line %d"), 2928 fname, lnum); 2929 } 2930 else if (do_mapline) 2931 { 2932 int c; 2933 2934 // Check that every character appears only once. 2935 for (p = items[1]; *p != NUL; ) 2936 { 2937 c = mb_ptr2char_adv(&p); 2938 if ((spin->si_map.ga_len > 0 2939 && vim_strchr(spin->si_map.ga_data, c) 2940 != NULL) 2941 || vim_strchr(p, c) != NULL) 2942 smsg(_("Duplicate character in MAP in %s line %d"), 2943 fname, lnum); 2944 } 2945 2946 // We simply concatenate all the MAP strings, separated by 2947 // slashes. 2948 ga_concat(&spin->si_map, items[1]); 2949 ga_append(&spin->si_map, '/'); 2950 } 2951 } 2952 // Accept "SAL from to" and "SAL from to #comment". 2953 else if (is_aff_rule(items, itemcnt, "SAL", 3)) 2954 { 2955 if (do_sal) 2956 { 2957 // SAL item (sounds-a-like) 2958 // Either one of the known keys or a from-to pair. 2959 if (STRCMP(items[1], "followup") == 0) 2960 spin->si_followup = sal_to_bool(items[2]); 2961 else if (STRCMP(items[1], "collapse_result") == 0) 2962 spin->si_collapse = sal_to_bool(items[2]); 2963 else if (STRCMP(items[1], "remove_accents") == 0) 2964 spin->si_rem_accents = sal_to_bool(items[2]); 2965 else 2966 // when "to" is "_" it means empty 2967 add_fromto(spin, &spin->si_sal, items[1], 2968 STRCMP(items[2], "_") == 0 ? (char_u *)"" 2969 : items[2]); 2970 } 2971 } 2972 else if (is_aff_rule(items, itemcnt, "SOFOFROM", 2) 2973 && sofofrom == NULL) 2974 { 2975 sofofrom = getroom_save(spin, items[1]); 2976 } 2977 else if (is_aff_rule(items, itemcnt, "SOFOTO", 2) 2978 && sofoto == NULL) 2979 { 2980 sofoto = getroom_save(spin, items[1]); 2981 } 2982 else if (STRCMP(items[0], "COMMON") == 0) 2983 { 2984 int i; 2985 2986 for (i = 1; i < itemcnt; ++i) 2987 { 2988 if (HASHITEM_EMPTY(hash_find(&spin->si_commonwords, 2989 items[i]))) 2990 { 2991 p = vim_strsave(items[i]); 2992 if (p == NULL) 2993 break; 2994 hash_add(&spin->si_commonwords, p); 2995 } 2996 } 2997 } 2998 else 2999 smsg(_("Unrecognized or duplicate item in %s line %d: %s"), 3000 fname, lnum, items[0]); 3001 } 3002 } 3003 3004 if (fol != NULL || low != NULL || upp != NULL) 3005 { 3006 if (spin->si_clear_chartab) 3007 { 3008 // Clear the char type tables, don't want to use any of the 3009 // currently used spell properties. 3010 init_spell_chartab(); 3011 spin->si_clear_chartab = FALSE; 3012 } 3013 3014 /* 3015 * Don't write a word table for an ASCII file, so that we don't check 3016 * for conflicts with a word table that matches 'encoding'. 3017 * Don't write one for utf-8 either, we use utf_*() and 3018 * mb_get_class(), the list of chars in the file will be incomplete. 3019 */ 3020 if (!spin->si_ascii && !enc_utf8) 3021 { 3022 if (fol == NULL || low == NULL || upp == NULL) 3023 smsg(_("Missing FOL/LOW/UPP line in %s"), fname); 3024 else 3025 (void)set_spell_chartab(fol, low, upp); 3026 } 3027 3028 vim_free(fol); 3029 vim_free(low); 3030 vim_free(upp); 3031 } 3032 3033 // Use compound specifications of the .aff file for the spell info. 3034 if (compmax != 0) 3035 { 3036 aff_check_number(spin->si_compmax, compmax, "COMPOUNDWORDMAX"); 3037 spin->si_compmax = compmax; 3038 } 3039 3040 if (compminlen != 0) 3041 { 3042 aff_check_number(spin->si_compminlen, compminlen, "COMPOUNDMIN"); 3043 spin->si_compminlen = compminlen; 3044 } 3045 3046 if (compsylmax != 0) 3047 { 3048 if (syllable == NULL) 3049 smsg(_("COMPOUNDSYLMAX used without SYLLABLE")); 3050 aff_check_number(spin->si_compsylmax, compsylmax, "COMPOUNDSYLMAX"); 3051 spin->si_compsylmax = compsylmax; 3052 } 3053 3054 if (compoptions != 0) 3055 { 3056 aff_check_number(spin->si_compoptions, compoptions, "COMPOUND options"); 3057 spin->si_compoptions |= compoptions; 3058 } 3059 3060 if (compflags != NULL) 3061 process_compflags(spin, aff, compflags); 3062 3063 // Check that we didn't use too many renumbered flags. 3064 if (spin->si_newcompID < spin->si_newprefID) 3065 { 3066 if (spin->si_newcompID == 127 || spin->si_newcompID == 255) 3067 msg(_("Too many postponed prefixes")); 3068 else if (spin->si_newprefID == 0 || spin->si_newprefID == 127) 3069 msg(_("Too many compound flags")); 3070 else 3071 msg(_("Too many postponed prefixes and/or compound flags")); 3072 } 3073 3074 if (syllable != NULL) 3075 { 3076 aff_check_string(spin->si_syllable, syllable, "SYLLABLE"); 3077 spin->si_syllable = syllable; 3078 } 3079 3080 if (sofofrom != NULL || sofoto != NULL) 3081 { 3082 if (sofofrom == NULL || sofoto == NULL) 3083 smsg(_("Missing SOFO%s line in %s"), 3084 sofofrom == NULL ? "FROM" : "TO", fname); 3085 else if (spin->si_sal.ga_len > 0) 3086 smsg(_("Both SAL and SOFO lines in %s"), fname); 3087 else 3088 { 3089 aff_check_string(spin->si_sofofr, sofofrom, "SOFOFROM"); 3090 aff_check_string(spin->si_sofoto, sofoto, "SOFOTO"); 3091 spin->si_sofofr = sofofrom; 3092 spin->si_sofoto = sofoto; 3093 } 3094 } 3095 3096 if (midword != NULL) 3097 { 3098 aff_check_string(spin->si_midword, midword, "MIDWORD"); 3099 spin->si_midword = midword; 3100 } 3101 3102 vim_free(pc); 3103 fclose(fd); 3104 return aff; 3105 } 3106 3107 /* 3108 * Return TRUE when items[0] equals "rulename", there are "mincount" items or 3109 * a comment is following after item "mincount". 3110 */ 3111 static int 3112 is_aff_rule( 3113 char_u **items, 3114 int itemcnt, 3115 char *rulename, 3116 int mincount) 3117 { 3118 return (STRCMP(items[0], rulename) == 0 3119 && (itemcnt == mincount 3120 || (itemcnt > mincount && items[mincount][0] == '#'))); 3121 } 3122 3123 /* 3124 * For affix "entry" move COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG from 3125 * ae_flags to ae_comppermit and ae_compforbid. 3126 */ 3127 static void 3128 aff_process_flags(afffile_T *affile, affentry_T *entry) 3129 { 3130 char_u *p; 3131 char_u *prevp; 3132 unsigned flag; 3133 3134 if (entry->ae_flags != NULL 3135 && (affile->af_compforbid != 0 || affile->af_comppermit != 0)) 3136 { 3137 for (p = entry->ae_flags; *p != NUL; ) 3138 { 3139 prevp = p; 3140 flag = get_affitem(affile->af_flagtype, &p); 3141 if (flag == affile->af_comppermit || flag == affile->af_compforbid) 3142 { 3143 STRMOVE(prevp, p); 3144 p = prevp; 3145 if (flag == affile->af_comppermit) 3146 entry->ae_comppermit = TRUE; 3147 else 3148 entry->ae_compforbid = TRUE; 3149 } 3150 if (affile->af_flagtype == AFT_NUM && *p == ',') 3151 ++p; 3152 } 3153 if (*entry->ae_flags == NUL) 3154 entry->ae_flags = NULL; // nothing left 3155 } 3156 } 3157 3158 /* 3159 * Return TRUE if "s" is the name of an info item in the affix file. 3160 */ 3161 static int 3162 spell_info_item(char_u *s) 3163 { 3164 return STRCMP(s, "NAME") == 0 3165 || STRCMP(s, "HOME") == 0 3166 || STRCMP(s, "VERSION") == 0 3167 || STRCMP(s, "AUTHOR") == 0 3168 || STRCMP(s, "EMAIL") == 0 3169 || STRCMP(s, "COPYRIGHT") == 0; 3170 } 3171 3172 /* 3173 * Turn an affix flag name into a number, according to the FLAG type. 3174 * returns zero for failure. 3175 */ 3176 static unsigned 3177 affitem2flag( 3178 int flagtype, 3179 char_u *item, 3180 char_u *fname, 3181 int lnum) 3182 { 3183 unsigned res; 3184 char_u *p = item; 3185 3186 res = get_affitem(flagtype, &p); 3187 if (res == 0) 3188 { 3189 if (flagtype == AFT_NUM) 3190 smsg(_("Flag is not a number in %s line %d: %s"), 3191 fname, lnum, item); 3192 else 3193 smsg(_("Illegal flag in %s line %d: %s"), 3194 fname, lnum, item); 3195 } 3196 if (*p != NUL) 3197 { 3198 smsg(_(e_affname), fname, lnum, item); 3199 return 0; 3200 } 3201 3202 return res; 3203 } 3204 3205 /* 3206 * Get one affix name from "*pp" and advance the pointer. 3207 * Returns ZERO_FLAG for "0". 3208 * Returns zero for an error, still advances the pointer then. 3209 */ 3210 static unsigned 3211 get_affitem(int flagtype, char_u **pp) 3212 { 3213 int res; 3214 3215 if (flagtype == AFT_NUM) 3216 { 3217 if (!VIM_ISDIGIT(**pp)) 3218 { 3219 ++*pp; // always advance, avoid getting stuck 3220 return 0; 3221 } 3222 res = getdigits(pp); 3223 if (res == 0) 3224 res = ZERO_FLAG; 3225 } 3226 else 3227 { 3228 res = mb_ptr2char_adv(pp); 3229 if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG 3230 && res >= 'A' && res <= 'Z')) 3231 { 3232 if (**pp == NUL) 3233 return 0; 3234 res = mb_ptr2char_adv(pp) + (res << 16); 3235 } 3236 } 3237 return res; 3238 } 3239 3240 /* 3241 * Process the "compflags" string used in an affix file and append it to 3242 * spin->si_compflags. 3243 * The processing involves changing the affix names to ID numbers, so that 3244 * they fit in one byte. 3245 */ 3246 static void 3247 process_compflags( 3248 spellinfo_T *spin, 3249 afffile_T *aff, 3250 char_u *compflags) 3251 { 3252 char_u *p; 3253 char_u *prevp; 3254 unsigned flag; 3255 compitem_T *ci; 3256 int id; 3257 int len; 3258 char_u *tp; 3259 char_u key[AH_KEY_LEN]; 3260 hashitem_T *hi; 3261 3262 // Make room for the old and the new compflags, concatenated with a / in 3263 // between. Processing it makes it shorter, but we don't know by how 3264 // much, thus allocate the maximum. 3265 len = (int)STRLEN(compflags) + 1; 3266 if (spin->si_compflags != NULL) 3267 len += (int)STRLEN(spin->si_compflags) + 1; 3268 p = getroom(spin, len, FALSE); 3269 if (p == NULL) 3270 return; 3271 if (spin->si_compflags != NULL) 3272 { 3273 STRCPY(p, spin->si_compflags); 3274 STRCAT(p, "/"); 3275 } 3276 spin->si_compflags = p; 3277 tp = p + STRLEN(p); 3278 3279 for (p = compflags; *p != NUL; ) 3280 { 3281 if (vim_strchr((char_u *)"/?*+[]", *p) != NULL) 3282 // Copy non-flag characters directly. 3283 *tp++ = *p++; 3284 else 3285 { 3286 // First get the flag number, also checks validity. 3287 prevp = p; 3288 flag = get_affitem(aff->af_flagtype, &p); 3289 if (flag != 0) 3290 { 3291 // Find the flag in the hashtable. If it was used before, use 3292 // the existing ID. Otherwise add a new entry. 3293 vim_strncpy(key, prevp, p - prevp); 3294 hi = hash_find(&aff->af_comp, key); 3295 if (!HASHITEM_EMPTY(hi)) 3296 id = HI2CI(hi)->ci_newID; 3297 else 3298 { 3299 ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE); 3300 if (ci == NULL) 3301 break; 3302 STRCPY(ci->ci_key, key); 3303 ci->ci_flag = flag; 3304 // Avoid using a flag ID that has a special meaning in a 3305 // regexp (also inside []). 3306 do 3307 { 3308 check_renumber(spin); 3309 id = spin->si_newcompID--; 3310 } while (vim_strchr((char_u *)"/?*+[]\\-^", id) != NULL); 3311 ci->ci_newID = id; 3312 hash_add(&aff->af_comp, ci->ci_key); 3313 } 3314 *tp++ = id; 3315 } 3316 if (aff->af_flagtype == AFT_NUM && *p == ',') 3317 ++p; 3318 } 3319 } 3320 3321 *tp = NUL; 3322 } 3323 3324 /* 3325 * Check that the new IDs for postponed affixes and compounding don't overrun 3326 * each other. We have almost 255 available, but start at 0-127 to avoid 3327 * using two bytes for utf-8. When the 0-127 range is used up go to 128-255. 3328 * When that is used up an error message is given. 3329 */ 3330 static void 3331 check_renumber(spellinfo_T *spin) 3332 { 3333 if (spin->si_newprefID == spin->si_newcompID && spin->si_newcompID < 128) 3334 { 3335 spin->si_newprefID = 127; 3336 spin->si_newcompID = 255; 3337 } 3338 } 3339 3340 /* 3341 * Return TRUE if flag "flag" appears in affix list "afflist". 3342 */ 3343 static int 3344 flag_in_afflist(int flagtype, char_u *afflist, unsigned flag) 3345 { 3346 char_u *p; 3347 unsigned n; 3348 3349 switch (flagtype) 3350 { 3351 case AFT_CHAR: 3352 return vim_strchr(afflist, flag) != NULL; 3353 3354 case AFT_CAPLONG: 3355 case AFT_LONG: 3356 for (p = afflist; *p != NUL; ) 3357 { 3358 n = mb_ptr2char_adv(&p); 3359 if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z')) 3360 && *p != NUL) 3361 n = mb_ptr2char_adv(&p) + (n << 16); 3362 if (n == flag) 3363 return TRUE; 3364 } 3365 break; 3366 3367 case AFT_NUM: 3368 for (p = afflist; *p != NUL; ) 3369 { 3370 n = getdigits(&p); 3371 if (n == 0) 3372 n = ZERO_FLAG; 3373 if (n == flag) 3374 return TRUE; 3375 if (*p != NUL) // skip over comma 3376 ++p; 3377 } 3378 break; 3379 } 3380 return FALSE; 3381 } 3382 3383 /* 3384 * Give a warning when "spinval" and "affval" numbers are set and not the same. 3385 */ 3386 static void 3387 aff_check_number(int spinval, int affval, char *name) 3388 { 3389 if (spinval != 0 && spinval != affval) 3390 smsg(_("%s value differs from what is used in another .aff file"), name); 3391 } 3392 3393 /* 3394 * Give a warning when "spinval" and "affval" strings are set and not the same. 3395 */ 3396 static void 3397 aff_check_string(char_u *spinval, char_u *affval, char *name) 3398 { 3399 if (spinval != NULL && STRCMP(spinval, affval) != 0) 3400 smsg(_("%s value differs from what is used in another .aff file"), name); 3401 } 3402 3403 /* 3404 * Return TRUE if strings "s1" and "s2" are equal. Also consider both being 3405 * NULL as equal. 3406 */ 3407 static int 3408 str_equal(char_u *s1, char_u *s2) 3409 { 3410 if (s1 == NULL || s2 == NULL) 3411 return s1 == s2; 3412 return STRCMP(s1, s2) == 0; 3413 } 3414 3415 /* 3416 * Add a from-to item to "gap". Used for REP and SAL items. 3417 * They are stored case-folded. 3418 */ 3419 static void 3420 add_fromto( 3421 spellinfo_T *spin, 3422 garray_T *gap, 3423 char_u *from, 3424 char_u *to) 3425 { 3426 fromto_T *ftp; 3427 char_u word[MAXWLEN]; 3428 3429 if (ga_grow(gap, 1) == OK) 3430 { 3431 ftp = ((fromto_T *)gap->ga_data) + gap->ga_len; 3432 (void)spell_casefold(from, (int)STRLEN(from), word, MAXWLEN); 3433 ftp->ft_from = getroom_save(spin, word); 3434 (void)spell_casefold(to, (int)STRLEN(to), word, MAXWLEN); 3435 ftp->ft_to = getroom_save(spin, word); 3436 ++gap->ga_len; 3437 } 3438 } 3439 3440 /* 3441 * Convert a boolean argument in a SAL line to TRUE or FALSE; 3442 */ 3443 static int 3444 sal_to_bool(char_u *s) 3445 { 3446 return STRCMP(s, "1") == 0 || STRCMP(s, "true") == 0; 3447 } 3448 3449 /* 3450 * Free the structure filled by spell_read_aff(). 3451 */ 3452 static void 3453 spell_free_aff(afffile_T *aff) 3454 { 3455 hashtab_T *ht; 3456 hashitem_T *hi; 3457 int todo; 3458 affheader_T *ah; 3459 affentry_T *ae; 3460 3461 vim_free(aff->af_enc); 3462 3463 // All this trouble to free the "ae_prog" items... 3464 for (ht = &aff->af_pref; ; ht = &aff->af_suff) 3465 { 3466 todo = (int)ht->ht_used; 3467 for (hi = ht->ht_array; todo > 0; ++hi) 3468 { 3469 if (!HASHITEM_EMPTY(hi)) 3470 { 3471 --todo; 3472 ah = HI2AH(hi); 3473 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 3474 vim_regfree(ae->ae_prog); 3475 } 3476 } 3477 if (ht == &aff->af_suff) 3478 break; 3479 } 3480 3481 hash_clear(&aff->af_pref); 3482 hash_clear(&aff->af_suff); 3483 hash_clear(&aff->af_comp); 3484 } 3485 3486 /* 3487 * Read dictionary file "fname". 3488 * Returns OK or FAIL; 3489 */ 3490 static int 3491 spell_read_dic(spellinfo_T *spin, char_u *fname, afffile_T *affile) 3492 { 3493 hashtab_T ht; 3494 char_u line[MAXLINELEN]; 3495 char_u *p; 3496 char_u *afflist; 3497 char_u store_afflist[MAXWLEN]; 3498 int pfxlen; 3499 int need_affix; 3500 char_u *dw; 3501 char_u *pc; 3502 char_u *w; 3503 int l; 3504 hash_T hash; 3505 hashitem_T *hi; 3506 FILE *fd; 3507 int lnum = 1; 3508 int non_ascii = 0; 3509 int retval = OK; 3510 char_u message[MAXLINELEN + MAXWLEN]; 3511 int flags; 3512 int duplicate = 0; 3513 time_T last_msg_time = 0; 3514 3515 /* 3516 * Open the file. 3517 */ 3518 fd = mch_fopen((char *)fname, "r"); 3519 if (fd == NULL) 3520 { 3521 semsg(_(e_notopen), fname); 3522 return FAIL; 3523 } 3524 3525 // The hashtable is only used to detect duplicated words. 3526 hash_init(&ht); 3527 3528 vim_snprintf((char *)IObuff, IOSIZE, 3529 _("Reading dictionary file %s..."), fname); 3530 spell_message(spin, IObuff); 3531 3532 // start with a message for the first line 3533 spin->si_msg_count = 999999; 3534 3535 // Read and ignore the first line: word count. 3536 if (vim_fgets(line, MAXLINELEN, fd) || !vim_isdigit(*skipwhite(line))) 3537 semsg(_("E760: No word count in %s"), fname); 3538 3539 /* 3540 * Read all the lines in the file one by one. 3541 * The words are converted to 'encoding' here, before being added to 3542 * the hashtable. 3543 */ 3544 while (!vim_fgets(line, MAXLINELEN, fd) && !got_int) 3545 { 3546 line_breakcheck(); 3547 ++lnum; 3548 if (line[0] == '#' || line[0] == '/') 3549 continue; // comment line 3550 3551 // Remove CR, LF and white space from the end. White space halfway 3552 // the word is kept to allow e.g., "et al.". 3553 l = (int)STRLEN(line); 3554 while (l > 0 && line[l - 1] <= ' ') 3555 --l; 3556 if (l == 0) 3557 continue; // empty line 3558 line[l] = NUL; 3559 3560 // Convert from "SET" to 'encoding' when needed. 3561 if (spin->si_conv.vc_type != CONV_NONE) 3562 { 3563 pc = string_convert(&spin->si_conv, line, NULL); 3564 if (pc == NULL) 3565 { 3566 smsg(_("Conversion failure for word in %s line %d: %s"), 3567 fname, lnum, line); 3568 continue; 3569 } 3570 w = pc; 3571 } 3572 else 3573 { 3574 pc = NULL; 3575 w = line; 3576 } 3577 3578 // Truncate the word at the "/", set "afflist" to what follows. 3579 // Replace "\/" by "/" and "\\" by "\". 3580 afflist = NULL; 3581 for (p = w; *p != NUL; MB_PTR_ADV(p)) 3582 { 3583 if (*p == '\\' && (p[1] == '\\' || p[1] == '/')) 3584 STRMOVE(p, p + 1); 3585 else if (*p == '/') 3586 { 3587 *p = NUL; 3588 afflist = p + 1; 3589 break; 3590 } 3591 } 3592 3593 // Skip non-ASCII words when "spin->si_ascii" is TRUE. 3594 if (spin->si_ascii && has_non_ascii(w)) 3595 { 3596 ++non_ascii; 3597 vim_free(pc); 3598 continue; 3599 } 3600 3601 // This takes time, print a message every 10000 words, but not more 3602 // often than once per second. 3603 if (spin->si_verbose && spin->si_msg_count > 10000) 3604 { 3605 spin->si_msg_count = 0; 3606 if (vim_time() > last_msg_time) 3607 { 3608 last_msg_time = vim_time(); 3609 vim_snprintf((char *)message, sizeof(message), 3610 _("line %6d, word %6ld - %s"), 3611 lnum, spin->si_foldwcount + spin->si_keepwcount, w); 3612 msg_start(); 3613 msg_outtrans_long_attr(message, 0); 3614 msg_clr_eos(); 3615 msg_didout = FALSE; 3616 msg_col = 0; 3617 out_flush(); 3618 } 3619 } 3620 3621 // Store the word in the hashtable to be able to find duplicates. 3622 dw = (char_u *)getroom_save(spin, w); 3623 if (dw == NULL) 3624 { 3625 retval = FAIL; 3626 vim_free(pc); 3627 break; 3628 } 3629 3630 hash = hash_hash(dw); 3631 hi = hash_lookup(&ht, dw, hash); 3632 if (!HASHITEM_EMPTY(hi)) 3633 { 3634 if (p_verbose > 0) 3635 smsg(_("Duplicate word in %s line %d: %s"), 3636 fname, lnum, dw); 3637 else if (duplicate == 0) 3638 smsg(_("First duplicate word in %s line %d: %s"), 3639 fname, lnum, dw); 3640 ++duplicate; 3641 } 3642 else 3643 hash_add_item(&ht, hi, dw, hash); 3644 3645 flags = 0; 3646 store_afflist[0] = NUL; 3647 pfxlen = 0; 3648 need_affix = FALSE; 3649 if (afflist != NULL) 3650 { 3651 // Extract flags from the affix list. 3652 flags |= get_affix_flags(affile, afflist); 3653 3654 if (affile->af_needaffix != 0 && flag_in_afflist( 3655 affile->af_flagtype, afflist, affile->af_needaffix)) 3656 need_affix = TRUE; 3657 3658 if (affile->af_pfxpostpone) 3659 // Need to store the list of prefix IDs with the word. 3660 pfxlen = get_pfxlist(affile, afflist, store_afflist); 3661 3662 if (spin->si_compflags != NULL) 3663 // Need to store the list of compound flags with the word. 3664 // Concatenate them to the list of prefix IDs. 3665 get_compflags(affile, afflist, store_afflist + pfxlen); 3666 } 3667 3668 // Add the word to the word tree(s). 3669 if (store_word(spin, dw, flags, spin->si_region, 3670 store_afflist, need_affix) == FAIL) 3671 retval = FAIL; 3672 3673 if (afflist != NULL) 3674 { 3675 // Find all matching suffixes and add the resulting words. 3676 // Additionally do matching prefixes that combine. 3677 if (store_aff_word(spin, dw, afflist, affile, 3678 &affile->af_suff, &affile->af_pref, 3679 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 3680 retval = FAIL; 3681 3682 // Find all matching prefixes and add the resulting words. 3683 if (store_aff_word(spin, dw, afflist, affile, 3684 &affile->af_pref, NULL, 3685 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 3686 retval = FAIL; 3687 } 3688 3689 vim_free(pc); 3690 } 3691 3692 if (duplicate > 0) 3693 smsg(_("%d duplicate word(s) in %s"), duplicate, fname); 3694 if (spin->si_ascii && non_ascii > 0) 3695 smsg(_("Ignored %d word(s) with non-ASCII characters in %s"), 3696 non_ascii, fname); 3697 hash_clear(&ht); 3698 3699 fclose(fd); 3700 return retval; 3701 } 3702 3703 /* 3704 * Check for affix flags in "afflist" that are turned into word flags. 3705 * Return WF_ flags. 3706 */ 3707 static int 3708 get_affix_flags(afffile_T *affile, char_u *afflist) 3709 { 3710 int flags = 0; 3711 3712 if (affile->af_keepcase != 0 && flag_in_afflist( 3713 affile->af_flagtype, afflist, affile->af_keepcase)) 3714 flags |= WF_KEEPCAP | WF_FIXCAP; 3715 if (affile->af_rare != 0 && flag_in_afflist( 3716 affile->af_flagtype, afflist, affile->af_rare)) 3717 flags |= WF_RARE; 3718 if (affile->af_bad != 0 && flag_in_afflist( 3719 affile->af_flagtype, afflist, affile->af_bad)) 3720 flags |= WF_BANNED; 3721 if (affile->af_needcomp != 0 && flag_in_afflist( 3722 affile->af_flagtype, afflist, affile->af_needcomp)) 3723 flags |= WF_NEEDCOMP; 3724 if (affile->af_comproot != 0 && flag_in_afflist( 3725 affile->af_flagtype, afflist, affile->af_comproot)) 3726 flags |= WF_COMPROOT; 3727 if (affile->af_nosuggest != 0 && flag_in_afflist( 3728 affile->af_flagtype, afflist, affile->af_nosuggest)) 3729 flags |= WF_NOSUGGEST; 3730 return flags; 3731 } 3732 3733 /* 3734 * Get the list of prefix IDs from the affix list "afflist". 3735 * Used for PFXPOSTPONE. 3736 * Put the resulting flags in "store_afflist[MAXWLEN]" with a terminating NUL 3737 * and return the number of affixes. 3738 */ 3739 static int 3740 get_pfxlist( 3741 afffile_T *affile, 3742 char_u *afflist, 3743 char_u *store_afflist) 3744 { 3745 char_u *p; 3746 char_u *prevp; 3747 int cnt = 0; 3748 int id; 3749 char_u key[AH_KEY_LEN]; 3750 hashitem_T *hi; 3751 3752 for (p = afflist; *p != NUL; ) 3753 { 3754 prevp = p; 3755 if (get_affitem(affile->af_flagtype, &p) != 0) 3756 { 3757 // A flag is a postponed prefix flag if it appears in "af_pref" 3758 // and its ID is not zero. 3759 vim_strncpy(key, prevp, p - prevp); 3760 hi = hash_find(&affile->af_pref, key); 3761 if (!HASHITEM_EMPTY(hi)) 3762 { 3763 id = HI2AH(hi)->ah_newID; 3764 if (id != 0) 3765 store_afflist[cnt++] = id; 3766 } 3767 } 3768 if (affile->af_flagtype == AFT_NUM && *p == ',') 3769 ++p; 3770 } 3771 3772 store_afflist[cnt] = NUL; 3773 return cnt; 3774 } 3775 3776 /* 3777 * Get the list of compound IDs from the affix list "afflist" that are used 3778 * for compound words. 3779 * Puts the flags in "store_afflist[]". 3780 */ 3781 static void 3782 get_compflags( 3783 afffile_T *affile, 3784 char_u *afflist, 3785 char_u *store_afflist) 3786 { 3787 char_u *p; 3788 char_u *prevp; 3789 int cnt = 0; 3790 char_u key[AH_KEY_LEN]; 3791 hashitem_T *hi; 3792 3793 for (p = afflist; *p != NUL; ) 3794 { 3795 prevp = p; 3796 if (get_affitem(affile->af_flagtype, &p) != 0) 3797 { 3798 // A flag is a compound flag if it appears in "af_comp". 3799 vim_strncpy(key, prevp, p - prevp); 3800 hi = hash_find(&affile->af_comp, key); 3801 if (!HASHITEM_EMPTY(hi)) 3802 store_afflist[cnt++] = HI2CI(hi)->ci_newID; 3803 } 3804 if (affile->af_flagtype == AFT_NUM && *p == ',') 3805 ++p; 3806 } 3807 3808 store_afflist[cnt] = NUL; 3809 } 3810 3811 /* 3812 * Apply affixes to a word and store the resulting words. 3813 * "ht" is the hashtable with affentry_T that need to be applied, either 3814 * prefixes or suffixes. 3815 * "xht", when not NULL, is the prefix hashtable, to be used additionally on 3816 * the resulting words for combining affixes. 3817 * 3818 * Returns FAIL when out of memory. 3819 */ 3820 static int 3821 store_aff_word( 3822 spellinfo_T *spin, // spell info 3823 char_u *word, // basic word start 3824 char_u *afflist, // list of names of supported affixes 3825 afffile_T *affile, 3826 hashtab_T *ht, 3827 hashtab_T *xht, 3828 int condit, // CONDIT_SUF et al. 3829 int flags, // flags for the word 3830 char_u *pfxlist, // list of prefix IDs 3831 int pfxlen) // nr of flags in "pfxlist" for prefixes, rest 3832 // is compound flags 3833 { 3834 int todo; 3835 hashitem_T *hi; 3836 affheader_T *ah; 3837 affentry_T *ae; 3838 char_u newword[MAXWLEN]; 3839 int retval = OK; 3840 int i, j; 3841 char_u *p; 3842 int use_flags; 3843 char_u *use_pfxlist; 3844 int use_pfxlen; 3845 int need_affix; 3846 char_u store_afflist[MAXWLEN]; 3847 char_u pfx_pfxlist[MAXWLEN]; 3848 size_t wordlen = STRLEN(word); 3849 int use_condit; 3850 3851 todo = (int)ht->ht_used; 3852 for (hi = ht->ht_array; todo > 0 && retval == OK; ++hi) 3853 { 3854 if (!HASHITEM_EMPTY(hi)) 3855 { 3856 --todo; 3857 ah = HI2AH(hi); 3858 3859 // Check that the affix combines, if required, and that the word 3860 // supports this affix. 3861 if (((condit & CONDIT_COMB) == 0 || ah->ah_combine) 3862 && flag_in_afflist(affile->af_flagtype, afflist, 3863 ah->ah_flag)) 3864 { 3865 // Loop over all affix entries with this name. 3866 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 3867 { 3868 // Check the condition. It's not logical to match case 3869 // here, but it is required for compatibility with 3870 // Myspell. 3871 // Another requirement from Myspell is that the chop 3872 // string is shorter than the word itself. 3873 // For prefixes, when "PFXPOSTPONE" was used, only do 3874 // prefixes with a chop string and/or flags. 3875 // When a previously added affix had CIRCUMFIX this one 3876 // must have it too, if it had not then this one must not 3877 // have one either. 3878 if ((xht != NULL || !affile->af_pfxpostpone 3879 || ae->ae_chop != NULL 3880 || ae->ae_flags != NULL) 3881 && (ae->ae_chop == NULL 3882 || STRLEN(ae->ae_chop) < wordlen) 3883 && (ae->ae_prog == NULL 3884 || vim_regexec_prog(&ae->ae_prog, FALSE, 3885 word, (colnr_T)0)) 3886 && (((condit & CONDIT_CFIX) == 0) 3887 == ((condit & CONDIT_AFF) == 0 3888 || ae->ae_flags == NULL 3889 || !flag_in_afflist(affile->af_flagtype, 3890 ae->ae_flags, affile->af_circumfix)))) 3891 { 3892 // Match. Remove the chop and add the affix. 3893 if (xht == NULL) 3894 { 3895 // prefix: chop/add at the start of the word 3896 if (ae->ae_add == NULL) 3897 *newword = NUL; 3898 else 3899 vim_strncpy(newword, ae->ae_add, MAXWLEN - 1); 3900 p = word; 3901 if (ae->ae_chop != NULL) 3902 { 3903 // Skip chop string. 3904 if (has_mbyte) 3905 { 3906 i = mb_charlen(ae->ae_chop); 3907 for ( ; i > 0; --i) 3908 MB_PTR_ADV(p); 3909 } 3910 else 3911 p += STRLEN(ae->ae_chop); 3912 } 3913 STRCAT(newword, p); 3914 } 3915 else 3916 { 3917 // suffix: chop/add at the end of the word 3918 vim_strncpy(newword, word, MAXWLEN - 1); 3919 if (ae->ae_chop != NULL) 3920 { 3921 // Remove chop string. 3922 p = newword + STRLEN(newword); 3923 i = (int)MB_CHARLEN(ae->ae_chop); 3924 for ( ; i > 0; --i) 3925 MB_PTR_BACK(newword, p); 3926 *p = NUL; 3927 } 3928 if (ae->ae_add != NULL) 3929 STRCAT(newword, ae->ae_add); 3930 } 3931 3932 use_flags = flags; 3933 use_pfxlist = pfxlist; 3934 use_pfxlen = pfxlen; 3935 need_affix = FALSE; 3936 use_condit = condit | CONDIT_COMB | CONDIT_AFF; 3937 if (ae->ae_flags != NULL) 3938 { 3939 // Extract flags from the affix list. 3940 use_flags |= get_affix_flags(affile, ae->ae_flags); 3941 3942 if (affile->af_needaffix != 0 && flag_in_afflist( 3943 affile->af_flagtype, ae->ae_flags, 3944 affile->af_needaffix)) 3945 need_affix = TRUE; 3946 3947 // When there is a CIRCUMFIX flag the other affix 3948 // must also have it and we don't add the word 3949 // with one affix. 3950 if (affile->af_circumfix != 0 && flag_in_afflist( 3951 affile->af_flagtype, ae->ae_flags, 3952 affile->af_circumfix)) 3953 { 3954 use_condit |= CONDIT_CFIX; 3955 if ((condit & CONDIT_CFIX) == 0) 3956 need_affix = TRUE; 3957 } 3958 3959 if (affile->af_pfxpostpone 3960 || spin->si_compflags != NULL) 3961 { 3962 if (affile->af_pfxpostpone) 3963 // Get prefix IDS from the affix list. 3964 use_pfxlen = get_pfxlist(affile, 3965 ae->ae_flags, store_afflist); 3966 else 3967 use_pfxlen = 0; 3968 use_pfxlist = store_afflist; 3969 3970 // Combine the prefix IDs. Avoid adding the 3971 // same ID twice. 3972 for (i = 0; i < pfxlen; ++i) 3973 { 3974 for (j = 0; j < use_pfxlen; ++j) 3975 if (pfxlist[i] == use_pfxlist[j]) 3976 break; 3977 if (j == use_pfxlen) 3978 use_pfxlist[use_pfxlen++] = pfxlist[i]; 3979 } 3980 3981 if (spin->si_compflags != NULL) 3982 // Get compound IDS from the affix list. 3983 get_compflags(affile, ae->ae_flags, 3984 use_pfxlist + use_pfxlen); 3985 3986 // Combine the list of compound flags. 3987 // Concatenate them to the prefix IDs list. 3988 // Avoid adding the same ID twice. 3989 for (i = pfxlen; pfxlist[i] != NUL; ++i) 3990 { 3991 for (j = use_pfxlen; 3992 use_pfxlist[j] != NUL; ++j) 3993 if (pfxlist[i] == use_pfxlist[j]) 3994 break; 3995 if (use_pfxlist[j] == NUL) 3996 { 3997 use_pfxlist[j++] = pfxlist[i]; 3998 use_pfxlist[j] = NUL; 3999 } 4000 } 4001 } 4002 } 4003 4004 // Obey a "COMPOUNDFORBIDFLAG" of the affix: don't 4005 // use the compound flags. 4006 if (use_pfxlist != NULL && ae->ae_compforbid) 4007 { 4008 vim_strncpy(pfx_pfxlist, use_pfxlist, use_pfxlen); 4009 use_pfxlist = pfx_pfxlist; 4010 } 4011 4012 // When there are postponed prefixes... 4013 if (spin->si_prefroot != NULL 4014 && spin->si_prefroot->wn_sibling != NULL) 4015 { 4016 // ... add a flag to indicate an affix was used. 4017 use_flags |= WF_HAS_AFF; 4018 4019 // ... don't use a prefix list if combining 4020 // affixes is not allowed. But do use the 4021 // compound flags after them. 4022 if (!ah->ah_combine && use_pfxlist != NULL) 4023 use_pfxlist += use_pfxlen; 4024 } 4025 4026 // When compounding is supported and there is no 4027 // "COMPOUNDPERMITFLAG" then forbid compounding on the 4028 // side where the affix is applied. 4029 if (spin->si_compflags != NULL && !ae->ae_comppermit) 4030 { 4031 if (xht != NULL) 4032 use_flags |= WF_NOCOMPAFT; 4033 else 4034 use_flags |= WF_NOCOMPBEF; 4035 } 4036 4037 // Store the modified word. 4038 if (store_word(spin, newword, use_flags, 4039 spin->si_region, use_pfxlist, 4040 need_affix) == FAIL) 4041 retval = FAIL; 4042 4043 // When added a prefix or a first suffix and the affix 4044 // has flags may add a(nother) suffix. RECURSIVE! 4045 if ((condit & CONDIT_SUF) && ae->ae_flags != NULL) 4046 if (store_aff_word(spin, newword, ae->ae_flags, 4047 affile, &affile->af_suff, xht, 4048 use_condit & (xht == NULL 4049 ? ~0 : ~CONDIT_SUF), 4050 use_flags, use_pfxlist, pfxlen) == FAIL) 4051 retval = FAIL; 4052 4053 // When added a suffix and combining is allowed also 4054 // try adding a prefix additionally. Both for the 4055 // word flags and for the affix flags. RECURSIVE! 4056 if (xht != NULL && ah->ah_combine) 4057 { 4058 if (store_aff_word(spin, newword, 4059 afflist, affile, 4060 xht, NULL, use_condit, 4061 use_flags, use_pfxlist, 4062 pfxlen) == FAIL 4063 || (ae->ae_flags != NULL 4064 && store_aff_word(spin, newword, 4065 ae->ae_flags, affile, 4066 xht, NULL, use_condit, 4067 use_flags, use_pfxlist, 4068 pfxlen) == FAIL)) 4069 retval = FAIL; 4070 } 4071 } 4072 } 4073 } 4074 } 4075 } 4076 4077 return retval; 4078 } 4079 4080 /* 4081 * Read a file with a list of words. 4082 */ 4083 static int 4084 spell_read_wordfile(spellinfo_T *spin, char_u *fname) 4085 { 4086 FILE *fd; 4087 long lnum = 0; 4088 char_u rline[MAXLINELEN]; 4089 char_u *line; 4090 char_u *pc = NULL; 4091 char_u *p; 4092 int l; 4093 int retval = OK; 4094 int did_word = FALSE; 4095 int non_ascii = 0; 4096 int flags; 4097 int regionmask; 4098 4099 /* 4100 * Open the file. 4101 */ 4102 fd = mch_fopen((char *)fname, "r"); 4103 if (fd == NULL) 4104 { 4105 semsg(_(e_notopen), fname); 4106 return FAIL; 4107 } 4108 4109 vim_snprintf((char *)IObuff, IOSIZE, _("Reading word file %s..."), fname); 4110 spell_message(spin, IObuff); 4111 4112 /* 4113 * Read all the lines in the file one by one. 4114 */ 4115 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 4116 { 4117 line_breakcheck(); 4118 ++lnum; 4119 4120 // Skip comment lines. 4121 if (*rline == '#') 4122 continue; 4123 4124 // Remove CR, LF and white space from the end. 4125 l = (int)STRLEN(rline); 4126 while (l > 0 && rline[l - 1] <= ' ') 4127 --l; 4128 if (l == 0) 4129 continue; // empty or blank line 4130 rline[l] = NUL; 4131 4132 // Convert from "/encoding={encoding}" to 'encoding' when needed. 4133 vim_free(pc); 4134 if (spin->si_conv.vc_type != CONV_NONE) 4135 { 4136 pc = string_convert(&spin->si_conv, rline, NULL); 4137 if (pc == NULL) 4138 { 4139 smsg(_("Conversion failure for word in %s line %ld: %s"), 4140 fname, lnum, rline); 4141 continue; 4142 } 4143 line = pc; 4144 } 4145 else 4146 { 4147 pc = NULL; 4148 line = rline; 4149 } 4150 4151 if (*line == '/') 4152 { 4153 ++line; 4154 if (STRNCMP(line, "encoding=", 9) == 0) 4155 { 4156 if (spin->si_conv.vc_type != CONV_NONE) 4157 smsg(_("Duplicate /encoding= line ignored in %s line %ld: %s"), 4158 fname, lnum, line - 1); 4159 else if (did_word) 4160 smsg(_("/encoding= line after word ignored in %s line %ld: %s"), 4161 fname, lnum, line - 1); 4162 else 4163 { 4164 char_u *enc; 4165 4166 // Setup for conversion to 'encoding'. 4167 line += 9; 4168 enc = enc_canonize(line); 4169 if (enc != NULL && !spin->si_ascii 4170 && convert_setup(&spin->si_conv, enc, 4171 p_enc) == FAIL) 4172 smsg(_("Conversion in %s not supported: from %s to %s"), 4173 fname, line, p_enc); 4174 vim_free(enc); 4175 spin->si_conv.vc_fail = TRUE; 4176 } 4177 continue; 4178 } 4179 4180 if (STRNCMP(line, "regions=", 8) == 0) 4181 { 4182 if (spin->si_region_count > 1) 4183 smsg(_("Duplicate /regions= line ignored in %s line %ld: %s"), 4184 fname, lnum, line); 4185 else 4186 { 4187 line += 8; 4188 if (STRLEN(line) > MAXREGIONS * 2) 4189 smsg(_("Too many regions in %s line %ld: %s"), 4190 fname, lnum, line); 4191 else 4192 { 4193 spin->si_region_count = (int)STRLEN(line) / 2; 4194 STRCPY(spin->si_region_name, line); 4195 4196 // Adjust the mask for a word valid in all regions. 4197 spin->si_region = (1 << spin->si_region_count) - 1; 4198 } 4199 } 4200 continue; 4201 } 4202 4203 smsg(_("/ line ignored in %s line %ld: %s"), 4204 fname, lnum, line - 1); 4205 continue; 4206 } 4207 4208 flags = 0; 4209 regionmask = spin->si_region; 4210 4211 // Check for flags and region after a slash. 4212 p = vim_strchr(line, '/'); 4213 if (p != NULL) 4214 { 4215 *p++ = NUL; 4216 while (*p != NUL) 4217 { 4218 if (*p == '=') // keep-case word 4219 flags |= WF_KEEPCAP | WF_FIXCAP; 4220 else if (*p == '!') // Bad, bad, wicked word. 4221 flags |= WF_BANNED; 4222 else if (*p == '?') // Rare word. 4223 flags |= WF_RARE; 4224 else if (VIM_ISDIGIT(*p)) // region number(s) 4225 { 4226 if ((flags & WF_REGION) == 0) // first one 4227 regionmask = 0; 4228 flags |= WF_REGION; 4229 4230 l = *p - '0'; 4231 if (l == 0 || l > spin->si_region_count) 4232 { 4233 smsg(_("Invalid region nr in %s line %ld: %s"), 4234 fname, lnum, p); 4235 break; 4236 } 4237 regionmask |= 1 << (l - 1); 4238 } 4239 else 4240 { 4241 smsg(_("Unrecognized flags in %s line %ld: %s"), 4242 fname, lnum, p); 4243 break; 4244 } 4245 ++p; 4246 } 4247 } 4248 4249 // Skip non-ASCII words when "spin->si_ascii" is TRUE. 4250 if (spin->si_ascii && has_non_ascii(line)) 4251 { 4252 ++non_ascii; 4253 continue; 4254 } 4255 4256 // Normal word: store it. 4257 if (store_word(spin, line, flags, regionmask, NULL, FALSE) == FAIL) 4258 { 4259 retval = FAIL; 4260 break; 4261 } 4262 did_word = TRUE; 4263 } 4264 4265 vim_free(pc); 4266 fclose(fd); 4267 4268 if (spin->si_ascii && non_ascii > 0) 4269 { 4270 vim_snprintf((char *)IObuff, IOSIZE, 4271 _("Ignored %d words with non-ASCII characters"), non_ascii); 4272 spell_message(spin, IObuff); 4273 } 4274 4275 return retval; 4276 } 4277 4278 /* 4279 * Get part of an sblock_T, "len" bytes long. 4280 * This avoids calling free() for every little struct we use (and keeping 4281 * track of them). 4282 * The memory is cleared to all zeros. 4283 * Returns NULL when out of memory. 4284 */ 4285 static void * 4286 getroom( 4287 spellinfo_T *spin, 4288 size_t len, // length needed 4289 int align) // align for pointer 4290 { 4291 char_u *p; 4292 sblock_T *bl = spin->si_blocks; 4293 4294 if (align && bl != NULL) 4295 // Round size up for alignment. On some systems structures need to be 4296 // aligned to the size of a pointer (e.g., SPARC). 4297 bl->sb_used = (bl->sb_used + sizeof(char *) - 1) 4298 & ~(sizeof(char *) - 1); 4299 4300 if (bl == NULL || bl->sb_used + len > SBLOCKSIZE) 4301 { 4302 if (len >= SBLOCKSIZE) 4303 bl = NULL; 4304 else 4305 // Allocate a block of memory. It is not freed until much later. 4306 bl = alloc_clear(sizeof(sblock_T) + SBLOCKSIZE); 4307 if (bl == NULL) 4308 { 4309 if (!spin->si_did_emsg) 4310 { 4311 emsg(_("E845: Insufficient memory, word list will be incomplete")); 4312 spin->si_did_emsg = TRUE; 4313 } 4314 return NULL; 4315 } 4316 bl->sb_next = spin->si_blocks; 4317 spin->si_blocks = bl; 4318 bl->sb_used = 0; 4319 ++spin->si_blocks_cnt; 4320 } 4321 4322 p = bl->sb_data + bl->sb_used; 4323 bl->sb_used += (int)len; 4324 4325 return p; 4326 } 4327 4328 /* 4329 * Make a copy of a string into memory allocated with getroom(). 4330 * Returns NULL when out of memory. 4331 */ 4332 static char_u * 4333 getroom_save(spellinfo_T *spin, char_u *s) 4334 { 4335 char_u *sc; 4336 4337 sc = (char_u *)getroom(spin, STRLEN(s) + 1, FALSE); 4338 if (sc != NULL) 4339 STRCPY(sc, s); 4340 return sc; 4341 } 4342 4343 4344 /* 4345 * Free the list of allocated sblock_T. 4346 */ 4347 static void 4348 free_blocks(sblock_T *bl) 4349 { 4350 sblock_T *next; 4351 4352 while (bl != NULL) 4353 { 4354 next = bl->sb_next; 4355 vim_free(bl); 4356 bl = next; 4357 } 4358 } 4359 4360 /* 4361 * Allocate the root of a word tree. 4362 * Returns NULL when out of memory. 4363 */ 4364 static wordnode_T * 4365 wordtree_alloc(spellinfo_T *spin) 4366 { 4367 return (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 4368 } 4369 4370 /* 4371 * Store a word in the tree(s). 4372 * Always store it in the case-folded tree. For a keep-case word this is 4373 * useful when the word can also be used with all caps (no WF_FIXCAP flag) and 4374 * used to find suggestions. 4375 * For a keep-case word also store it in the keep-case tree. 4376 * When "pfxlist" is not NULL store the word for each postponed prefix ID and 4377 * compound flag. 4378 */ 4379 static int 4380 store_word( 4381 spellinfo_T *spin, 4382 char_u *word, 4383 int flags, // extra flags, WF_BANNED 4384 int region, // supported region(s) 4385 char_u *pfxlist, // list of prefix IDs or NULL 4386 int need_affix) // only store word with affix ID 4387 { 4388 int len = (int)STRLEN(word); 4389 int ct = captype(word, word + len); 4390 char_u foldword[MAXWLEN]; 4391 int res = OK; 4392 char_u *p; 4393 4394 (void)spell_casefold(word, len, foldword, MAXWLEN); 4395 for (p = pfxlist; res == OK; ++p) 4396 { 4397 if (!need_affix || (p != NULL && *p != NUL)) 4398 res = tree_add_word(spin, foldword, spin->si_foldroot, ct | flags, 4399 region, p == NULL ? 0 : *p); 4400 if (p == NULL || *p == NUL) 4401 break; 4402 } 4403 ++spin->si_foldwcount; 4404 4405 if (res == OK && (ct == WF_KEEPCAP || (flags & WF_KEEPCAP))) 4406 { 4407 for (p = pfxlist; res == OK; ++p) 4408 { 4409 if (!need_affix || (p != NULL && *p != NUL)) 4410 res = tree_add_word(spin, word, spin->si_keeproot, flags, 4411 region, p == NULL ? 0 : *p); 4412 if (p == NULL || *p == NUL) 4413 break; 4414 } 4415 ++spin->si_keepwcount; 4416 } 4417 return res; 4418 } 4419 4420 /* 4421 * Add word "word" to a word tree at "root". 4422 * When "flags" < 0 we are adding to the prefix tree where "flags" is used for 4423 * "rare" and "region" is the condition nr. 4424 * Returns FAIL when out of memory. 4425 */ 4426 static int 4427 tree_add_word( 4428 spellinfo_T *spin, 4429 char_u *word, 4430 wordnode_T *root, 4431 int flags, 4432 int region, 4433 int affixID) 4434 { 4435 wordnode_T *node = root; 4436 wordnode_T *np; 4437 wordnode_T *copyp, **copyprev; 4438 wordnode_T **prev = NULL; 4439 int i; 4440 4441 // Add each byte of the word to the tree, including the NUL at the end. 4442 for (i = 0; ; ++i) 4443 { 4444 // When there is more than one reference to this node we need to make 4445 // a copy, so that we can modify it. Copy the whole list of siblings 4446 // (we don't optimize for a partly shared list of siblings). 4447 if (node != NULL && node->wn_refs > 1) 4448 { 4449 --node->wn_refs; 4450 copyprev = prev; 4451 FOR_ALL_NODE_SIBLINGS(node, copyp) 4452 { 4453 // Allocate a new node and copy the info. 4454 np = get_wordnode(spin); 4455 if (np == NULL) 4456 return FAIL; 4457 np->wn_child = copyp->wn_child; 4458 if (np->wn_child != NULL) 4459 ++np->wn_child->wn_refs; // child gets extra ref 4460 np->wn_byte = copyp->wn_byte; 4461 if (np->wn_byte == NUL) 4462 { 4463 np->wn_flags = copyp->wn_flags; 4464 np->wn_region = copyp->wn_region; 4465 np->wn_affixID = copyp->wn_affixID; 4466 } 4467 4468 // Link the new node in the list, there will be one ref. 4469 np->wn_refs = 1; 4470 if (copyprev != NULL) 4471 *copyprev = np; 4472 copyprev = &np->wn_sibling; 4473 4474 // Let "node" point to the head of the copied list. 4475 if (copyp == node) 4476 node = np; 4477 } 4478 } 4479 4480 // Look for the sibling that has the same character. They are sorted 4481 // on byte value, thus stop searching when a sibling is found with a 4482 // higher byte value. For zero bytes (end of word) the sorting is 4483 // done on flags and then on affixID. 4484 while (node != NULL 4485 && (node->wn_byte < word[i] 4486 || (node->wn_byte == NUL 4487 && (flags < 0 4488 ? node->wn_affixID < (unsigned)affixID 4489 : (node->wn_flags < (unsigned)(flags & WN_MASK) 4490 || (node->wn_flags == (flags & WN_MASK) 4491 && (spin->si_sugtree 4492 ? (node->wn_region & 0xffff) < region 4493 : node->wn_affixID 4494 < (unsigned)affixID))))))) 4495 { 4496 prev = &node->wn_sibling; 4497 node = *prev; 4498 } 4499 if (node == NULL 4500 || node->wn_byte != word[i] 4501 || (word[i] == NUL 4502 && (flags < 0 4503 || spin->si_sugtree 4504 || node->wn_flags != (flags & WN_MASK) 4505 || node->wn_affixID != affixID))) 4506 { 4507 // Allocate a new node. 4508 np = get_wordnode(spin); 4509 if (np == NULL) 4510 return FAIL; 4511 np->wn_byte = word[i]; 4512 4513 // If "node" is NULL this is a new child or the end of the sibling 4514 // list: ref count is one. Otherwise use ref count of sibling and 4515 // make ref count of sibling one (matters when inserting in front 4516 // of the list of siblings). 4517 if (node == NULL) 4518 np->wn_refs = 1; 4519 else 4520 { 4521 np->wn_refs = node->wn_refs; 4522 node->wn_refs = 1; 4523 } 4524 if (prev != NULL) 4525 *prev = np; 4526 np->wn_sibling = node; 4527 node = np; 4528 } 4529 4530 if (word[i] == NUL) 4531 { 4532 node->wn_flags = flags; 4533 node->wn_region |= region; 4534 node->wn_affixID = affixID; 4535 break; 4536 } 4537 prev = &node->wn_child; 4538 node = *prev; 4539 } 4540 #ifdef SPELL_PRINTTREE 4541 smsg("Added \"%s\"", word); 4542 spell_print_tree(root->wn_sibling); 4543 #endif 4544 4545 // count nr of words added since last message 4546 ++spin->si_msg_count; 4547 4548 if (spin->si_compress_cnt > 1) 4549 { 4550 if (--spin->si_compress_cnt == 1) 4551 // Did enough words to lower the block count limit. 4552 spin->si_blocks_cnt += compress_inc; 4553 } 4554 4555 /* 4556 * When we have allocated lots of memory we need to compress the word tree 4557 * to free up some room. But compression is slow, and we might actually 4558 * need that room, thus only compress in the following situations: 4559 * 1. When not compressed before (si_compress_cnt == 0): when using 4560 * "compress_start" blocks. 4561 * 2. When compressed before and used "compress_inc" blocks before 4562 * adding "compress_added" words (si_compress_cnt > 1). 4563 * 3. When compressed before, added "compress_added" words 4564 * (si_compress_cnt == 1) and the number of free nodes drops below the 4565 * maximum word length. 4566 */ 4567 #ifndef SPELL_COMPRESS_ALLWAYS 4568 if (spin->si_compress_cnt == 1 4569 ? spin->si_free_count < MAXWLEN 4570 : spin->si_blocks_cnt >= compress_start) 4571 #endif 4572 { 4573 // Decrement the block counter. The effect is that we compress again 4574 // when the freed up room has been used and another "compress_inc" 4575 // blocks have been allocated. Unless "compress_added" words have 4576 // been added, then the limit is put back again. 4577 spin->si_blocks_cnt -= compress_inc; 4578 spin->si_compress_cnt = compress_added; 4579 4580 if (spin->si_verbose) 4581 { 4582 msg_start(); 4583 msg_puts(_(msg_compressing)); 4584 msg_clr_eos(); 4585 msg_didout = FALSE; 4586 msg_col = 0; 4587 out_flush(); 4588 } 4589 4590 // Compress both trees. Either they both have many nodes, which makes 4591 // compression useful, or one of them is small, which means 4592 // compression goes fast. But when filling the soundfold word tree 4593 // there is no keep-case tree. 4594 wordtree_compress(spin, spin->si_foldroot, "case-folded"); 4595 if (affixID >= 0) 4596 wordtree_compress(spin, spin->si_keeproot, "keep-case"); 4597 } 4598 4599 return OK; 4600 } 4601 4602 /* 4603 * Get a wordnode_T, either from the list of previously freed nodes or 4604 * allocate a new one. 4605 * Returns NULL when out of memory. 4606 */ 4607 static wordnode_T * 4608 get_wordnode(spellinfo_T *spin) 4609 { 4610 wordnode_T *n; 4611 4612 if (spin->si_first_free == NULL) 4613 n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 4614 else 4615 { 4616 n = spin->si_first_free; 4617 spin->si_first_free = n->wn_child; 4618 CLEAR_POINTER(n); 4619 --spin->si_free_count; 4620 } 4621 #ifdef SPELL_PRINTTREE 4622 if (n != NULL) 4623 n->wn_nr = ++spin->si_wordnode_nr; 4624 #endif 4625 return n; 4626 } 4627 4628 /* 4629 * Decrement the reference count on a node (which is the head of a list of 4630 * siblings). If the reference count becomes zero free the node and its 4631 * siblings. 4632 * Returns the number of nodes actually freed. 4633 */ 4634 static int 4635 deref_wordnode(spellinfo_T *spin, wordnode_T *node) 4636 { 4637 wordnode_T *np; 4638 int cnt = 0; 4639 4640 if (--node->wn_refs == 0) 4641 { 4642 FOR_ALL_NODE_SIBLINGS(node, np) 4643 { 4644 if (np->wn_child != NULL) 4645 cnt += deref_wordnode(spin, np->wn_child); 4646 free_wordnode(spin, np); 4647 ++cnt; 4648 } 4649 ++cnt; // length field 4650 } 4651 return cnt; 4652 } 4653 4654 /* 4655 * Free a wordnode_T for re-use later. 4656 * Only the "wn_child" field becomes invalid. 4657 */ 4658 static void 4659 free_wordnode(spellinfo_T *spin, wordnode_T *n) 4660 { 4661 n->wn_child = spin->si_first_free; 4662 spin->si_first_free = n; 4663 ++spin->si_free_count; 4664 } 4665 4666 /* 4667 * Compress a tree: find tails that are identical and can be shared. 4668 */ 4669 static void 4670 wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name) 4671 { 4672 hashtab_T ht; 4673 long n; 4674 long tot = 0; 4675 long perc; 4676 4677 // Skip the root itself, it's not actually used. The first sibling is the 4678 // start of the tree. 4679 if (root->wn_sibling != NULL) 4680 { 4681 hash_init(&ht); 4682 n = node_compress(spin, root->wn_sibling, &ht, &tot); 4683 4684 #ifndef SPELL_PRINTTREE 4685 if (spin->si_verbose || p_verbose > 2) 4686 #endif 4687 { 4688 if (tot > 1000000) 4689 perc = (tot - n) / (tot / 100); 4690 else if (tot == 0) 4691 perc = 0; 4692 else 4693 perc = (tot - n) * 100 / tot; 4694 vim_snprintf((char *)IObuff, IOSIZE, 4695 _("Compressed %s: %ld of %ld nodes; %ld (%ld%%) remaining"), 4696 name, n, tot, tot - n, perc); 4697 spell_message(spin, IObuff); 4698 } 4699 #ifdef SPELL_PRINTTREE 4700 spell_print_tree(root->wn_sibling); 4701 #endif 4702 hash_clear(&ht); 4703 } 4704 } 4705 4706 /* 4707 * Compress a node, its siblings and its children, depth first. 4708 * Returns the number of compressed nodes. 4709 */ 4710 static long 4711 node_compress( 4712 spellinfo_T *spin, 4713 wordnode_T *node, 4714 hashtab_T *ht, 4715 long *tot) // total count of nodes before compressing, 4716 // incremented while going through the tree 4717 { 4718 wordnode_T *np; 4719 wordnode_T *tp; 4720 wordnode_T *child; 4721 hash_T hash; 4722 hashitem_T *hi; 4723 long len = 0; 4724 unsigned nr, n; 4725 long compressed = 0; 4726 4727 /* 4728 * Go through the list of siblings. Compress each child and then try 4729 * finding an identical child to replace it. 4730 * Note that with "child" we mean not just the node that is pointed to, 4731 * but the whole list of siblings of which the child node is the first. 4732 */ 4733 for (np = node; np != NULL && !got_int; np = np->wn_sibling) 4734 { 4735 ++len; 4736 if ((child = np->wn_child) != NULL) 4737 { 4738 // Compress the child first. This fills hashkey. 4739 compressed += node_compress(spin, child, ht, tot); 4740 4741 // Try to find an identical child. 4742 hash = hash_hash(child->wn_u1.hashkey); 4743 hi = hash_lookup(ht, child->wn_u1.hashkey, hash); 4744 if (!HASHITEM_EMPTY(hi)) 4745 { 4746 // There are children we encountered before with a hash value 4747 // identical to the current child. Now check if there is one 4748 // that is really identical. 4749 for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next) 4750 if (node_equal(child, tp)) 4751 { 4752 // Found one! Now use that child in place of the 4753 // current one. This means the current child and all 4754 // its siblings is unlinked from the tree. 4755 ++tp->wn_refs; 4756 compressed += deref_wordnode(spin, child); 4757 np->wn_child = tp; 4758 break; 4759 } 4760 if (tp == NULL) 4761 { 4762 // No other child with this hash value equals the child of 4763 // the node, add it to the linked list after the first 4764 // item. 4765 tp = HI2WN(hi); 4766 child->wn_u2.next = tp->wn_u2.next; 4767 tp->wn_u2.next = child; 4768 } 4769 } 4770 else 4771 // No other child has this hash value, add it to the 4772 // hashtable. 4773 hash_add_item(ht, hi, child->wn_u1.hashkey, hash); 4774 } 4775 } 4776 *tot += len + 1; // add one for the node that stores the length 4777 4778 /* 4779 * Make a hash key for the node and its siblings, so that we can quickly 4780 * find a lookalike node. This must be done after compressing the sibling 4781 * list, otherwise the hash key would become invalid by the compression. 4782 */ 4783 node->wn_u1.hashkey[0] = len; 4784 nr = 0; 4785 FOR_ALL_NODE_SIBLINGS(node, np) 4786 { 4787 if (np->wn_byte == NUL) 4788 // end node: use wn_flags, wn_region and wn_affixID 4789 n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16); 4790 else 4791 // byte node: use the byte value and the child pointer 4792 n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8)); 4793 nr = nr * 101 + n; 4794 } 4795 4796 // Avoid NUL bytes, it terminates the hash key. 4797 n = nr & 0xff; 4798 node->wn_u1.hashkey[1] = n == 0 ? 1 : n; 4799 n = (nr >> 8) & 0xff; 4800 node->wn_u1.hashkey[2] = n == 0 ? 1 : n; 4801 n = (nr >> 16) & 0xff; 4802 node->wn_u1.hashkey[3] = n == 0 ? 1 : n; 4803 n = (nr >> 24) & 0xff; 4804 node->wn_u1.hashkey[4] = n == 0 ? 1 : n; 4805 node->wn_u1.hashkey[5] = NUL; 4806 4807 // Check for CTRL-C pressed now and then. 4808 veryfast_breakcheck(); 4809 4810 return compressed; 4811 } 4812 4813 /* 4814 * Return TRUE when two nodes have identical siblings and children. 4815 */ 4816 static int 4817 node_equal(wordnode_T *n1, wordnode_T *n2) 4818 { 4819 wordnode_T *p1; 4820 wordnode_T *p2; 4821 4822 for (p1 = n1, p2 = n2; p1 != NULL && p2 != NULL; 4823 p1 = p1->wn_sibling, p2 = p2->wn_sibling) 4824 if (p1->wn_byte != p2->wn_byte 4825 || (p1->wn_byte == NUL 4826 ? (p1->wn_flags != p2->wn_flags 4827 || p1->wn_region != p2->wn_region 4828 || p1->wn_affixID != p2->wn_affixID) 4829 : (p1->wn_child != p2->wn_child))) 4830 break; 4831 4832 return p1 == NULL && p2 == NULL; 4833 } 4834 4835 static int rep_compare(const void *s1, const void *s2); 4836 4837 /* 4838 * Function given to qsort() to sort the REP items on "from" string. 4839 */ 4840 static int 4841 rep_compare(const void *s1, const void *s2) 4842 { 4843 fromto_T *p1 = (fromto_T *)s1; 4844 fromto_T *p2 = (fromto_T *)s2; 4845 4846 return STRCMP(p1->ft_from, p2->ft_from); 4847 } 4848 4849 /* 4850 * Write the Vim .spl file "fname". 4851 * Return FAIL or OK; 4852 */ 4853 static int 4854 write_vim_spell(spellinfo_T *spin, char_u *fname) 4855 { 4856 FILE *fd; 4857 int regionmask; 4858 int round; 4859 wordnode_T *tree; 4860 int nodecount; 4861 int i; 4862 int l; 4863 garray_T *gap; 4864 fromto_T *ftp; 4865 char_u *p; 4866 int rr; 4867 int retval = OK; 4868 size_t fwv = 1; // collect return value of fwrite() to avoid 4869 // warnings from picky compiler 4870 4871 fd = mch_fopen((char *)fname, "w"); 4872 if (fd == NULL) 4873 { 4874 semsg(_(e_notopen), fname); 4875 return FAIL; 4876 } 4877 4878 // <HEADER>: <fileID> <versionnr> 4879 // <fileID> 4880 fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd); 4881 if (fwv != (size_t)1) 4882 // Catch first write error, don't try writing more. 4883 goto theend; 4884 4885 putc(VIMSPELLVERSION, fd); // <versionnr> 4886 4887 /* 4888 * <SECTIONS>: <section> ... <sectionend> 4889 */ 4890 4891 // SN_INFO: <infotext> 4892 if (spin->si_info != NULL) 4893 { 4894 putc(SN_INFO, fd); // <sectionID> 4895 putc(0, fd); // <sectionflags> 4896 4897 i = (int)STRLEN(spin->si_info); 4898 put_bytes(fd, (long_u)i, 4); // <sectionlen> 4899 fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); // <infotext> 4900 } 4901 4902 // SN_REGION: <regionname> ... 4903 // Write the region names only if there is more than one. 4904 if (spin->si_region_count > 1) 4905 { 4906 putc(SN_REGION, fd); // <sectionID> 4907 putc(SNF_REQUIRED, fd); // <sectionflags> 4908 l = spin->si_region_count * 2; 4909 put_bytes(fd, (long_u)l, 4); // <sectionlen> 4910 fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd); 4911 // <regionname> ... 4912 regionmask = (1 << spin->si_region_count) - 1; 4913 } 4914 else 4915 regionmask = 0; 4916 4917 // SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars> 4918 // 4919 // The table with character flags and the table for case folding. 4920 // This makes sure the same characters are recognized as word characters 4921 // when generating an when using a spell file. 4922 // Skip this for ASCII, the table may conflict with the one used for 4923 // 'encoding'. 4924 // Also skip this for an .add.spl file, the main spell file must contain 4925 // the table (avoids that it conflicts). File is shorter too. 4926 if (!spin->si_ascii && !spin->si_add) 4927 { 4928 char_u folchars[128 * 8]; 4929 int flags; 4930 4931 putc(SN_CHARFLAGS, fd); // <sectionID> 4932 putc(SNF_REQUIRED, fd); // <sectionflags> 4933 4934 // Form the <folchars> string first, we need to know its length. 4935 l = 0; 4936 for (i = 128; i < 256; ++i) 4937 { 4938 if (has_mbyte) 4939 l += mb_char2bytes(spelltab.st_fold[i], folchars + l); 4940 else 4941 folchars[l++] = spelltab.st_fold[i]; 4942 } 4943 put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4); // <sectionlen> 4944 4945 fputc(128, fd); // <charflagslen> 4946 for (i = 128; i < 256; ++i) 4947 { 4948 flags = 0; 4949 if (spelltab.st_isw[i]) 4950 flags |= CF_WORD; 4951 if (spelltab.st_isu[i]) 4952 flags |= CF_UPPER; 4953 fputc(flags, fd); // <charflags> 4954 } 4955 4956 put_bytes(fd, (long_u)l, 2); // <folcharslen> 4957 fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); // <folchars> 4958 } 4959 4960 // SN_MIDWORD: <midword> 4961 if (spin->si_midword != NULL) 4962 { 4963 putc(SN_MIDWORD, fd); // <sectionID> 4964 putc(SNF_REQUIRED, fd); // <sectionflags> 4965 4966 i = (int)STRLEN(spin->si_midword); 4967 put_bytes(fd, (long_u)i, 4); // <sectionlen> 4968 fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd); 4969 // <midword> 4970 } 4971 4972 // SN_PREFCOND: <prefcondcnt> <prefcond> ... 4973 if (spin->si_prefcond.ga_len > 0) 4974 { 4975 putc(SN_PREFCOND, fd); // <sectionID> 4976 putc(SNF_REQUIRED, fd); // <sectionflags> 4977 4978 l = write_spell_prefcond(NULL, &spin->si_prefcond); 4979 put_bytes(fd, (long_u)l, 4); // <sectionlen> 4980 4981 write_spell_prefcond(fd, &spin->si_prefcond); 4982 } 4983 4984 // SN_REP: <repcount> <rep> ... 4985 // SN_SAL: <salflags> <salcount> <sal> ... 4986 // SN_REPSAL: <repcount> <rep> ... 4987 4988 // round 1: SN_REP section 4989 // round 2: SN_SAL section (unless SN_SOFO is used) 4990 // round 3: SN_REPSAL section 4991 for (round = 1; round <= 3; ++round) 4992 { 4993 if (round == 1) 4994 gap = &spin->si_rep; 4995 else if (round == 2) 4996 { 4997 // Don't write SN_SAL when using a SN_SOFO section 4998 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 4999 continue; 5000 gap = &spin->si_sal; 5001 } 5002 else 5003 gap = &spin->si_repsal; 5004 5005 // Don't write the section if there are no items. 5006 if (gap->ga_len == 0) 5007 continue; 5008 5009 // Sort the REP/REPSAL items. 5010 if (round != 2) 5011 qsort(gap->ga_data, (size_t)gap->ga_len, 5012 sizeof(fromto_T), rep_compare); 5013 5014 i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL); 5015 putc(i, fd); // <sectionID> 5016 5017 // This is for making suggestions, section is not required. 5018 putc(0, fd); // <sectionflags> 5019 5020 // Compute the length of what follows. 5021 l = 2; // count <repcount> or <salcount> 5022 for (i = 0; i < gap->ga_len; ++i) 5023 { 5024 ftp = &((fromto_T *)gap->ga_data)[i]; 5025 l += 1 + (int)STRLEN(ftp->ft_from); // count <*fromlen> and <*from> 5026 l += 1 + (int)STRLEN(ftp->ft_to); // count <*tolen> and <*to> 5027 } 5028 if (round == 2) 5029 ++l; // count <salflags> 5030 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5031 5032 if (round == 2) 5033 { 5034 i = 0; 5035 if (spin->si_followup) 5036 i |= SAL_F0LLOWUP; 5037 if (spin->si_collapse) 5038 i |= SAL_COLLAPSE; 5039 if (spin->si_rem_accents) 5040 i |= SAL_REM_ACCENTS; 5041 putc(i, fd); // <salflags> 5042 } 5043 5044 put_bytes(fd, (long_u)gap->ga_len, 2); // <repcount> or <salcount> 5045 for (i = 0; i < gap->ga_len; ++i) 5046 { 5047 // <rep> : <repfromlen> <repfrom> <reptolen> <repto> 5048 // <sal> : <salfromlen> <salfrom> <saltolen> <salto> 5049 ftp = &((fromto_T *)gap->ga_data)[i]; 5050 for (rr = 1; rr <= 2; ++rr) 5051 { 5052 p = rr == 1 ? ftp->ft_from : ftp->ft_to; 5053 l = (int)STRLEN(p); 5054 putc(l, fd); 5055 if (l > 0) 5056 fwv &= fwrite(p, l, (size_t)1, fd); 5057 } 5058 } 5059 5060 } 5061 5062 // SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 5063 // This is for making suggestions, section is not required. 5064 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 5065 { 5066 putc(SN_SOFO, fd); // <sectionID> 5067 putc(0, fd); // <sectionflags> 5068 5069 l = (int)STRLEN(spin->si_sofofr); 5070 put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4); 5071 // <sectionlen> 5072 5073 put_bytes(fd, (long_u)l, 2); // <sofofromlen> 5074 fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); // <sofofrom> 5075 5076 l = (int)STRLEN(spin->si_sofoto); 5077 put_bytes(fd, (long_u)l, 2); // <sofotolen> 5078 fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); // <sofoto> 5079 } 5080 5081 // SN_WORDS: <word> ... 5082 // This is for making suggestions, section is not required. 5083 if (spin->si_commonwords.ht_used > 0) 5084 { 5085 putc(SN_WORDS, fd); // <sectionID> 5086 putc(0, fd); // <sectionflags> 5087 5088 // round 1: count the bytes 5089 // round 2: write the bytes 5090 for (round = 1; round <= 2; ++round) 5091 { 5092 int todo; 5093 int len = 0; 5094 hashitem_T *hi; 5095 5096 todo = (int)spin->si_commonwords.ht_used; 5097 for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi) 5098 if (!HASHITEM_EMPTY(hi)) 5099 { 5100 l = (int)STRLEN(hi->hi_key) + 1; 5101 len += l; 5102 if (round == 2) // <word> 5103 fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd); 5104 --todo; 5105 } 5106 if (round == 1) 5107 put_bytes(fd, (long_u)len, 4); // <sectionlen> 5108 } 5109 } 5110 5111 // SN_MAP: <mapstr> 5112 // This is for making suggestions, section is not required. 5113 if (spin->si_map.ga_len > 0) 5114 { 5115 putc(SN_MAP, fd); // <sectionID> 5116 putc(0, fd); // <sectionflags> 5117 l = spin->si_map.ga_len; 5118 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5119 fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd); 5120 // <mapstr> 5121 } 5122 5123 // SN_SUGFILE: <timestamp> 5124 // This is used to notify that a .sug file may be available and at the 5125 // same time allows for checking that a .sug file that is found matches 5126 // with this .spl file. That's because the word numbers must be exactly 5127 // right. 5128 if (!spin->si_nosugfile 5129 && (spin->si_sal.ga_len > 0 5130 || (spin->si_sofofr != NULL && spin->si_sofoto != NULL))) 5131 { 5132 putc(SN_SUGFILE, fd); // <sectionID> 5133 putc(0, fd); // <sectionflags> 5134 put_bytes(fd, (long_u)8, 4); // <sectionlen> 5135 5136 // Set si_sugtime and write it to the file. 5137 spin->si_sugtime = time(NULL); 5138 put_time(fd, spin->si_sugtime); // <timestamp> 5139 } 5140 5141 // SN_NOSPLITSUGS: nothing 5142 // This is used to notify that no suggestions with word splits are to be 5143 // made. 5144 if (spin->si_nosplitsugs) 5145 { 5146 putc(SN_NOSPLITSUGS, fd); // <sectionID> 5147 putc(0, fd); // <sectionflags> 5148 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5149 } 5150 5151 // SN_NOCOMPUNDSUGS: nothing 5152 // This is used to notify that no suggestions with compounds are to be 5153 // made. 5154 if (spin->si_nocompoundsugs) 5155 { 5156 putc(SN_NOCOMPOUNDSUGS, fd); // <sectionID> 5157 putc(0, fd); // <sectionflags> 5158 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5159 } 5160 5161 // SN_COMPOUND: compound info. 5162 // We don't mark it required, when not supported all compound words will 5163 // be bad words. 5164 if (spin->si_compflags != NULL) 5165 { 5166 putc(SN_COMPOUND, fd); // <sectionID> 5167 putc(0, fd); // <sectionflags> 5168 5169 l = (int)STRLEN(spin->si_compflags); 5170 for (i = 0; i < spin->si_comppat.ga_len; ++i) 5171 l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1; 5172 put_bytes(fd, (long_u)(l + 7), 4); // <sectionlen> 5173 5174 putc(spin->si_compmax, fd); // <compmax> 5175 putc(spin->si_compminlen, fd); // <compminlen> 5176 putc(spin->si_compsylmax, fd); // <compsylmax> 5177 putc(0, fd); // for Vim 7.0b compatibility 5178 putc(spin->si_compoptions, fd); // <compoptions> 5179 put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2); 5180 // <comppatcount> 5181 for (i = 0; i < spin->si_comppat.ga_len; ++i) 5182 { 5183 p = ((char_u **)(spin->si_comppat.ga_data))[i]; 5184 putc((int)STRLEN(p), fd); // <comppatlen> 5185 fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd); 5186 // <comppattext> 5187 } 5188 // <compflags> 5189 fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags), 5190 (size_t)1, fd); 5191 } 5192 5193 // SN_NOBREAK: NOBREAK flag 5194 if (spin->si_nobreak) 5195 { 5196 putc(SN_NOBREAK, fd); // <sectionID> 5197 putc(0, fd); // <sectionflags> 5198 5199 // It's empty, the presence of the section flags the feature. 5200 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5201 } 5202 5203 // SN_SYLLABLE: syllable info. 5204 // We don't mark it required, when not supported syllables will not be 5205 // counted. 5206 if (spin->si_syllable != NULL) 5207 { 5208 putc(SN_SYLLABLE, fd); // <sectionID> 5209 putc(0, fd); // <sectionflags> 5210 5211 l = (int)STRLEN(spin->si_syllable); 5212 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5213 fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd); 5214 // <syllable> 5215 } 5216 5217 // end of <SECTIONS> 5218 putc(SN_END, fd); // <sectionend> 5219 5220 5221 /* 5222 * <LWORDTREE> <KWORDTREE> <PREFIXTREE> 5223 */ 5224 spin->si_memtot = 0; 5225 for (round = 1; round <= 3; ++round) 5226 { 5227 if (round == 1) 5228 tree = spin->si_foldroot->wn_sibling; 5229 else if (round == 2) 5230 tree = spin->si_keeproot->wn_sibling; 5231 else 5232 tree = spin->si_prefroot->wn_sibling; 5233 5234 // Clear the index and wnode fields in the tree. 5235 clear_node(tree); 5236 5237 // Count the number of nodes. Needed to be able to allocate the 5238 // memory when reading the nodes. Also fills in index for shared 5239 // nodes. 5240 nodecount = put_node(NULL, tree, 0, regionmask, round == 3); 5241 5242 // number of nodes in 4 bytes 5243 put_bytes(fd, (long_u)nodecount, 4); // <nodecount> 5244 spin->si_memtot += nodecount + nodecount * sizeof(int); 5245 5246 // Write the nodes. 5247 (void)put_node(fd, tree, 0, regionmask, round == 3); 5248 } 5249 5250 // Write another byte to check for errors (file system full). 5251 if (putc(0, fd) == EOF) 5252 retval = FAIL; 5253 theend: 5254 if (fclose(fd) == EOF) 5255 retval = FAIL; 5256 5257 if (fwv != (size_t)1) 5258 retval = FAIL; 5259 if (retval == FAIL) 5260 emsg(_(e_write)); 5261 5262 return retval; 5263 } 5264 5265 /* 5266 * Clear the index and wnode fields of "node", it siblings and its 5267 * children. This is needed because they are a union with other items to save 5268 * space. 5269 */ 5270 static void 5271 clear_node(wordnode_T *node) 5272 { 5273 wordnode_T *np; 5274 5275 if (node != NULL) 5276 FOR_ALL_NODE_SIBLINGS(node, np) 5277 { 5278 np->wn_u1.index = 0; 5279 np->wn_u2.wnode = NULL; 5280 5281 if (np->wn_byte != NUL) 5282 clear_node(np->wn_child); 5283 } 5284 } 5285 5286 5287 /* 5288 * Dump a word tree at node "node". 5289 * 5290 * This first writes the list of possible bytes (siblings). Then for each 5291 * byte recursively write the children. 5292 * 5293 * NOTE: The code here must match the code in read_tree_node(), since 5294 * assumptions are made about the indexes (so that we don't have to write them 5295 * in the file). 5296 * 5297 * Returns the number of nodes used. 5298 */ 5299 static int 5300 put_node( 5301 FILE *fd, // NULL when only counting 5302 wordnode_T *node, 5303 int idx, 5304 int regionmask, 5305 int prefixtree) // TRUE for PREFIXTREE 5306 { 5307 int newindex = idx; 5308 int siblingcount = 0; 5309 wordnode_T *np; 5310 int flags; 5311 5312 // If "node" is zero the tree is empty. 5313 if (node == NULL) 5314 return 0; 5315 5316 // Store the index where this node is written. 5317 node->wn_u1.index = idx; 5318 5319 // Count the number of siblings. 5320 FOR_ALL_NODE_SIBLINGS(node, np) 5321 ++siblingcount; 5322 5323 // Write the sibling count. 5324 if (fd != NULL) 5325 putc(siblingcount, fd); // <siblingcount> 5326 5327 // Write each sibling byte and optionally extra info. 5328 FOR_ALL_NODE_SIBLINGS(node, np) 5329 { 5330 if (np->wn_byte == 0) 5331 { 5332 if (fd != NULL) 5333 { 5334 // For a NUL byte (end of word) write the flags etc. 5335 if (prefixtree) 5336 { 5337 // In PREFIXTREE write the required affixID and the 5338 // associated condition nr (stored in wn_region). The 5339 // byte value is misused to store the "rare" and "not 5340 // combining" flags 5341 if (np->wn_flags == (short_u)PFX_FLAGS) 5342 putc(BY_NOFLAGS, fd); // <byte> 5343 else 5344 { 5345 putc(BY_FLAGS, fd); // <byte> 5346 putc(np->wn_flags, fd); // <pflags> 5347 } 5348 putc(np->wn_affixID, fd); // <affixID> 5349 put_bytes(fd, (long_u)np->wn_region, 2); // <prefcondnr> 5350 } 5351 else 5352 { 5353 // For word trees we write the flag/region items. 5354 flags = np->wn_flags; 5355 if (regionmask != 0 && np->wn_region != regionmask) 5356 flags |= WF_REGION; 5357 if (np->wn_affixID != 0) 5358 flags |= WF_AFX; 5359 if (flags == 0) 5360 { 5361 // word without flags or region 5362 putc(BY_NOFLAGS, fd); // <byte> 5363 } 5364 else 5365 { 5366 if (np->wn_flags >= 0x100) 5367 { 5368 putc(BY_FLAGS2, fd); // <byte> 5369 putc(flags, fd); // <flags> 5370 putc((unsigned)flags >> 8, fd); // <flags2> 5371 } 5372 else 5373 { 5374 putc(BY_FLAGS, fd); // <byte> 5375 putc(flags, fd); // <flags> 5376 } 5377 if (flags & WF_REGION) 5378 putc(np->wn_region, fd); // <region> 5379 if (flags & WF_AFX) 5380 putc(np->wn_affixID, fd); // <affixID> 5381 } 5382 } 5383 } 5384 } 5385 else 5386 { 5387 if (np->wn_child->wn_u1.index != 0 5388 && np->wn_child->wn_u2.wnode != node) 5389 { 5390 // The child is written elsewhere, write the reference. 5391 if (fd != NULL) 5392 { 5393 putc(BY_INDEX, fd); // <byte> 5394 // <nodeidx> 5395 put_bytes(fd, (long_u)np->wn_child->wn_u1.index, 3); 5396 } 5397 } 5398 else if (np->wn_child->wn_u2.wnode == NULL) 5399 // We will write the child below and give it an index. 5400 np->wn_child->wn_u2.wnode = node; 5401 5402 if (fd != NULL) 5403 if (putc(np->wn_byte, fd) == EOF) // <byte> or <xbyte> 5404 { 5405 emsg(_(e_write)); 5406 return 0; 5407 } 5408 } 5409 } 5410 5411 // Space used in the array when reading: one for each sibling and one for 5412 // the count. 5413 newindex += siblingcount + 1; 5414 5415 // Recursively dump the children of each sibling. 5416 FOR_ALL_NODE_SIBLINGS(node, np) 5417 if (np->wn_byte != 0 && np->wn_child->wn_u2.wnode == node) 5418 newindex = put_node(fd, np->wn_child, newindex, regionmask, 5419 prefixtree); 5420 5421 return newindex; 5422 } 5423 5424 5425 /* 5426 * ":mkspell [-ascii] outfile infile ..." 5427 * ":mkspell [-ascii] addfile" 5428 */ 5429 void 5430 ex_mkspell(exarg_T *eap) 5431 { 5432 int fcount; 5433 char_u **fnames; 5434 char_u *arg = eap->arg; 5435 int ascii = FALSE; 5436 5437 if (STRNCMP(arg, "-ascii", 6) == 0) 5438 { 5439 ascii = TRUE; 5440 arg = skipwhite(arg + 6); 5441 } 5442 5443 // Expand all the remaining arguments (e.g., $VIMRUNTIME). 5444 if (get_arglist_exp(arg, &fcount, &fnames, FALSE) == OK) 5445 { 5446 mkspell(fcount, fnames, ascii, eap->forceit, FALSE); 5447 FreeWild(fcount, fnames); 5448 } 5449 } 5450 5451 /* 5452 * Create the .sug file. 5453 * Uses the soundfold info in "spin". 5454 * Writes the file with the name "wfname", with ".spl" changed to ".sug". 5455 */ 5456 static void 5457 spell_make_sugfile(spellinfo_T *spin, char_u *wfname) 5458 { 5459 char_u *fname = NULL; 5460 int len; 5461 slang_T *slang; 5462 int free_slang = FALSE; 5463 5464 /* 5465 * Read back the .spl file that was written. This fills the required 5466 * info for soundfolding. This also uses less memory than the 5467 * pointer-linked version of the trie. And it avoids having two versions 5468 * of the code for the soundfolding stuff. 5469 * It might have been done already by spell_reload_one(). 5470 */ 5471 FOR_ALL_SPELL_LANGS(slang) 5472 if (fullpathcmp(wfname, slang->sl_fname, FALSE, TRUE) == FPC_SAME) 5473 break; 5474 if (slang == NULL) 5475 { 5476 spell_message(spin, (char_u *)_("Reading back spell file...")); 5477 slang = spell_load_file(wfname, NULL, NULL, FALSE); 5478 if (slang == NULL) 5479 return; 5480 free_slang = TRUE; 5481 } 5482 5483 /* 5484 * Clear the info in "spin" that is used. 5485 */ 5486 spin->si_blocks = NULL; 5487 spin->si_blocks_cnt = 0; 5488 spin->si_compress_cnt = 0; // will stay at 0 all the time 5489 spin->si_free_count = 0; 5490 spin->si_first_free = NULL; 5491 spin->si_foldwcount = 0; 5492 5493 /* 5494 * Go through the trie of good words, soundfold each word and add it to 5495 * the soundfold trie. 5496 */ 5497 spell_message(spin, (char_u *)_("Performing soundfolding...")); 5498 if (sug_filltree(spin, slang) == FAIL) 5499 goto theend; 5500 5501 /* 5502 * Create the table which links each soundfold word with a list of the 5503 * good words it may come from. Creates buffer "spin->si_spellbuf". 5504 * This also removes the wordnr from the NUL byte entries to make 5505 * compression possible. 5506 */ 5507 if (sug_maketable(spin) == FAIL) 5508 goto theend; 5509 5510 smsg(_("Number of words after soundfolding: %ld"), 5511 (long)spin->si_spellbuf->b_ml.ml_line_count); 5512 5513 /* 5514 * Compress the soundfold trie. 5515 */ 5516 spell_message(spin, (char_u *)_(msg_compressing)); 5517 wordtree_compress(spin, spin->si_foldroot, "case-folded"); 5518 5519 /* 5520 * Write the .sug file. 5521 * Make the file name by changing ".spl" to ".sug". 5522 */ 5523 fname = alloc(MAXPATHL); 5524 if (fname == NULL) 5525 goto theend; 5526 vim_strncpy(fname, wfname, MAXPATHL - 1); 5527 len = (int)STRLEN(fname); 5528 fname[len - 2] = 'u'; 5529 fname[len - 1] = 'g'; 5530 sug_write(spin, fname); 5531 5532 theend: 5533 vim_free(fname); 5534 if (free_slang) 5535 slang_free(slang); 5536 free_blocks(spin->si_blocks); 5537 close_spellbuf(spin->si_spellbuf); 5538 } 5539 5540 /* 5541 * Build the soundfold trie for language "slang". 5542 */ 5543 static int 5544 sug_filltree(spellinfo_T *spin, slang_T *slang) 5545 { 5546 char_u *byts; 5547 idx_T *idxs; 5548 int depth; 5549 idx_T arridx[MAXWLEN]; 5550 int curi[MAXWLEN]; 5551 char_u tword[MAXWLEN]; 5552 char_u tsalword[MAXWLEN]; 5553 int c; 5554 idx_T n; 5555 unsigned words_done = 0; 5556 int wordcount[MAXWLEN]; 5557 5558 // We use si_foldroot for the soundfolded trie. 5559 spin->si_foldroot = wordtree_alloc(spin); 5560 if (spin->si_foldroot == NULL) 5561 return FAIL; 5562 5563 // let tree_add_word() know we're adding to the soundfolded tree 5564 spin->si_sugtree = TRUE; 5565 5566 /* 5567 * Go through the whole case-folded tree, soundfold each word and put it 5568 * in the trie. 5569 */ 5570 byts = slang->sl_fbyts; 5571 idxs = slang->sl_fidxs; 5572 5573 arridx[0] = 0; 5574 curi[0] = 1; 5575 wordcount[0] = 0; 5576 5577 depth = 0; 5578 while (depth >= 0 && !got_int) 5579 { 5580 if (curi[depth] > byts[arridx[depth]]) 5581 { 5582 // Done all bytes at this node, go up one level. 5583 idxs[arridx[depth]] = wordcount[depth]; 5584 if (depth > 0) 5585 wordcount[depth - 1] += wordcount[depth]; 5586 5587 --depth; 5588 line_breakcheck(); 5589 } 5590 else 5591 { 5592 5593 // Do one more byte at this node. 5594 n = arridx[depth] + curi[depth]; 5595 ++curi[depth]; 5596 5597 c = byts[n]; 5598 if (c == 0) 5599 { 5600 // Sound-fold the word. 5601 tword[depth] = NUL; 5602 spell_soundfold(slang, tword, TRUE, tsalword); 5603 5604 // We use the "flags" field for the MSB of the wordnr, 5605 // "region" for the LSB of the wordnr. 5606 if (tree_add_word(spin, tsalword, spin->si_foldroot, 5607 words_done >> 16, words_done & 0xffff, 5608 0) == FAIL) 5609 return FAIL; 5610 5611 ++words_done; 5612 ++wordcount[depth]; 5613 5614 // Reset the block count each time to avoid compression 5615 // kicking in. 5616 spin->si_blocks_cnt = 0; 5617 5618 // Skip over any other NUL bytes (same word with different 5619 // flags). But don't go over the end. 5620 while (n + 1 < slang->sl_fbyts_len && byts[n + 1] == 0) 5621 { 5622 ++n; 5623 ++curi[depth]; 5624 } 5625 } 5626 else 5627 { 5628 // Normal char, go one level deeper. 5629 tword[depth++] = c; 5630 arridx[depth] = idxs[n]; 5631 curi[depth] = 1; 5632 wordcount[depth] = 0; 5633 } 5634 } 5635 } 5636 5637 smsg(_("Total number of words: %d"), words_done); 5638 5639 return OK; 5640 } 5641 5642 /* 5643 * Make the table that links each word in the soundfold trie to the words it 5644 * can be produced from. 5645 * This is not unlike lines in a file, thus use a memfile to be able to access 5646 * the table efficiently. 5647 * Returns FAIL when out of memory. 5648 */ 5649 static int 5650 sug_maketable(spellinfo_T *spin) 5651 { 5652 garray_T ga; 5653 int res = OK; 5654 5655 // Allocate a buffer, open a memline for it and create the swap file 5656 // (uses a temp file, not a .swp file). 5657 spin->si_spellbuf = open_spellbuf(); 5658 if (spin->si_spellbuf == NULL) 5659 return FAIL; 5660 5661 // Use a buffer to store the line info, avoids allocating many small 5662 // pieces of memory. 5663 ga_init2(&ga, 1, 100); 5664 5665 // recursively go through the tree 5666 if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1) 5667 res = FAIL; 5668 5669 ga_clear(&ga); 5670 return res; 5671 } 5672 5673 /* 5674 * Fill the table for one node and its children. 5675 * Returns the wordnr at the start of the node. 5676 * Returns -1 when out of memory. 5677 */ 5678 static int 5679 sug_filltable( 5680 spellinfo_T *spin, 5681 wordnode_T *node, 5682 int startwordnr, 5683 garray_T *gap) // place to store line of numbers 5684 { 5685 wordnode_T *p, *np; 5686 int wordnr = startwordnr; 5687 int nr; 5688 int prev_nr; 5689 5690 FOR_ALL_NODE_SIBLINGS(node, p) 5691 { 5692 if (p->wn_byte == NUL) 5693 { 5694 gap->ga_len = 0; 5695 prev_nr = 0; 5696 for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling) 5697 { 5698 if (ga_grow(gap, 10) == FAIL) 5699 return -1; 5700 5701 nr = (np->wn_flags << 16) + (np->wn_region & 0xffff); 5702 // Compute the offset from the previous nr and store the 5703 // offset in a way that it takes a minimum number of bytes. 5704 // It's a bit like utf-8, but without the need to mark 5705 // following bytes. 5706 nr -= prev_nr; 5707 prev_nr += nr; 5708 gap->ga_len += offset2bytes(nr, 5709 (char_u *)gap->ga_data + gap->ga_len); 5710 } 5711 5712 // add the NUL byte 5713 ((char_u *)gap->ga_data)[gap->ga_len++] = NUL; 5714 5715 if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr, 5716 gap->ga_data, gap->ga_len, TRUE) == FAIL) 5717 return -1; 5718 ++wordnr; 5719 5720 // Remove extra NUL entries, we no longer need them. We don't 5721 // bother freeing the nodes, the won't be reused anyway. 5722 while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL) 5723 p->wn_sibling = p->wn_sibling->wn_sibling; 5724 5725 // Clear the flags on the remaining NUL node, so that compression 5726 // works a lot better. 5727 p->wn_flags = 0; 5728 p->wn_region = 0; 5729 } 5730 else 5731 { 5732 wordnr = sug_filltable(spin, p->wn_child, wordnr, gap); 5733 if (wordnr == -1) 5734 return -1; 5735 } 5736 } 5737 return wordnr; 5738 } 5739 5740 /* 5741 * Convert an offset into a minimal number of bytes. 5742 * Similar to utf_char2byters, but use 8 bits in followup bytes and avoid NUL 5743 * bytes. 5744 */ 5745 static int 5746 offset2bytes(int nr, char_u *buf) 5747 { 5748 int rem; 5749 int b1, b2, b3, b4; 5750 5751 // Split the number in parts of base 255. We need to avoid NUL bytes. 5752 b1 = nr % 255 + 1; 5753 rem = nr / 255; 5754 b2 = rem % 255 + 1; 5755 rem = rem / 255; 5756 b3 = rem % 255 + 1; 5757 b4 = rem / 255 + 1; 5758 5759 if (b4 > 1 || b3 > 0x1f) // 4 bytes 5760 { 5761 buf[0] = 0xe0 + b4; 5762 buf[1] = b3; 5763 buf[2] = b2; 5764 buf[3] = b1; 5765 return 4; 5766 } 5767 if (b3 > 1 || b2 > 0x3f ) // 3 bytes 5768 { 5769 buf[0] = 0xc0 + b3; 5770 buf[1] = b2; 5771 buf[2] = b1; 5772 return 3; 5773 } 5774 if (b2 > 1 || b1 > 0x7f ) // 2 bytes 5775 { 5776 buf[0] = 0x80 + b2; 5777 buf[1] = b1; 5778 return 2; 5779 } 5780 // 1 byte 5781 buf[0] = b1; 5782 return 1; 5783 } 5784 5785 /* 5786 * Write the .sug file in "fname". 5787 */ 5788 static void 5789 sug_write(spellinfo_T *spin, char_u *fname) 5790 { 5791 FILE *fd; 5792 wordnode_T *tree; 5793 int nodecount; 5794 int wcount; 5795 char_u *line; 5796 linenr_T lnum; 5797 int len; 5798 5799 // Create the file. Note that an existing file is silently overwritten! 5800 fd = mch_fopen((char *)fname, "w"); 5801 if (fd == NULL) 5802 { 5803 semsg(_(e_notopen), fname); 5804 return; 5805 } 5806 5807 vim_snprintf((char *)IObuff, IOSIZE, 5808 _("Writing suggestion file %s..."), fname); 5809 spell_message(spin, IObuff); 5810 5811 /* 5812 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 5813 */ 5814 if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) // <fileID> 5815 { 5816 emsg(_(e_write)); 5817 goto theend; 5818 } 5819 putc(VIMSUGVERSION, fd); // <versionnr> 5820 5821 // Write si_sugtime to the file. 5822 put_time(fd, spin->si_sugtime); // <timestamp> 5823 5824 /* 5825 * <SUGWORDTREE> 5826 */ 5827 spin->si_memtot = 0; 5828 tree = spin->si_foldroot->wn_sibling; 5829 5830 // Clear the index and wnode fields in the tree. 5831 clear_node(tree); 5832 5833 // Count the number of nodes. Needed to be able to allocate the 5834 // memory when reading the nodes. Also fills in index for shared 5835 // nodes. 5836 nodecount = put_node(NULL, tree, 0, 0, FALSE); 5837 5838 // number of nodes in 4 bytes 5839 put_bytes(fd, (long_u)nodecount, 4); // <nodecount> 5840 spin->si_memtot += nodecount + nodecount * sizeof(int); 5841 5842 // Write the nodes. 5843 (void)put_node(fd, tree, 0, 0, FALSE); 5844 5845 /* 5846 * <SUGTABLE>: <sugwcount> <sugline> ... 5847 */ 5848 wcount = spin->si_spellbuf->b_ml.ml_line_count; 5849 put_bytes(fd, (long_u)wcount, 4); // <sugwcount> 5850 5851 for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum) 5852 { 5853 // <sugline>: <sugnr> ... NUL 5854 line = ml_get_buf(spin->si_spellbuf, lnum, FALSE); 5855 len = (int)STRLEN(line) + 1; 5856 if (fwrite(line, (size_t)len, (size_t)1, fd) == 0) 5857 { 5858 emsg(_(e_write)); 5859 goto theend; 5860 } 5861 spin->si_memtot += len; 5862 } 5863 5864 // Write another byte to check for errors. 5865 if (putc(0, fd) == EOF) 5866 emsg(_(e_write)); 5867 5868 vim_snprintf((char *)IObuff, IOSIZE, 5869 _("Estimated runtime memory use: %d bytes"), spin->si_memtot); 5870 spell_message(spin, IObuff); 5871 5872 theend: 5873 // close the file 5874 fclose(fd); 5875 } 5876 5877 5878 /* 5879 * Create a Vim spell file from one or more word lists. 5880 * "fnames[0]" is the output file name. 5881 * "fnames[fcount - 1]" is the last input file name. 5882 * Exception: when "fnames[0]" ends in ".add" it's used as the input file name 5883 * and ".spl" is appended to make the output file name. 5884 */ 5885 void 5886 mkspell( 5887 int fcount, 5888 char_u **fnames, 5889 int ascii, // -ascii argument given 5890 int over_write, // overwrite existing output file 5891 int added_word) // invoked through "zg" 5892 { 5893 char_u *fname = NULL; 5894 char_u *wfname; 5895 char_u **innames; 5896 int incount; 5897 afffile_T *(afile[MAXREGIONS]); 5898 int i; 5899 int len; 5900 stat_T st; 5901 int error = FALSE; 5902 spellinfo_T spin; 5903 5904 CLEAR_FIELD(spin); 5905 spin.si_verbose = !added_word; 5906 spin.si_ascii = ascii; 5907 spin.si_followup = TRUE; 5908 spin.si_rem_accents = TRUE; 5909 ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20); 5910 ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20); 5911 ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20); 5912 ga_init2(&spin.si_map, (int)sizeof(char_u), 100); 5913 ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20); 5914 ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50); 5915 hash_init(&spin.si_commonwords); 5916 spin.si_newcompID = 127; // start compound ID at first maximum 5917 5918 // default: fnames[0] is output file, following are input files 5919 // When "fcount" is 1 there is only one file. 5920 innames = &fnames[fcount == 1 ? 0 : 1]; 5921 incount = fcount - 1; 5922 5923 wfname = alloc(MAXPATHL); 5924 if (wfname == NULL) 5925 return; 5926 5927 if (fcount >= 1) 5928 { 5929 len = (int)STRLEN(fnames[0]); 5930 if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0) 5931 { 5932 // For ":mkspell path/en.latin1.add" output file is 5933 // "path/en.latin1.add.spl". 5934 incount = 1; 5935 vim_snprintf((char *)wfname, MAXPATHL, "%s.spl", fnames[0]); 5936 } 5937 else if (fcount == 1) 5938 { 5939 // For ":mkspell path/vim" output file is "path/vim.latin1.spl". 5940 incount = 1; 5941 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL, 5942 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 5943 } 5944 else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0) 5945 { 5946 // Name ends in ".spl", use as the file name. 5947 vim_strncpy(wfname, fnames[0], MAXPATHL - 1); 5948 } 5949 else 5950 // Name should be language, make the file name from it. 5951 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL, 5952 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 5953 5954 // Check for .ascii.spl. 5955 if (strstr((char *)gettail(wfname), SPL_FNAME_ASCII) != NULL) 5956 spin.si_ascii = TRUE; 5957 5958 // Check for .add.spl. 5959 if (strstr((char *)gettail(wfname), SPL_FNAME_ADD) != NULL) 5960 spin.si_add = TRUE; 5961 } 5962 5963 if (incount <= 0) 5964 emsg(_(e_invarg)); // need at least output and input names 5965 else if (vim_strchr(gettail(wfname), '_') != NULL) 5966 emsg(_("E751: Output file name must not have region name")); 5967 else if (incount > MAXREGIONS) 5968 semsg(_("E754: Only up to %d regions supported"), MAXREGIONS); 5969 else 5970 { 5971 // Check for overwriting before doing things that may take a lot of 5972 // time. 5973 if (!over_write && mch_stat((char *)wfname, &st) >= 0) 5974 { 5975 emsg(_(e_exists)); 5976 goto theend; 5977 } 5978 if (mch_isdir(wfname)) 5979 { 5980 semsg(_(e_isadir2), wfname); 5981 goto theend; 5982 } 5983 5984 fname = alloc(MAXPATHL); 5985 if (fname == NULL) 5986 goto theend; 5987 5988 /* 5989 * Init the aff and dic pointers. 5990 * Get the region names if there are more than 2 arguments. 5991 */ 5992 for (i = 0; i < incount; ++i) 5993 { 5994 afile[i] = NULL; 5995 5996 if (incount > 1) 5997 { 5998 len = (int)STRLEN(innames[i]); 5999 if (STRLEN(gettail(innames[i])) < 5 6000 || innames[i][len - 3] != '_') 6001 { 6002 semsg(_("E755: Invalid region in %s"), innames[i]); 6003 goto theend; 6004 } 6005 spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]); 6006 spin.si_region_name[i * 2 + 1] = 6007 TOLOWER_ASC(innames[i][len - 1]); 6008 } 6009 } 6010 spin.si_region_count = incount; 6011 6012 spin.si_foldroot = wordtree_alloc(&spin); 6013 spin.si_keeproot = wordtree_alloc(&spin); 6014 spin.si_prefroot = wordtree_alloc(&spin); 6015 if (spin.si_foldroot == NULL 6016 || spin.si_keeproot == NULL 6017 || spin.si_prefroot == NULL) 6018 { 6019 free_blocks(spin.si_blocks); 6020 goto theend; 6021 } 6022 6023 // When not producing a .add.spl file clear the character table when 6024 // we encounter one in the .aff file. This means we dump the current 6025 // one in the .spl file if the .aff file doesn't define one. That's 6026 // better than guessing the contents, the table will match a 6027 // previously loaded spell file. 6028 if (!spin.si_add) 6029 spin.si_clear_chartab = TRUE; 6030 6031 /* 6032 * Read all the .aff and .dic files. 6033 * Text is converted to 'encoding'. 6034 * Words are stored in the case-folded and keep-case trees. 6035 */ 6036 for (i = 0; i < incount && !error; ++i) 6037 { 6038 spin.si_conv.vc_type = CONV_NONE; 6039 spin.si_region = 1 << i; 6040 6041 vim_snprintf((char *)fname, MAXPATHL, "%s.aff", innames[i]); 6042 if (mch_stat((char *)fname, &st) >= 0) 6043 { 6044 // Read the .aff file. Will init "spin->si_conv" based on the 6045 // "SET" line. 6046 afile[i] = spell_read_aff(&spin, fname); 6047 if (afile[i] == NULL) 6048 error = TRUE; 6049 else 6050 { 6051 // Read the .dic file and store the words in the trees. 6052 vim_snprintf((char *)fname, MAXPATHL, "%s.dic", 6053 innames[i]); 6054 if (spell_read_dic(&spin, fname, afile[i]) == FAIL) 6055 error = TRUE; 6056 } 6057 } 6058 else 6059 { 6060 // No .aff file, try reading the file as a word list. Store 6061 // the words in the trees. 6062 if (spell_read_wordfile(&spin, innames[i]) == FAIL) 6063 error = TRUE; 6064 } 6065 6066 // Free any conversion stuff. 6067 convert_setup(&spin.si_conv, NULL, NULL); 6068 } 6069 6070 if (spin.si_compflags != NULL && spin.si_nobreak) 6071 msg(_("Warning: both compounding and NOBREAK specified")); 6072 6073 if (!error && !got_int) 6074 { 6075 /* 6076 * Combine tails in the tree. 6077 */ 6078 spell_message(&spin, (char_u *)_(msg_compressing)); 6079 wordtree_compress(&spin, spin.si_foldroot, "case-folded"); 6080 wordtree_compress(&spin, spin.si_keeproot, "keep-case"); 6081 wordtree_compress(&spin, spin.si_prefroot, "prefixes"); 6082 } 6083 6084 if (!error && !got_int) 6085 { 6086 /* 6087 * Write the info in the spell file. 6088 */ 6089 vim_snprintf((char *)IObuff, IOSIZE, 6090 _("Writing spell file %s..."), wfname); 6091 spell_message(&spin, IObuff); 6092 6093 error = write_vim_spell(&spin, wfname) == FAIL; 6094 6095 spell_message(&spin, (char_u *)_("Done!")); 6096 vim_snprintf((char *)IObuff, IOSIZE, 6097 _("Estimated runtime memory use: %d bytes"), spin.si_memtot); 6098 spell_message(&spin, IObuff); 6099 6100 /* 6101 * If the file is loaded need to reload it. 6102 */ 6103 if (!error) 6104 spell_reload_one(wfname, added_word); 6105 } 6106 6107 // Free the allocated memory. 6108 ga_clear(&spin.si_rep); 6109 ga_clear(&spin.si_repsal); 6110 ga_clear(&spin.si_sal); 6111 ga_clear(&spin.si_map); 6112 ga_clear(&spin.si_comppat); 6113 ga_clear(&spin.si_prefcond); 6114 hash_clear_all(&spin.si_commonwords, 0); 6115 6116 // Free the .aff file structures. 6117 for (i = 0; i < incount; ++i) 6118 if (afile[i] != NULL) 6119 spell_free_aff(afile[i]); 6120 6121 // Free all the bits and pieces at once. 6122 free_blocks(spin.si_blocks); 6123 6124 /* 6125 * If there is soundfolding info and no NOSUGFILE item create the 6126 * .sug file with the soundfolded word trie. 6127 */ 6128 if (spin.si_sugtime != 0 && !error && !got_int) 6129 spell_make_sugfile(&spin, wfname); 6130 6131 } 6132 6133 theend: 6134 vim_free(fname); 6135 vim_free(wfname); 6136 } 6137 6138 /* 6139 * Display a message for spell file processing when 'verbose' is set or using 6140 * ":mkspell". "str" can be IObuff. 6141 */ 6142 static void 6143 spell_message(spellinfo_T *spin, char_u *str) 6144 { 6145 if (spin->si_verbose || p_verbose > 2) 6146 { 6147 if (!spin->si_verbose) 6148 verbose_enter(); 6149 msg((char *)str); 6150 out_flush(); 6151 if (!spin->si_verbose) 6152 verbose_leave(); 6153 } 6154 } 6155 6156 /* 6157 * ":[count]spellgood {word}" 6158 * ":[count]spellwrong {word}" 6159 * ":[count]spellundo {word}" 6160 * ":[count]spellrare {word}" 6161 */ 6162 void 6163 ex_spell(exarg_T *eap) 6164 { 6165 spell_add_word(eap->arg, (int)STRLEN(eap->arg), 6166 eap->cmdidx == CMD_spellwrong ? SPELL_ADD_BAD : 6167 eap->cmdidx == CMD_spellrare ? SPELL_ADD_RARE : SPELL_ADD_GOOD, 6168 eap->forceit ? 0 : (int)eap->line2, 6169 eap->cmdidx == CMD_spellundo); 6170 } 6171 6172 /* 6173 * Add "word[len]" to 'spellfile' as a good, rare or bad word. 6174 */ 6175 void 6176 spell_add_word( 6177 char_u *word, 6178 int len, 6179 int what, // SPELL_ADD_ values 6180 int idx, // "zG" and "zW": zero, otherwise index in 6181 // 'spellfile' 6182 int undo) // TRUE for "zug", "zuG", "zuw" and "zuW" 6183 { 6184 FILE *fd = NULL; 6185 buf_T *buf = NULL; 6186 int new_spf = FALSE; 6187 char_u *fname; 6188 char_u *fnamebuf = NULL; 6189 char_u line[MAXWLEN * 2]; 6190 long fpos, fpos_next = 0; 6191 int i; 6192 char_u *spf; 6193 6194 if (idx == 0) // use internal wordlist 6195 { 6196 if (int_wordlist == NULL) 6197 { 6198 int_wordlist = vim_tempname('s', FALSE); 6199 if (int_wordlist == NULL) 6200 return; 6201 } 6202 fname = int_wordlist; 6203 } 6204 else 6205 { 6206 // If 'spellfile' isn't set figure out a good default value. 6207 if (*curwin->w_s->b_p_spf == NUL) 6208 { 6209 init_spellfile(); 6210 new_spf = TRUE; 6211 } 6212 6213 if (*curwin->w_s->b_p_spf == NUL) 6214 { 6215 semsg(_(e_notset), "spellfile"); 6216 return; 6217 } 6218 fnamebuf = alloc(MAXPATHL); 6219 if (fnamebuf == NULL) 6220 return; 6221 6222 for (spf = curwin->w_s->b_p_spf, i = 1; *spf != NUL; ++i) 6223 { 6224 copy_option_part(&spf, fnamebuf, MAXPATHL, ","); 6225 if (i == idx) 6226 break; 6227 if (*spf == NUL) 6228 { 6229 semsg(_("E765: 'spellfile' does not have %d entries"), idx); 6230 vim_free(fnamebuf); 6231 return; 6232 } 6233 } 6234 6235 // Check that the user isn't editing the .add file somewhere. 6236 buf = buflist_findname_exp(fnamebuf); 6237 if (buf != NULL && buf->b_ml.ml_mfp == NULL) 6238 buf = NULL; 6239 if (buf != NULL && bufIsChanged(buf)) 6240 { 6241 emsg(_(e_bufloaded)); 6242 vim_free(fnamebuf); 6243 return; 6244 } 6245 6246 fname = fnamebuf; 6247 } 6248 6249 if (what == SPELL_ADD_BAD || undo) 6250 { 6251 // When the word appears as good word we need to remove that one, 6252 // since its flags sort before the one with WF_BANNED. 6253 fd = mch_fopen((char *)fname, "r"); 6254 if (fd != NULL) 6255 { 6256 while (!vim_fgets(line, MAXWLEN * 2, fd)) 6257 { 6258 fpos = fpos_next; 6259 fpos_next = ftell(fd); 6260 if (STRNCMP(word, line, len) == 0 6261 && (line[len] == '/' || line[len] < ' ')) 6262 { 6263 // Found duplicate word. Remove it by writing a '#' at 6264 // the start of the line. Mixing reading and writing 6265 // doesn't work for all systems, close the file first. 6266 fclose(fd); 6267 fd = mch_fopen((char *)fname, "r+"); 6268 if (fd == NULL) 6269 break; 6270 if (fseek(fd, fpos, SEEK_SET) == 0) 6271 { 6272 fputc('#', fd); 6273 if (undo) 6274 { 6275 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 6276 smsg(_("Word '%.*s' removed from %s"), 6277 len, word, NameBuff); 6278 } 6279 } 6280 fseek(fd, fpos_next, SEEK_SET); 6281 } 6282 } 6283 if (fd != NULL) 6284 fclose(fd); 6285 } 6286 } 6287 6288 if (!undo) 6289 { 6290 fd = mch_fopen((char *)fname, "a"); 6291 if (fd == NULL && new_spf) 6292 { 6293 char_u *p; 6294 6295 // We just initialized the 'spellfile' option and can't open the 6296 // file. We may need to create the "spell" directory first. We 6297 // already checked the runtime directory is writable in 6298 // init_spellfile(). 6299 if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname) 6300 { 6301 int c = *p; 6302 6303 // The directory doesn't exist. Try creating it and opening 6304 // the file again. 6305 *p = NUL; 6306 vim_mkdir(fname, 0755); 6307 *p = c; 6308 fd = mch_fopen((char *)fname, "a"); 6309 } 6310 } 6311 6312 if (fd == NULL) 6313 semsg(_(e_notopen), fname); 6314 else 6315 { 6316 if (what == SPELL_ADD_BAD) 6317 fprintf(fd, "%.*s/!\n", len, word); 6318 else if (what == SPELL_ADD_RARE) 6319 fprintf(fd, "%.*s/?\n", len, word); 6320 else 6321 fprintf(fd, "%.*s\n", len, word); 6322 fclose(fd); 6323 6324 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 6325 smsg(_("Word '%.*s' added to %s"), len, word, NameBuff); 6326 } 6327 } 6328 6329 if (fd != NULL) 6330 { 6331 // Update the .add.spl file. 6332 mkspell(1, &fname, FALSE, TRUE, TRUE); 6333 6334 // If the .add file is edited somewhere, reload it. 6335 if (buf != NULL) 6336 buf_reload(buf, buf->b_orig_mode); 6337 6338 redraw_all_later(SOME_VALID); 6339 } 6340 vim_free(fnamebuf); 6341 } 6342 6343 /* 6344 * Initialize 'spellfile' for the current buffer. 6345 */ 6346 static void 6347 init_spellfile(void) 6348 { 6349 char_u *buf; 6350 int l; 6351 char_u *fname; 6352 char_u *rtp; 6353 char_u *lend; 6354 int aspath = FALSE; 6355 char_u *lstart = curbuf->b_s.b_p_spl; 6356 6357 if (*curwin->w_s->b_p_spl != NUL && curwin->w_s->b_langp.ga_len > 0) 6358 { 6359 buf = alloc(MAXPATHL); 6360 if (buf == NULL) 6361 return; 6362 6363 // Find the end of the language name. Exclude the region. If there 6364 // is a path separator remember the start of the tail. 6365 for (lend = curwin->w_s->b_p_spl; *lend != NUL 6366 && vim_strchr((char_u *)",._", *lend) == NULL; ++lend) 6367 if (vim_ispathsep(*lend)) 6368 { 6369 aspath = TRUE; 6370 lstart = lend + 1; 6371 } 6372 6373 // Loop over all entries in 'runtimepath'. Use the first one where we 6374 // are allowed to write. 6375 rtp = p_rtp; 6376 while (*rtp != NUL) 6377 { 6378 if (aspath) 6379 // Use directory of an entry with path, e.g., for 6380 // "/dir/lg.utf-8.spl" use "/dir". 6381 vim_strncpy(buf, curbuf->b_s.b_p_spl, 6382 lstart - curbuf->b_s.b_p_spl - 1); 6383 else 6384 // Copy the path from 'runtimepath' to buf[]. 6385 copy_option_part(&rtp, buf, MAXPATHL, ","); 6386 if (filewritable(buf) == 2) 6387 { 6388 // Use the first language name from 'spelllang' and the 6389 // encoding used in the first loaded .spl file. 6390 if (aspath) 6391 vim_strncpy(buf, curbuf->b_s.b_p_spl, 6392 lend - curbuf->b_s.b_p_spl); 6393 else 6394 { 6395 // Create the "spell" directory if it doesn't exist yet. 6396 l = (int)STRLEN(buf); 6397 vim_snprintf((char *)buf + l, MAXPATHL - l, "/spell"); 6398 if (filewritable(buf) != 2) 6399 vim_mkdir(buf, 0755); 6400 6401 l = (int)STRLEN(buf); 6402 vim_snprintf((char *)buf + l, MAXPATHL - l, 6403 "/%.*s", (int)(lend - lstart), lstart); 6404 } 6405 l = (int)STRLEN(buf); 6406 fname = LANGP_ENTRY(curwin->w_s->b_langp, 0) 6407 ->lp_slang->sl_fname; 6408 vim_snprintf((char *)buf + l, MAXPATHL - l, ".%s.add", 6409 fname != NULL 6410 && strstr((char *)gettail(fname), ".ascii.") != NULL 6411 ? (char_u *)"ascii" : spell_enc()); 6412 set_option_value((char_u *)"spellfile", 0L, buf, OPT_LOCAL); 6413 break; 6414 } 6415 aspath = FALSE; 6416 } 6417 6418 vim_free(buf); 6419 } 6420 } 6421 6422 6423 6424 /* 6425 * Set the spell character tables from strings in the affix file. 6426 */ 6427 static int 6428 set_spell_chartab(char_u *fol, char_u *low, char_u *upp) 6429 { 6430 // We build the new tables here first, so that we can compare with the 6431 // previous one. 6432 spelltab_T new_st; 6433 char_u *pf = fol, *pl = low, *pu = upp; 6434 int f, l, u; 6435 6436 clear_spell_chartab(&new_st); 6437 6438 while (*pf != NUL) 6439 { 6440 if (*pl == NUL || *pu == NUL) 6441 { 6442 emsg(_(e_affform)); 6443 return FAIL; 6444 } 6445 f = mb_ptr2char_adv(&pf); 6446 l = mb_ptr2char_adv(&pl); 6447 u = mb_ptr2char_adv(&pu); 6448 6449 // Every character that appears is a word character. 6450 if (f < 256) 6451 new_st.st_isw[f] = TRUE; 6452 if (l < 256) 6453 new_st.st_isw[l] = TRUE; 6454 if (u < 256) 6455 new_st.st_isw[u] = TRUE; 6456 6457 // if "LOW" and "FOL" are not the same the "LOW" char needs 6458 // case-folding 6459 if (l < 256 && l != f) 6460 { 6461 if (f >= 256) 6462 { 6463 emsg(_(e_affrange)); 6464 return FAIL; 6465 } 6466 new_st.st_fold[l] = f; 6467 } 6468 6469 // if "UPP" and "FOL" are not the same the "UPP" char needs 6470 // case-folding, it's upper case and the "UPP" is the upper case of 6471 // "FOL" . 6472 if (u < 256 && u != f) 6473 { 6474 if (f >= 256) 6475 { 6476 emsg(_(e_affrange)); 6477 return FAIL; 6478 } 6479 new_st.st_fold[u] = f; 6480 new_st.st_isu[u] = TRUE; 6481 new_st.st_upper[f] = u; 6482 } 6483 } 6484 6485 if (*pl != NUL || *pu != NUL) 6486 { 6487 emsg(_(e_affform)); 6488 return FAIL; 6489 } 6490 6491 return set_spell_finish(&new_st); 6492 } 6493 6494 /* 6495 * Set the spell character tables from strings in the .spl file. 6496 */ 6497 static void 6498 set_spell_charflags( 6499 char_u *flags, 6500 int cnt, // length of "flags" 6501 char_u *fol) 6502 { 6503 // We build the new tables here first, so that we can compare with the 6504 // previous one. 6505 spelltab_T new_st; 6506 int i; 6507 char_u *p = fol; 6508 int c; 6509 6510 clear_spell_chartab(&new_st); 6511 6512 for (i = 0; i < 128; ++i) 6513 { 6514 if (i < cnt) 6515 { 6516 new_st.st_isw[i + 128] = (flags[i] & CF_WORD) != 0; 6517 new_st.st_isu[i + 128] = (flags[i] & CF_UPPER) != 0; 6518 } 6519 6520 if (*p != NUL) 6521 { 6522 c = mb_ptr2char_adv(&p); 6523 new_st.st_fold[i + 128] = c; 6524 if (i + 128 != c && new_st.st_isu[i + 128] && c < 256) 6525 new_st.st_upper[c] = i + 128; 6526 } 6527 } 6528 6529 (void)set_spell_finish(&new_st); 6530 } 6531 6532 static int 6533 set_spell_finish(spelltab_T *new_st) 6534 { 6535 int i; 6536 6537 if (did_set_spelltab) 6538 { 6539 // check that it's the same table 6540 for (i = 0; i < 256; ++i) 6541 { 6542 if (spelltab.st_isw[i] != new_st->st_isw[i] 6543 || spelltab.st_isu[i] != new_st->st_isu[i] 6544 || spelltab.st_fold[i] != new_st->st_fold[i] 6545 || spelltab.st_upper[i] != new_st->st_upper[i]) 6546 { 6547 emsg(_("E763: Word characters differ between spell files")); 6548 return FAIL; 6549 } 6550 } 6551 } 6552 else 6553 { 6554 // copy the new spelltab into the one being used 6555 spelltab = *new_st; 6556 did_set_spelltab = TRUE; 6557 } 6558 6559 return OK; 6560 } 6561 6562 /* 6563 * Write the table with prefix conditions to the .spl file. 6564 * When "fd" is NULL only count the length of what is written. 6565 */ 6566 static int 6567 write_spell_prefcond(FILE *fd, garray_T *gap) 6568 { 6569 int i; 6570 char_u *p; 6571 int len; 6572 int totlen; 6573 size_t x = 1; // collect return value of fwrite() 6574 6575 if (fd != NULL) 6576 put_bytes(fd, (long_u)gap->ga_len, 2); // <prefcondcnt> 6577 6578 totlen = 2 + gap->ga_len; // length of <prefcondcnt> and <condlen> bytes 6579 6580 for (i = 0; i < gap->ga_len; ++i) 6581 { 6582 // <prefcond> : <condlen> <condstr> 6583 p = ((char_u **)gap->ga_data)[i]; 6584 if (p != NULL) 6585 { 6586 len = (int)STRLEN(p); 6587 if (fd != NULL) 6588 { 6589 fputc(len, fd); 6590 x &= fwrite(p, (size_t)len, (size_t)1, fd); 6591 } 6592 totlen += len; 6593 } 6594 else if (fd != NULL) 6595 fputc(0, fd); 6596 } 6597 6598 return totlen; 6599 } 6600 6601 6602 /* 6603 * Use map string "map" for languages "lp". 6604 */ 6605 static void 6606 set_map_str(slang_T *lp, char_u *map) 6607 { 6608 char_u *p; 6609 int headc = 0; 6610 int c; 6611 int i; 6612 6613 if (*map == NUL) 6614 { 6615 lp->sl_has_map = FALSE; 6616 return; 6617 } 6618 lp->sl_has_map = TRUE; 6619 6620 // Init the array and hash tables empty. 6621 for (i = 0; i < 256; ++i) 6622 lp->sl_map_array[i] = 0; 6623 hash_init(&lp->sl_map_hash); 6624 6625 /* 6626 * The similar characters are stored separated with slashes: 6627 * "aaa/bbb/ccc/". Fill sl_map_array[c] with the character before c and 6628 * before the same slash. For characters above 255 sl_map_hash is used. 6629 */ 6630 for (p = map; *p != NUL; ) 6631 { 6632 c = mb_cptr2char_adv(&p); 6633 if (c == '/') 6634 headc = 0; 6635 else 6636 { 6637 if (headc == 0) 6638 headc = c; 6639 6640 // Characters above 255 don't fit in sl_map_array[], put them in 6641 // the hash table. Each entry is the char, a NUL the headchar and 6642 // a NUL. 6643 if (c >= 256) 6644 { 6645 int cl = mb_char2len(c); 6646 int headcl = mb_char2len(headc); 6647 char_u *b; 6648 hash_T hash; 6649 hashitem_T *hi; 6650 6651 b = alloc(cl + headcl + 2); 6652 if (b == NULL) 6653 return; 6654 mb_char2bytes(c, b); 6655 b[cl] = NUL; 6656 mb_char2bytes(headc, b + cl + 1); 6657 b[cl + 1 + headcl] = NUL; 6658 hash = hash_hash(b); 6659 hi = hash_lookup(&lp->sl_map_hash, b, hash); 6660 if (HASHITEM_EMPTY(hi)) 6661 hash_add_item(&lp->sl_map_hash, hi, b, hash); 6662 else 6663 { 6664 // This should have been checked when generating the .spl 6665 // file. 6666 emsg(_("E783: duplicate char in MAP entry")); 6667 vim_free(b); 6668 } 6669 } 6670 else 6671 lp->sl_map_array[c] = headc; 6672 } 6673 } 6674 } 6675 6676 #endif // FEAT_SPELL 6677