1 /* vi:set ts=8 sts=4 sw=4 noet: 2 * 3 * VIM - Vi IMproved by Bram Moolenaar 4 * 5 * Do ":help uganda" in Vim to read copying and usage conditions. 6 * Do ":help credits" in Vim to see a list of people who contributed. 7 * See README.txt for an overview of the Vim source code. 8 */ 9 10 /* 11 * spellfile.c: code for reading and writing spell files. 12 * 13 * See spell.c for information about spell checking. 14 */ 15 16 /* 17 * Vim spell file format: <HEADER> 18 * <SECTIONS> 19 * <LWORDTREE> 20 * <KWORDTREE> 21 * <PREFIXTREE> 22 * 23 * <HEADER>: <fileID> <versionnr> 24 * 25 * <fileID> 8 bytes "VIMspell" 26 * <versionnr> 1 byte VIMSPELLVERSION 27 * 28 * 29 * Sections make it possible to add information to the .spl file without 30 * making it incompatible with previous versions. There are two kinds of 31 * sections: 32 * 1. Not essential for correct spell checking. E.g. for making suggestions. 33 * These are skipped when not supported. 34 * 2. Optional information, but essential for spell checking when present. 35 * E.g. conditions for affixes. When this section is present but not 36 * supported an error message is given. 37 * 38 * <SECTIONS>: <section> ... <sectionend> 39 * 40 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 41 * 42 * <sectionID> 1 byte number from 0 to 254 identifying the section 43 * 44 * <sectionflags> 1 byte SNF_REQUIRED: this section is required for correct 45 * spell checking 46 * 47 * <sectionlen> 4 bytes length of section contents, MSB first 48 * 49 * <sectionend> 1 byte SN_END 50 * 51 * 52 * sectionID == SN_INFO: <infotext> 53 * <infotext> N bytes free format text with spell file info (version, 54 * website, etc) 55 * 56 * sectionID == SN_REGION: <regionname> ... 57 * <regionname> 2 bytes Up to MAXREGIONS region names: ca, au, etc. Lower 58 * case. First <regionname> is region 1. 59 * 60 * sectionID == SN_CHARFLAGS: <charflagslen> <charflags> 61 * <folcharslen> <folchars> 62 * <charflagslen> 1 byte Number of bytes in <charflags> (should be 128). 63 * <charflags> N bytes List of flags (first one is for character 128): 64 * 0x01 word character CF_WORD 65 * 0x02 upper-case character CF_UPPER 66 * <folcharslen> 2 bytes Number of bytes in <folchars>. 67 * <folchars> N bytes Folded characters, first one is for character 128. 68 * 69 * sectionID == SN_MIDWORD: <midword> 70 * <midword> N bytes Characters that are word characters only when used 71 * in the middle of a word. 72 * 73 * sectionID == SN_PREFCOND: <prefcondcnt> <prefcond> ... 74 * <prefcondcnt> 2 bytes Number of <prefcond> items following. 75 * <prefcond> : <condlen> <condstr> 76 * <condlen> 1 byte Length of <condstr>. 77 * <condstr> N bytes Condition for the prefix. 78 * 79 * sectionID == SN_REP: <repcount> <rep> ... 80 * <repcount> 2 bytes number of <rep> items, MSB first. 81 * <rep> : <repfromlen> <repfrom> <reptolen> <repto> 82 * <repfromlen> 1 byte length of <repfrom> 83 * <repfrom> N bytes "from" part of replacement 84 * <reptolen> 1 byte length of <repto> 85 * <repto> N bytes "to" part of replacement 86 * 87 * sectionID == SN_REPSAL: <repcount> <rep> ... 88 * just like SN_REP but for soundfolded words 89 * 90 * sectionID == SN_SAL: <salflags> <salcount> <sal> ... 91 * <salflags> 1 byte flags for soundsalike conversion: 92 * SAL_F0LLOWUP 93 * SAL_COLLAPSE 94 * SAL_REM_ACCENTS 95 * <salcount> 2 bytes number of <sal> items following 96 * <sal> : <salfromlen> <salfrom> <saltolen> <salto> 97 * <salfromlen> 1 byte length of <salfrom> 98 * <salfrom> N bytes "from" part of soundsalike 99 * <saltolen> 1 byte length of <salto> 100 * <salto> N bytes "to" part of soundsalike 101 * 102 * sectionID == SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 103 * <sofofromlen> 2 bytes length of <sofofrom> 104 * <sofofrom> N bytes "from" part of soundfold 105 * <sofotolen> 2 bytes length of <sofoto> 106 * <sofoto> N bytes "to" part of soundfold 107 * 108 * sectionID == SN_SUGFILE: <timestamp> 109 * <timestamp> 8 bytes time in seconds that must match with .sug file 110 * 111 * sectionID == SN_NOSPLITSUGS: nothing 112 * 113 * sectionID == SN_NOCOMPOUNDSUGS: nothing 114 * 115 * sectionID == SN_WORDS: <word> ... 116 * <word> N bytes NUL terminated common word 117 * 118 * sectionID == SN_MAP: <mapstr> 119 * <mapstr> N bytes String with sequences of similar characters, 120 * separated by slashes. 121 * 122 * sectionID == SN_COMPOUND: <compmax> <compminlen> <compsylmax> <compoptions> 123 * <comppatcount> <comppattern> ... <compflags> 124 * <compmax> 1 byte Maximum nr of words in compound word. 125 * <compminlen> 1 byte Minimal word length for compounding. 126 * <compsylmax> 1 byte Maximum nr of syllables in compound word. 127 * <compoptions> 2 bytes COMP_ flags. 128 * <comppatcount> 2 bytes number of <comppattern> following 129 * <compflags> N bytes Flags from COMPOUNDRULE items, separated by 130 * slashes. 131 * 132 * <comppattern>: <comppatlen> <comppattext> 133 * <comppatlen> 1 byte length of <comppattext> 134 * <comppattext> N bytes end or begin chars from CHECKCOMPOUNDPATTERN 135 * 136 * sectionID == SN_NOBREAK: (empty, its presence is what matters) 137 * 138 * sectionID == SN_SYLLABLE: <syllable> 139 * <syllable> N bytes String from SYLLABLE item. 140 * 141 * <LWORDTREE>: <wordtree> 142 * 143 * <KWORDTREE>: <wordtree> 144 * 145 * <PREFIXTREE>: <wordtree> 146 * 147 * 148 * <wordtree>: <nodecount> <nodedata> ... 149 * 150 * <nodecount> 4 bytes Number of nodes following. MSB first. 151 * 152 * <nodedata>: <siblingcount> <sibling> ... 153 * 154 * <siblingcount> 1 byte Number of siblings in this node. The siblings 155 * follow in sorted order. 156 * 157 * <sibling>: <byte> [ <nodeidx> <xbyte> 158 * | <flags> [<flags2>] [<region>] [<affixID>] 159 * | [<pflags>] <affixID> <prefcondnr> ] 160 * 161 * <byte> 1 byte Byte value of the sibling. Special cases: 162 * BY_NOFLAGS: End of word without flags and for all 163 * regions. 164 * For PREFIXTREE <affixID> and 165 * <prefcondnr> follow. 166 * BY_FLAGS: End of word, <flags> follow. 167 * For PREFIXTREE <pflags>, <affixID> 168 * and <prefcondnr> follow. 169 * BY_FLAGS2: End of word, <flags> and <flags2> 170 * follow. Not used in PREFIXTREE. 171 * BY_INDEX: Child of sibling is shared, <nodeidx> 172 * and <xbyte> follow. 173 * 174 * <nodeidx> 3 bytes Index of child for this sibling, MSB first. 175 * 176 * <xbyte> 1 byte byte value of the sibling. 177 * 178 * <flags> 1 byte bitmask of: 179 * WF_ALLCAP word must have only capitals 180 * WF_ONECAP first char of word must be capital 181 * WF_KEEPCAP keep-case word 182 * WF_FIXCAP keep-case word, all caps not allowed 183 * WF_RARE rare word 184 * WF_BANNED bad word 185 * WF_REGION <region> follows 186 * WF_AFX <affixID> follows 187 * 188 * <flags2> 1 byte Bitmask of: 189 * WF_HAS_AFF >> 8 word includes affix 190 * WF_NEEDCOMP >> 8 word only valid in compound 191 * WF_NOSUGGEST >> 8 word not used for suggestions 192 * WF_COMPROOT >> 8 word already a compound 193 * WF_NOCOMPBEF >> 8 no compounding before this word 194 * WF_NOCOMPAFT >> 8 no compounding after this word 195 * 196 * <pflags> 1 byte bitmask of: 197 * WFP_RARE rare prefix 198 * WFP_NC non-combining prefix 199 * WFP_UP letter after prefix made upper case 200 * 201 * <region> 1 byte Bitmask for regions in which word is valid. When 202 * omitted it's valid in all regions. 203 * Lowest bit is for region 1. 204 * 205 * <affixID> 1 byte ID of affix that can be used with this word. In 206 * PREFIXTREE used for the required prefix ID. 207 * 208 * <prefcondnr> 2 bytes Prefix condition number, index in <prefcond> list 209 * from HEADER. 210 * 211 * All text characters are in 'encoding', but stored as single bytes. 212 */ 213 214 /* 215 * Vim .sug file format: <SUGHEADER> 216 * <SUGWORDTREE> 217 * <SUGTABLE> 218 * 219 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 220 * 221 * <fileID> 6 bytes "VIMsug" 222 * <versionnr> 1 byte VIMSUGVERSION 223 * <timestamp> 8 bytes timestamp that must match with .spl file 224 * 225 * 226 * <SUGWORDTREE>: <wordtree> (see above, no flags or region used) 227 * 228 * 229 * <SUGTABLE>: <sugwcount> <sugline> ... 230 * 231 * <sugwcount> 4 bytes number of <sugline> following 232 * 233 * <sugline>: <sugnr> ... NUL 234 * 235 * <sugnr>: X bytes word number that results in this soundfolded word, 236 * stored as an offset to the previous number in as 237 * few bytes as possible, see offset2bytes()) 238 */ 239 240 #include "vim.h" 241 242 #if defined(FEAT_SPELL) || defined(PROTO) 243 244 #ifndef UNIX // it's in os_unix.h for Unix 245 # include <time.h> // for time_t 246 #endif 247 248 #ifndef UNIX // it's in os_unix.h for Unix 249 # include <time.h> // for time_t 250 #endif 251 252 // Special byte values for <byte>. Some are only used in the tree for 253 // postponed prefixes, some only in the other trees. This is a bit messy... 254 #define BY_NOFLAGS 0 // end of word without flags or region; for 255 // postponed prefix: no <pflags> 256 #define BY_INDEX 1 // child is shared, index follows 257 #define BY_FLAGS 2 // end of word, <flags> byte follows; for 258 // postponed prefix: <pflags> follows 259 #define BY_FLAGS2 3 // end of word, <flags> and <flags2> bytes 260 // follow; never used in prefix tree 261 #define BY_SPECIAL BY_FLAGS2 // highest special byte value 262 263 #define ZERO_FLAG 65009 // used when flag is zero: "0" 264 265 // Flags used in .spl file for soundsalike flags. 266 #define SAL_F0LLOWUP 1 267 #define SAL_COLLAPSE 2 268 #define SAL_REM_ACCENTS 4 269 270 #define VIMSPELLMAGIC "VIMspell" // string at start of Vim spell file 271 #define VIMSPELLMAGICL 8 272 #define VIMSPELLVERSION 50 273 274 // Section IDs. Only renumber them when VIMSPELLVERSION changes! 275 #define SN_REGION 0 // <regionname> section 276 #define SN_CHARFLAGS 1 // charflags section 277 #define SN_MIDWORD 2 // <midword> section 278 #define SN_PREFCOND 3 // <prefcond> section 279 #define SN_REP 4 // REP items section 280 #define SN_SAL 5 // SAL items section 281 #define SN_SOFO 6 // soundfolding section 282 #define SN_MAP 7 // MAP items section 283 #define SN_COMPOUND 8 // compound words section 284 #define SN_SYLLABLE 9 // syllable section 285 #define SN_NOBREAK 10 // NOBREAK section 286 #define SN_SUGFILE 11 // timestamp for .sug file 287 #define SN_REPSAL 12 // REPSAL items section 288 #define SN_WORDS 13 // common words 289 #define SN_NOSPLITSUGS 14 // don't split word for suggestions 290 #define SN_INFO 15 // info section 291 #define SN_NOCOMPOUNDSUGS 16 // don't compound for suggestions 292 #define SN_END 255 // end of sections 293 294 #define SNF_REQUIRED 1 // <sectionflags>: required section 295 296 #define CF_WORD 0x01 297 #define CF_UPPER 0x02 298 299 /* 300 * Loop through all the siblings of a node (including the node) 301 */ 302 #define FOR_ALL_NODE_SIBLINGS(node, np) \ 303 for ((np) = (node); (np) != NULL; (np) = (np)->wn_sibling) 304 305 static int set_spell_finish(spelltab_T *new_st); 306 static int write_spell_prefcond(FILE *fd, garray_T *gap); 307 static int read_region_section(FILE *fd, slang_T *slang, int len); 308 static int read_charflags_section(FILE *fd); 309 static int read_prefcond_section(FILE *fd, slang_T *lp); 310 static int read_rep_section(FILE *fd, garray_T *gap, short *first); 311 static int read_sal_section(FILE *fd, slang_T *slang); 312 static int read_words_section(FILE *fd, slang_T *lp, int len); 313 static int read_sofo_section(FILE *fd, slang_T *slang); 314 static int read_compound(FILE *fd, slang_T *slang, int len); 315 static int set_sofo(slang_T *lp, char_u *from, char_u *to); 316 static void set_sal_first(slang_T *lp); 317 static int *mb_str2wide(char_u *s); 318 static int spell_read_tree(FILE *fd, char_u **bytsp, long *bytsp_len, idx_T **idxsp, int prefixtree, int prefixcnt); 319 static idx_T read_tree_node(FILE *fd, char_u *byts, idx_T *idxs, int maxidx, idx_T startidx, int prefixtree, int maxprefcondnr); 320 static void set_spell_charflags(char_u *flags, int cnt, char_u *upp); 321 static int set_spell_chartab(char_u *fol, char_u *low, char_u *upp); 322 static void set_map_str(slang_T *lp, char_u *map); 323 324 325 static char *e_spell_trunc = N_("E758: Truncated spell file"); 326 static char *e_afftrailing = N_("Trailing text in %s line %d: %s"); 327 static char *e_affname = N_("Affix name too long in %s line %d: %s"); 328 static char *e_affform = N_("E761: Format error in affix file FOL, LOW or UPP"); 329 static char *e_affrange = N_("E762: Character in FOL, LOW or UPP is out of range"); 330 static char *msg_compressing = N_("Compressing word tree..."); 331 332 /* 333 * Load one spell file and store the info into a slang_T. 334 * 335 * This is invoked in three ways: 336 * - From spell_load_cb() to load a spell file for the first time. "lang" is 337 * the language name, "old_lp" is NULL. Will allocate an slang_T. 338 * - To reload a spell file that was changed. "lang" is NULL and "old_lp" 339 * points to the existing slang_T. 340 * - Just after writing a .spl file; it's read back to produce the .sug file. 341 * "old_lp" is NULL and "lang" is NULL. Will allocate an slang_T. 342 * 343 * Returns the slang_T the spell file was loaded into. NULL for error. 344 */ 345 slang_T * 346 spell_load_file( 347 char_u *fname, 348 char_u *lang, 349 slang_T *old_lp, 350 int silent) // no error if file doesn't exist 351 { 352 FILE *fd; 353 char_u buf[VIMSPELLMAGICL]; 354 char_u *p; 355 int i; 356 int n; 357 int len; 358 slang_T *lp = NULL; 359 int c = 0; 360 int res; 361 int did_estack_push = FALSE; 362 ESTACK_CHECK_DECLARATION 363 364 fd = mch_fopen((char *)fname, "r"); 365 if (fd == NULL) 366 { 367 if (!silent) 368 semsg(_(e_notopen), fname); 369 else if (p_verbose > 2) 370 { 371 verbose_enter(); 372 smsg((const char *)e_notopen, fname); 373 verbose_leave(); 374 } 375 goto endFAIL; 376 } 377 if (p_verbose > 2) 378 { 379 verbose_enter(); 380 smsg(_("Reading spell file \"%s\""), fname); 381 verbose_leave(); 382 } 383 384 if (old_lp == NULL) 385 { 386 lp = slang_alloc(lang); 387 if (lp == NULL) 388 goto endFAIL; 389 390 // Remember the file name, used to reload the file when it's updated. 391 lp->sl_fname = vim_strsave(fname); 392 if (lp->sl_fname == NULL) 393 goto endFAIL; 394 395 // Check for .add.spl (_add.spl for VMS). 396 lp->sl_add = strstr((char *)gettail(fname), SPL_FNAME_ADD) != NULL; 397 } 398 else 399 lp = old_lp; 400 401 // Set sourcing_name, so that error messages mention the file name. 402 estack_push(ETYPE_SPELL, fname, 0); 403 ESTACK_CHECK_SETUP 404 did_estack_push = TRUE; 405 406 /* 407 * <HEADER>: <fileID> 408 */ 409 for (i = 0; i < VIMSPELLMAGICL; ++i) 410 buf[i] = getc(fd); // <fileID> 411 if (STRNCMP(buf, VIMSPELLMAGIC, VIMSPELLMAGICL) != 0) 412 { 413 emsg(_("E757: This does not look like a spell file")); 414 goto endFAIL; 415 } 416 c = getc(fd); // <versionnr> 417 if (c < VIMSPELLVERSION) 418 { 419 emsg(_("E771: Old spell file, needs to be updated")); 420 goto endFAIL; 421 } 422 else if (c > VIMSPELLVERSION) 423 { 424 emsg(_("E772: Spell file is for newer version of Vim")); 425 goto endFAIL; 426 } 427 428 429 /* 430 * <SECTIONS>: <section> ... <sectionend> 431 * <section>: <sectionID> <sectionflags> <sectionlen> (section contents) 432 */ 433 for (;;) 434 { 435 n = getc(fd); // <sectionID> or <sectionend> 436 if (n == SN_END) 437 break; 438 c = getc(fd); // <sectionflags> 439 len = get4c(fd); // <sectionlen> 440 if (len < 0) 441 goto truncerr; 442 443 res = 0; 444 switch (n) 445 { 446 case SN_INFO: 447 lp->sl_info = read_string(fd, len); // <infotext> 448 if (lp->sl_info == NULL) 449 goto endFAIL; 450 break; 451 452 case SN_REGION: 453 res = read_region_section(fd, lp, len); 454 break; 455 456 case SN_CHARFLAGS: 457 res = read_charflags_section(fd); 458 break; 459 460 case SN_MIDWORD: 461 lp->sl_midword = read_string(fd, len); // <midword> 462 if (lp->sl_midword == NULL) 463 goto endFAIL; 464 break; 465 466 case SN_PREFCOND: 467 res = read_prefcond_section(fd, lp); 468 break; 469 470 case SN_REP: 471 res = read_rep_section(fd, &lp->sl_rep, lp->sl_rep_first); 472 break; 473 474 case SN_REPSAL: 475 res = read_rep_section(fd, &lp->sl_repsal, lp->sl_repsal_first); 476 break; 477 478 case SN_SAL: 479 res = read_sal_section(fd, lp); 480 break; 481 482 case SN_SOFO: 483 res = read_sofo_section(fd, lp); 484 break; 485 486 case SN_MAP: 487 p = read_string(fd, len); // <mapstr> 488 if (p == NULL) 489 goto endFAIL; 490 set_map_str(lp, p); 491 vim_free(p); 492 break; 493 494 case SN_WORDS: 495 res = read_words_section(fd, lp, len); 496 break; 497 498 case SN_SUGFILE: 499 lp->sl_sugtime = get8ctime(fd); // <timestamp> 500 break; 501 502 case SN_NOSPLITSUGS: 503 lp->sl_nosplitsugs = TRUE; 504 break; 505 506 case SN_NOCOMPOUNDSUGS: 507 lp->sl_nocompoundsugs = TRUE; 508 break; 509 510 case SN_COMPOUND: 511 res = read_compound(fd, lp, len); 512 break; 513 514 case SN_NOBREAK: 515 lp->sl_nobreak = TRUE; 516 break; 517 518 case SN_SYLLABLE: 519 lp->sl_syllable = read_string(fd, len); // <syllable> 520 if (lp->sl_syllable == NULL) 521 goto endFAIL; 522 if (init_syl_tab(lp) != OK) 523 goto endFAIL; 524 break; 525 526 default: 527 // Unsupported section. When it's required give an error 528 // message. When it's not required skip the contents. 529 if (c & SNF_REQUIRED) 530 { 531 emsg(_("E770: Unsupported section in spell file")); 532 goto endFAIL; 533 } 534 while (--len >= 0) 535 if (getc(fd) < 0) 536 goto truncerr; 537 break; 538 } 539 someerror: 540 if (res == SP_FORMERROR) 541 { 542 emsg(_(e_format)); 543 goto endFAIL; 544 } 545 if (res == SP_TRUNCERROR) 546 { 547 truncerr: 548 emsg(_(e_spell_trunc)); 549 goto endFAIL; 550 } 551 if (res == SP_OTHERERROR) 552 goto endFAIL; 553 } 554 555 // <LWORDTREE> 556 res = spell_read_tree(fd, &lp->sl_fbyts, &lp->sl_fbyts_len, 557 &lp->sl_fidxs, FALSE, 0); 558 if (res != 0) 559 goto someerror; 560 561 // <KWORDTREE> 562 res = spell_read_tree(fd, &lp->sl_kbyts, NULL, &lp->sl_kidxs, FALSE, 0); 563 if (res != 0) 564 goto someerror; 565 566 // <PREFIXTREE> 567 res = spell_read_tree(fd, &lp->sl_pbyts, NULL, &lp->sl_pidxs, TRUE, 568 lp->sl_prefixcnt); 569 if (res != 0) 570 goto someerror; 571 572 // For a new file link it in the list of spell files. 573 if (old_lp == NULL && lang != NULL) 574 { 575 lp->sl_next = first_lang; 576 first_lang = lp; 577 } 578 579 goto endOK; 580 581 endFAIL: 582 if (lang != NULL) 583 // truncating the name signals the error to spell_load_lang() 584 *lang = NUL; 585 if (lp != NULL && old_lp == NULL) 586 slang_free(lp); 587 lp = NULL; 588 589 endOK: 590 if (fd != NULL) 591 fclose(fd); 592 if (did_estack_push) 593 { 594 ESTACK_CHECK_NOW 595 estack_pop(); 596 } 597 598 return lp; 599 } 600 601 /* 602 * Fill in the wordcount fields for a trie. 603 * Returns the total number of words. 604 */ 605 static void 606 tree_count_words(char_u *byts, idx_T *idxs) 607 { 608 int depth; 609 idx_T arridx[MAXWLEN]; 610 int curi[MAXWLEN]; 611 int c; 612 idx_T n; 613 int wordcount[MAXWLEN]; 614 615 arridx[0] = 0; 616 curi[0] = 1; 617 wordcount[0] = 0; 618 depth = 0; 619 while (depth >= 0 && !got_int) 620 { 621 if (curi[depth] > byts[arridx[depth]]) 622 { 623 // Done all bytes at this node, go up one level. 624 idxs[arridx[depth]] = wordcount[depth]; 625 if (depth > 0) 626 wordcount[depth - 1] += wordcount[depth]; 627 628 --depth; 629 fast_breakcheck(); 630 } 631 else 632 { 633 // Do one more byte at this node. 634 n = arridx[depth] + curi[depth]; 635 ++curi[depth]; 636 637 c = byts[n]; 638 if (c == 0) 639 { 640 // End of word, count it. 641 ++wordcount[depth]; 642 643 // Skip over any other NUL bytes (same word with different 644 // flags). 645 while (byts[n + 1] == 0) 646 { 647 ++n; 648 ++curi[depth]; 649 } 650 } 651 else 652 { 653 // Normal char, go one level deeper to count the words. 654 ++depth; 655 arridx[depth] = idxs[n]; 656 curi[depth] = 1; 657 wordcount[depth] = 0; 658 } 659 } 660 } 661 } 662 663 /* 664 * Load the .sug files for languages that have one and weren't loaded yet. 665 */ 666 void 667 suggest_load_files(void) 668 { 669 langp_T *lp; 670 int lpi; 671 slang_T *slang; 672 char_u *dotp; 673 FILE *fd; 674 char_u buf[MAXWLEN]; 675 int i; 676 time_t timestamp; 677 int wcount; 678 int wordnr; 679 garray_T ga; 680 int c; 681 682 // Do this for all languages that support sound folding. 683 for (lpi = 0; lpi < curwin->w_s->b_langp.ga_len; ++lpi) 684 { 685 lp = LANGP_ENTRY(curwin->w_s->b_langp, lpi); 686 slang = lp->lp_slang; 687 if (slang->sl_sugtime != 0 && !slang->sl_sugloaded) 688 { 689 // Change ".spl" to ".sug" and open the file. When the file isn't 690 // found silently skip it. Do set "sl_sugloaded" so that we 691 // don't try again and again. 692 slang->sl_sugloaded = TRUE; 693 694 dotp = vim_strrchr(slang->sl_fname, '.'); 695 if (dotp == NULL || fnamecmp(dotp, ".spl") != 0) 696 continue; 697 STRCPY(dotp, ".sug"); 698 fd = mch_fopen((char *)slang->sl_fname, "r"); 699 if (fd == NULL) 700 goto nextone; 701 702 /* 703 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 704 */ 705 for (i = 0; i < VIMSUGMAGICL; ++i) 706 buf[i] = getc(fd); // <fileID> 707 if (STRNCMP(buf, VIMSUGMAGIC, VIMSUGMAGICL) != 0) 708 { 709 semsg(_("E778: This does not look like a .sug file: %s"), 710 slang->sl_fname); 711 goto nextone; 712 } 713 c = getc(fd); // <versionnr> 714 if (c < VIMSUGVERSION) 715 { 716 semsg(_("E779: Old .sug file, needs to be updated: %s"), 717 slang->sl_fname); 718 goto nextone; 719 } 720 else if (c > VIMSUGVERSION) 721 { 722 semsg(_("E780: .sug file is for newer version of Vim: %s"), 723 slang->sl_fname); 724 goto nextone; 725 } 726 727 // Check the timestamp, it must be exactly the same as the one in 728 // the .spl file. Otherwise the word numbers won't match. 729 timestamp = get8ctime(fd); // <timestamp> 730 if (timestamp != slang->sl_sugtime) 731 { 732 semsg(_("E781: .sug file doesn't match .spl file: %s"), 733 slang->sl_fname); 734 goto nextone; 735 } 736 737 /* 738 * <SUGWORDTREE>: <wordtree> 739 * Read the trie with the soundfolded words. 740 */ 741 if (spell_read_tree(fd, &slang->sl_sbyts, NULL, &slang->sl_sidxs, 742 FALSE, 0) != 0) 743 { 744 someerror: 745 semsg(_("E782: error while reading .sug file: %s"), 746 slang->sl_fname); 747 slang_clear_sug(slang); 748 goto nextone; 749 } 750 751 /* 752 * <SUGTABLE>: <sugwcount> <sugline> ... 753 * 754 * Read the table with word numbers. We use a file buffer for 755 * this, because it's so much like a file with lines. Makes it 756 * possible to swap the info and save on memory use. 757 */ 758 slang->sl_sugbuf = open_spellbuf(); 759 if (slang->sl_sugbuf == NULL) 760 goto someerror; 761 // <sugwcount> 762 wcount = get4c(fd); 763 if (wcount < 0) 764 goto someerror; 765 766 // Read all the wordnr lists into the buffer, one NUL terminated 767 // list per line. 768 ga_init2(&ga, 1, 100); 769 for (wordnr = 0; wordnr < wcount; ++wordnr) 770 { 771 ga.ga_len = 0; 772 for (;;) 773 { 774 c = getc(fd); // <sugline> 775 if (c < 0 || ga_grow(&ga, 1) == FAIL) 776 goto someerror; 777 ((char_u *)ga.ga_data)[ga.ga_len++] = c; 778 if (c == NUL) 779 break; 780 } 781 if (ml_append_buf(slang->sl_sugbuf, (linenr_T)wordnr, 782 ga.ga_data, ga.ga_len, TRUE) == FAIL) 783 goto someerror; 784 } 785 ga_clear(&ga); 786 787 /* 788 * Need to put word counts in the word tries, so that we can find 789 * a word by its number. 790 */ 791 tree_count_words(slang->sl_fbyts, slang->sl_fidxs); 792 tree_count_words(slang->sl_sbyts, slang->sl_sidxs); 793 794 nextone: 795 if (fd != NULL) 796 fclose(fd); 797 STRCPY(dotp, ".spl"); 798 } 799 } 800 } 801 802 803 /* 804 * Read a length field from "fd" in "cnt_bytes" bytes. 805 * Allocate memory, read the string into it and add a NUL at the end. 806 * Returns NULL when the count is zero. 807 * Sets "*cntp" to SP_*ERROR when there is an error, length of the result 808 * otherwise. 809 */ 810 static char_u * 811 read_cnt_string(FILE *fd, int cnt_bytes, int *cntp) 812 { 813 int cnt = 0; 814 int i; 815 char_u *str; 816 817 // read the length bytes, MSB first 818 for (i = 0; i < cnt_bytes; ++i) 819 { 820 int c = getc(fd); 821 822 if (c == EOF) 823 { 824 *cntp = SP_TRUNCERROR; 825 return NULL; 826 } 827 cnt = (cnt << 8) + (unsigned)c; 828 } 829 *cntp = cnt; 830 if (cnt == 0) 831 return NULL; // nothing to read, return NULL 832 833 str = read_string(fd, cnt); 834 if (str == NULL) 835 *cntp = SP_OTHERERROR; 836 return str; 837 } 838 839 /* 840 * Read SN_REGION: <regionname> ... 841 * Return SP_*ERROR flags. 842 */ 843 static int 844 read_region_section(FILE *fd, slang_T *lp, int len) 845 { 846 int i; 847 848 if (len > MAXREGIONS * 2) 849 return SP_FORMERROR; 850 for (i = 0; i < len; ++i) 851 lp->sl_regions[i] = getc(fd); // <regionname> 852 lp->sl_regions[len] = NUL; 853 return 0; 854 } 855 856 /* 857 * Read SN_CHARFLAGS section: <charflagslen> <charflags> 858 * <folcharslen> <folchars> 859 * Return SP_*ERROR flags. 860 */ 861 static int 862 read_charflags_section(FILE *fd) 863 { 864 char_u *flags; 865 char_u *fol; 866 int flagslen, follen; 867 868 // <charflagslen> <charflags> 869 flags = read_cnt_string(fd, 1, &flagslen); 870 if (flagslen < 0) 871 return flagslen; 872 873 // <folcharslen> <folchars> 874 fol = read_cnt_string(fd, 2, &follen); 875 if (follen < 0) 876 { 877 vim_free(flags); 878 return follen; 879 } 880 881 // Set the word-char flags and fill SPELL_ISUPPER() table. 882 if (flags != NULL && fol != NULL) 883 set_spell_charflags(flags, flagslen, fol); 884 885 vim_free(flags); 886 vim_free(fol); 887 888 // When <charflagslen> is zero then <fcharlen> must also be zero. 889 if ((flags == NULL) != (fol == NULL)) 890 return SP_FORMERROR; 891 return 0; 892 } 893 894 /* 895 * Read SN_PREFCOND section. 896 * Return SP_*ERROR flags. 897 */ 898 static int 899 read_prefcond_section(FILE *fd, slang_T *lp) 900 { 901 int cnt; 902 int i; 903 int n; 904 char_u *p; 905 char_u buf[MAXWLEN + 1]; 906 907 // <prefcondcnt> <prefcond> ... 908 cnt = get2c(fd); // <prefcondcnt> 909 if (cnt <= 0) 910 return SP_FORMERROR; 911 912 lp->sl_prefprog = ALLOC_CLEAR_MULT(regprog_T *, cnt); 913 if (lp->sl_prefprog == NULL) 914 return SP_OTHERERROR; 915 lp->sl_prefixcnt = cnt; 916 917 for (i = 0; i < cnt; ++i) 918 { 919 // <prefcond> : <condlen> <condstr> 920 n = getc(fd); // <condlen> 921 if (n < 0 || n >= MAXWLEN) 922 return SP_FORMERROR; 923 924 // When <condlen> is zero we have an empty condition. Otherwise 925 // compile the regexp program used to check for the condition. 926 if (n > 0) 927 { 928 buf[0] = '^'; // always match at one position only 929 p = buf + 1; 930 while (n-- > 0) 931 *p++ = getc(fd); // <condstr> 932 *p = NUL; 933 lp->sl_prefprog[i] = vim_regcomp(buf, RE_MAGIC + RE_STRING); 934 } 935 } 936 return 0; 937 } 938 939 /* 940 * Read REP or REPSAL items section from "fd": <repcount> <rep> ... 941 * Return SP_*ERROR flags. 942 */ 943 static int 944 read_rep_section(FILE *fd, garray_T *gap, short *first) 945 { 946 int cnt; 947 fromto_T *ftp; 948 int i; 949 950 cnt = get2c(fd); // <repcount> 951 if (cnt < 0) 952 return SP_TRUNCERROR; 953 954 if (ga_grow(gap, cnt) == FAIL) 955 return SP_OTHERERROR; 956 957 // <rep> : <repfromlen> <repfrom> <reptolen> <repto> 958 for (; gap->ga_len < cnt; ++gap->ga_len) 959 { 960 ftp = &((fromto_T *)gap->ga_data)[gap->ga_len]; 961 ftp->ft_from = read_cnt_string(fd, 1, &i); 962 if (i < 0) 963 return i; 964 if (i == 0) 965 return SP_FORMERROR; 966 ftp->ft_to = read_cnt_string(fd, 1, &i); 967 if (i <= 0) 968 { 969 vim_free(ftp->ft_from); 970 if (i < 0) 971 return i; 972 return SP_FORMERROR; 973 } 974 } 975 976 // Fill the first-index table. 977 for (i = 0; i < 256; ++i) 978 first[i] = -1; 979 for (i = 0; i < gap->ga_len; ++i) 980 { 981 ftp = &((fromto_T *)gap->ga_data)[i]; 982 if (first[*ftp->ft_from] == -1) 983 first[*ftp->ft_from] = i; 984 } 985 return 0; 986 } 987 988 /* 989 * Read SN_SAL section: <salflags> <salcount> <sal> ... 990 * Return SP_*ERROR flags. 991 */ 992 static int 993 read_sal_section(FILE *fd, slang_T *slang) 994 { 995 int i; 996 int cnt; 997 garray_T *gap; 998 salitem_T *smp; 999 int ccnt; 1000 char_u *p; 1001 1002 slang->sl_sofo = FALSE; 1003 1004 i = getc(fd); // <salflags> 1005 if (i & SAL_F0LLOWUP) 1006 slang->sl_followup = TRUE; 1007 if (i & SAL_COLLAPSE) 1008 slang->sl_collapse = TRUE; 1009 if (i & SAL_REM_ACCENTS) 1010 slang->sl_rem_accents = TRUE; 1011 1012 cnt = get2c(fd); // <salcount> 1013 if (cnt < 0) 1014 return SP_TRUNCERROR; 1015 1016 gap = &slang->sl_sal; 1017 ga_init2(gap, sizeof(salitem_T), 10); 1018 if (ga_grow(gap, cnt + 1) == FAIL) 1019 return SP_OTHERERROR; 1020 1021 // <sal> : <salfromlen> <salfrom> <saltolen> <salto> 1022 for (; gap->ga_len < cnt; ++gap->ga_len) 1023 { 1024 int c = NUL; 1025 1026 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 1027 ccnt = getc(fd); // <salfromlen> 1028 if (ccnt < 0) 1029 return SP_TRUNCERROR; 1030 if ((p = alloc(ccnt + 2)) == NULL) 1031 return SP_OTHERERROR; 1032 smp->sm_lead = p; 1033 1034 // Read up to the first special char into sm_lead. 1035 for (i = 0; i < ccnt; ++i) 1036 { 1037 c = getc(fd); // <salfrom> 1038 if (vim_strchr((char_u *)"0123456789(-<^$", c) != NULL) 1039 break; 1040 *p++ = c; 1041 } 1042 smp->sm_leadlen = (int)(p - smp->sm_lead); 1043 *p++ = NUL; 1044 1045 // Put (abc) chars in sm_oneof, if any. 1046 if (c == '(') 1047 { 1048 smp->sm_oneof = p; 1049 for (++i; i < ccnt; ++i) 1050 { 1051 c = getc(fd); // <salfrom> 1052 if (c == ')') 1053 break; 1054 *p++ = c; 1055 } 1056 *p++ = NUL; 1057 if (++i < ccnt) 1058 c = getc(fd); 1059 } 1060 else 1061 smp->sm_oneof = NULL; 1062 1063 // Any following chars go in sm_rules. 1064 smp->sm_rules = p; 1065 if (i < ccnt) 1066 // store the char we got while checking for end of sm_lead 1067 *p++ = c; 1068 for (++i; i < ccnt; ++i) 1069 *p++ = getc(fd); // <salfrom> 1070 *p++ = NUL; 1071 1072 // <saltolen> <salto> 1073 smp->sm_to = read_cnt_string(fd, 1, &ccnt); 1074 if (ccnt < 0) 1075 { 1076 vim_free(smp->sm_lead); 1077 return ccnt; 1078 } 1079 1080 if (has_mbyte) 1081 { 1082 // convert the multi-byte strings to wide char strings 1083 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 1084 smp->sm_leadlen = mb_charlen(smp->sm_lead); 1085 if (smp->sm_oneof == NULL) 1086 smp->sm_oneof_w = NULL; 1087 else 1088 smp->sm_oneof_w = mb_str2wide(smp->sm_oneof); 1089 if (smp->sm_to == NULL) 1090 smp->sm_to_w = NULL; 1091 else 1092 smp->sm_to_w = mb_str2wide(smp->sm_to); 1093 if (smp->sm_lead_w == NULL 1094 || (smp->sm_oneof_w == NULL && smp->sm_oneof != NULL) 1095 || (smp->sm_to_w == NULL && smp->sm_to != NULL)) 1096 { 1097 vim_free(smp->sm_lead); 1098 vim_free(smp->sm_to); 1099 vim_free(smp->sm_lead_w); 1100 vim_free(smp->sm_oneof_w); 1101 vim_free(smp->sm_to_w); 1102 return SP_OTHERERROR; 1103 } 1104 } 1105 } 1106 1107 if (gap->ga_len > 0) 1108 { 1109 // Add one extra entry to mark the end with an empty sm_lead. Avoids 1110 // that we need to check the index every time. 1111 smp = &((salitem_T *)gap->ga_data)[gap->ga_len]; 1112 if ((p = alloc(1)) == NULL) 1113 return SP_OTHERERROR; 1114 p[0] = NUL; 1115 smp->sm_lead = p; 1116 smp->sm_leadlen = 0; 1117 smp->sm_oneof = NULL; 1118 smp->sm_rules = p; 1119 smp->sm_to = NULL; 1120 if (has_mbyte) 1121 { 1122 smp->sm_lead_w = mb_str2wide(smp->sm_lead); 1123 smp->sm_leadlen = 0; 1124 smp->sm_oneof_w = NULL; 1125 smp->sm_to_w = NULL; 1126 } 1127 ++gap->ga_len; 1128 } 1129 1130 // Fill the first-index table. 1131 set_sal_first(slang); 1132 1133 return 0; 1134 } 1135 1136 /* 1137 * Read SN_WORDS: <word> ... 1138 * Return SP_*ERROR flags. 1139 */ 1140 static int 1141 read_words_section(FILE *fd, slang_T *lp, int len) 1142 { 1143 int done = 0; 1144 int i; 1145 int c; 1146 char_u word[MAXWLEN]; 1147 1148 while (done < len) 1149 { 1150 // Read one word at a time. 1151 for (i = 0; ; ++i) 1152 { 1153 c = getc(fd); 1154 if (c == EOF) 1155 return SP_TRUNCERROR; 1156 word[i] = c; 1157 if (word[i] == NUL) 1158 break; 1159 if (i == MAXWLEN - 1) 1160 return SP_FORMERROR; 1161 } 1162 1163 // Init the count to 10. 1164 count_common_word(lp, word, -1, 10); 1165 done += i + 1; 1166 } 1167 return 0; 1168 } 1169 1170 /* 1171 * SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 1172 * Return SP_*ERROR flags. 1173 */ 1174 static int 1175 read_sofo_section(FILE *fd, slang_T *slang) 1176 { 1177 int cnt; 1178 char_u *from, *to; 1179 int res; 1180 1181 slang->sl_sofo = TRUE; 1182 1183 // <sofofromlen> <sofofrom> 1184 from = read_cnt_string(fd, 2, &cnt); 1185 if (cnt < 0) 1186 return cnt; 1187 1188 // <sofotolen> <sofoto> 1189 to = read_cnt_string(fd, 2, &cnt); 1190 if (cnt < 0) 1191 { 1192 vim_free(from); 1193 return cnt; 1194 } 1195 1196 // Store the info in slang->sl_sal and/or slang->sl_sal_first. 1197 if (from != NULL && to != NULL) 1198 res = set_sofo(slang, from, to); 1199 else if (from != NULL || to != NULL) 1200 res = SP_FORMERROR; // only one of two strings is an error 1201 else 1202 res = 0; 1203 1204 vim_free(from); 1205 vim_free(to); 1206 return res; 1207 } 1208 1209 /* 1210 * Read the compound section from the .spl file: 1211 * <compmax> <compminlen> <compsylmax> <compoptions> <compflags> 1212 * Returns SP_*ERROR flags. 1213 */ 1214 static int 1215 read_compound(FILE *fd, slang_T *slang, int len) 1216 { 1217 int todo = len; 1218 int c; 1219 int atstart; 1220 char_u *pat; 1221 char_u *pp; 1222 char_u *cp; 1223 char_u *ap; 1224 char_u *crp; 1225 int cnt; 1226 garray_T *gap; 1227 1228 if (todo < 2) 1229 return SP_FORMERROR; // need at least two bytes 1230 1231 --todo; 1232 c = getc(fd); // <compmax> 1233 if (c < 2) 1234 c = MAXWLEN; 1235 slang->sl_compmax = c; 1236 1237 --todo; 1238 c = getc(fd); // <compminlen> 1239 if (c < 1) 1240 c = 0; 1241 slang->sl_compminlen = c; 1242 1243 --todo; 1244 c = getc(fd); // <compsylmax> 1245 if (c < 1) 1246 c = MAXWLEN; 1247 slang->sl_compsylmax = c; 1248 1249 c = getc(fd); // <compoptions> 1250 if (c != 0) 1251 ungetc(c, fd); // be backwards compatible with Vim 7.0b 1252 else 1253 { 1254 --todo; 1255 c = getc(fd); // only use the lower byte for now 1256 --todo; 1257 slang->sl_compoptions = c; 1258 1259 gap = &slang->sl_comppat; 1260 c = get2c(fd); // <comppatcount> 1261 if (c < 0) 1262 return SP_TRUNCERROR; 1263 todo -= 2; 1264 ga_init2(gap, sizeof(char_u *), c); 1265 if (ga_grow(gap, c) == OK) 1266 while (--c >= 0) 1267 { 1268 ((char_u **)(gap->ga_data))[gap->ga_len++] = 1269 read_cnt_string(fd, 1, &cnt); 1270 // <comppatlen> <comppattext> 1271 if (cnt < 0) 1272 return cnt; 1273 todo -= cnt + 1; 1274 } 1275 } 1276 if (todo < 0) 1277 return SP_FORMERROR; 1278 1279 // Turn the COMPOUNDRULE items into a regexp pattern: 1280 // "a[bc]/a*b+" -> "^\(a[bc]\|a*b\+\)$". 1281 // Inserting backslashes may double the length, "^\(\)$<Nul>" is 7 bytes. 1282 // Conversion to utf-8 may double the size. 1283 c = todo * 2 + 7; 1284 if (enc_utf8) 1285 c += todo * 2; 1286 pat = alloc(c); 1287 if (pat == NULL) 1288 return SP_OTHERERROR; 1289 1290 // We also need a list of all flags that can appear at the start and one 1291 // for all flags. 1292 cp = alloc(todo + 1); 1293 if (cp == NULL) 1294 { 1295 vim_free(pat); 1296 return SP_OTHERERROR; 1297 } 1298 slang->sl_compstartflags = cp; 1299 *cp = NUL; 1300 1301 ap = alloc(todo + 1); 1302 if (ap == NULL) 1303 { 1304 vim_free(pat); 1305 return SP_OTHERERROR; 1306 } 1307 slang->sl_compallflags = ap; 1308 *ap = NUL; 1309 1310 // And a list of all patterns in their original form, for checking whether 1311 // compounding may work in match_compoundrule(). This is freed when we 1312 // encounter a wildcard, the check doesn't work then. 1313 crp = alloc(todo + 1); 1314 slang->sl_comprules = crp; 1315 1316 pp = pat; 1317 *pp++ = '^'; 1318 *pp++ = '\\'; 1319 *pp++ = '('; 1320 1321 atstart = 1; 1322 while (todo-- > 0) 1323 { 1324 c = getc(fd); // <compflags> 1325 if (c == EOF) 1326 { 1327 vim_free(pat); 1328 return SP_TRUNCERROR; 1329 } 1330 1331 // Add all flags to "sl_compallflags". 1332 if (vim_strchr((char_u *)"?*+[]/", c) == NULL 1333 && !byte_in_str(slang->sl_compallflags, c)) 1334 { 1335 *ap++ = c; 1336 *ap = NUL; 1337 } 1338 1339 if (atstart != 0) 1340 { 1341 // At start of item: copy flags to "sl_compstartflags". For a 1342 // [abc] item set "atstart" to 2 and copy up to the ']'. 1343 if (c == '[') 1344 atstart = 2; 1345 else if (c == ']') 1346 atstart = 0; 1347 else 1348 { 1349 if (!byte_in_str(slang->sl_compstartflags, c)) 1350 { 1351 *cp++ = c; 1352 *cp = NUL; 1353 } 1354 if (atstart == 1) 1355 atstart = 0; 1356 } 1357 } 1358 1359 // Copy flag to "sl_comprules", unless we run into a wildcard. 1360 if (crp != NULL) 1361 { 1362 if (c == '?' || c == '+' || c == '*') 1363 { 1364 VIM_CLEAR(slang->sl_comprules); 1365 crp = NULL; 1366 } 1367 else 1368 *crp++ = c; 1369 } 1370 1371 if (c == '/') // slash separates two items 1372 { 1373 *pp++ = '\\'; 1374 *pp++ = '|'; 1375 atstart = 1; 1376 } 1377 else // normal char, "[abc]" and '*' are copied as-is 1378 { 1379 if (c == '?' || c == '+' || c == '~') 1380 *pp++ = '\\'; // "a?" becomes "a\?", "a+" becomes "a\+" 1381 if (enc_utf8) 1382 pp += mb_char2bytes(c, pp); 1383 else 1384 *pp++ = c; 1385 } 1386 } 1387 1388 *pp++ = '\\'; 1389 *pp++ = ')'; 1390 *pp++ = '$'; 1391 *pp = NUL; 1392 1393 if (crp != NULL) 1394 *crp = NUL; 1395 1396 slang->sl_compprog = vim_regcomp(pat, RE_MAGIC + RE_STRING + RE_STRICT); 1397 vim_free(pat); 1398 if (slang->sl_compprog == NULL) 1399 return SP_FORMERROR; 1400 1401 return 0; 1402 } 1403 1404 /* 1405 * Set the SOFOFROM and SOFOTO items in language "lp". 1406 * Returns SP_*ERROR flags when there is something wrong. 1407 */ 1408 static int 1409 set_sofo(slang_T *lp, char_u *from, char_u *to) 1410 { 1411 int i; 1412 1413 garray_T *gap; 1414 char_u *s; 1415 char_u *p; 1416 int c; 1417 int *inp; 1418 1419 if (has_mbyte) 1420 { 1421 // Use "sl_sal" as an array with 256 pointers to a list of wide 1422 // characters. The index is the low byte of the character. 1423 // The list contains from-to pairs with a terminating NUL. 1424 // sl_sal_first[] is used for latin1 "from" characters. 1425 gap = &lp->sl_sal; 1426 ga_init2(gap, sizeof(int *), 1); 1427 if (ga_grow(gap, 256) == FAIL) 1428 return SP_OTHERERROR; 1429 vim_memset(gap->ga_data, 0, sizeof(int *) * 256); 1430 gap->ga_len = 256; 1431 1432 // First count the number of items for each list. Temporarily use 1433 // sl_sal_first[] for this. 1434 for (p = from, s = to; *p != NUL && *s != NUL; ) 1435 { 1436 c = mb_cptr2char_adv(&p); 1437 MB_CPTR_ADV(s); 1438 if (c >= 256) 1439 ++lp->sl_sal_first[c & 0xff]; 1440 } 1441 if (*p != NUL || *s != NUL) // lengths differ 1442 return SP_FORMERROR; 1443 1444 // Allocate the lists. 1445 for (i = 0; i < 256; ++i) 1446 if (lp->sl_sal_first[i] > 0) 1447 { 1448 p = alloc(sizeof(int) * (lp->sl_sal_first[i] * 2 + 1)); 1449 if (p == NULL) 1450 return SP_OTHERERROR; 1451 ((int **)gap->ga_data)[i] = (int *)p; 1452 *(int *)p = 0; 1453 } 1454 1455 // Put the characters up to 255 in sl_sal_first[] the rest in a sl_sal 1456 // list. 1457 vim_memset(lp->sl_sal_first, 0, sizeof(salfirst_T) * 256); 1458 for (p = from, s = to; *p != NUL && *s != NUL; ) 1459 { 1460 c = mb_cptr2char_adv(&p); 1461 i = mb_cptr2char_adv(&s); 1462 if (c >= 256) 1463 { 1464 // Append the from-to chars at the end of the list with 1465 // the low byte. 1466 inp = ((int **)gap->ga_data)[c & 0xff]; 1467 while (*inp != 0) 1468 ++inp; 1469 *inp++ = c; // from char 1470 *inp++ = i; // to char 1471 *inp++ = NUL; // NUL at the end 1472 } 1473 else 1474 // mapping byte to char is done in sl_sal_first[] 1475 lp->sl_sal_first[c] = i; 1476 } 1477 } 1478 else 1479 { 1480 // mapping bytes to bytes is done in sl_sal_first[] 1481 if (STRLEN(from) != STRLEN(to)) 1482 return SP_FORMERROR; 1483 1484 for (i = 0; to[i] != NUL; ++i) 1485 lp->sl_sal_first[from[i]] = to[i]; 1486 lp->sl_sal.ga_len = 1; // indicates we have soundfolding 1487 } 1488 1489 return 0; 1490 } 1491 1492 /* 1493 * Fill the first-index table for "lp". 1494 */ 1495 static void 1496 set_sal_first(slang_T *lp) 1497 { 1498 salfirst_T *sfirst; 1499 int i; 1500 salitem_T *smp; 1501 int c; 1502 garray_T *gap = &lp->sl_sal; 1503 1504 sfirst = lp->sl_sal_first; 1505 for (i = 0; i < 256; ++i) 1506 sfirst[i] = -1; 1507 smp = (salitem_T *)gap->ga_data; 1508 for (i = 0; i < gap->ga_len; ++i) 1509 { 1510 if (has_mbyte) 1511 // Use the lowest byte of the first character. For latin1 it's 1512 // the character, for other encodings it should differ for most 1513 // characters. 1514 c = *smp[i].sm_lead_w & 0xff; 1515 else 1516 c = *smp[i].sm_lead; 1517 if (sfirst[c] == -1) 1518 { 1519 sfirst[c] = i; 1520 if (has_mbyte) 1521 { 1522 int n; 1523 1524 // Make sure all entries with this byte are following each 1525 // other. Move the ones that are in the wrong position. Do 1526 // keep the same ordering! 1527 while (i + 1 < gap->ga_len 1528 && (*smp[i + 1].sm_lead_w & 0xff) == c) 1529 // Skip over entry with same index byte. 1530 ++i; 1531 1532 for (n = 1; i + n < gap->ga_len; ++n) 1533 if ((*smp[i + n].sm_lead_w & 0xff) == c) 1534 { 1535 salitem_T tsal; 1536 1537 // Move entry with same index byte after the entries 1538 // we already found. 1539 ++i; 1540 --n; 1541 tsal = smp[i + n]; 1542 mch_memmove(smp + i + 1, smp + i, 1543 sizeof(salitem_T) * n); 1544 smp[i] = tsal; 1545 } 1546 } 1547 } 1548 } 1549 } 1550 1551 /* 1552 * Turn a multi-byte string into a wide character string. 1553 * Return it in allocated memory (NULL for out-of-memory) 1554 */ 1555 static int * 1556 mb_str2wide(char_u *s) 1557 { 1558 int *res; 1559 char_u *p; 1560 int i = 0; 1561 1562 res = ALLOC_MULT(int, mb_charlen(s) + 1); 1563 if (res != NULL) 1564 { 1565 for (p = s; *p != NUL; ) 1566 res[i++] = mb_ptr2char_adv(&p); 1567 res[i] = NUL; 1568 } 1569 return res; 1570 } 1571 1572 /* 1573 * Read a tree from the .spl or .sug file. 1574 * Allocates the memory and stores pointers in "bytsp" and "idxsp". 1575 * This is skipped when the tree has zero length. 1576 * Returns zero when OK, SP_ value for an error. 1577 */ 1578 static int 1579 spell_read_tree( 1580 FILE *fd, 1581 char_u **bytsp, 1582 long *bytsp_len, 1583 idx_T **idxsp, 1584 int prefixtree, // TRUE for the prefix tree 1585 int prefixcnt) // when "prefixtree" is TRUE: prefix count 1586 { 1587 long len; 1588 int idx; 1589 char_u *bp; 1590 idx_T *ip; 1591 1592 // The tree size was computed when writing the file, so that we can 1593 // allocate it as one long block. <nodecount> 1594 len = get4c(fd); 1595 if (len < 0) 1596 return SP_TRUNCERROR; 1597 if (len >= LONG_MAX / (long)sizeof(int)) 1598 // Invalid length, multiply with sizeof(int) would overflow. 1599 return SP_FORMERROR; 1600 if (len > 0) 1601 { 1602 // Allocate the byte array. 1603 bp = alloc(len); 1604 if (bp == NULL) 1605 return SP_OTHERERROR; 1606 *bytsp = bp; 1607 if (bytsp_len != NULL) 1608 *bytsp_len = len; 1609 1610 // Allocate the index array. 1611 ip = lalloc_clear(len * sizeof(int), TRUE); 1612 if (ip == NULL) 1613 return SP_OTHERERROR; 1614 *idxsp = ip; 1615 1616 // Recursively read the tree and store it in the array. 1617 idx = read_tree_node(fd, bp, ip, len, 0, prefixtree, prefixcnt); 1618 if (idx < 0) 1619 return idx; 1620 } 1621 return 0; 1622 } 1623 1624 /* 1625 * Read one row of siblings from the spell file and store it in the byte array 1626 * "byts" and index array "idxs". Recursively read the children. 1627 * 1628 * NOTE: The code here must match put_node()! 1629 * 1630 * Returns the index (>= 0) following the siblings. 1631 * Returns SP_TRUNCERROR if the file is shorter than expected. 1632 * Returns SP_FORMERROR if there is a format error. 1633 */ 1634 static idx_T 1635 read_tree_node( 1636 FILE *fd, 1637 char_u *byts, 1638 idx_T *idxs, 1639 int maxidx, // size of arrays 1640 idx_T startidx, // current index in "byts" and "idxs" 1641 int prefixtree, // TRUE for reading PREFIXTREE 1642 int maxprefcondnr) // maximum for <prefcondnr> 1643 { 1644 int len; 1645 int i; 1646 int n; 1647 idx_T idx = startidx; 1648 int c; 1649 int c2; 1650 #define SHARED_MASK 0x8000000 1651 1652 len = getc(fd); // <siblingcount> 1653 if (len <= 0) 1654 return SP_TRUNCERROR; 1655 1656 if (startidx + len >= maxidx) 1657 return SP_FORMERROR; 1658 byts[idx++] = len; 1659 1660 // Read the byte values, flag/region bytes and shared indexes. 1661 for (i = 1; i <= len; ++i) 1662 { 1663 c = getc(fd); // <byte> 1664 if (c < 0) 1665 return SP_TRUNCERROR; 1666 if (c <= BY_SPECIAL) 1667 { 1668 if (c == BY_NOFLAGS && !prefixtree) 1669 { 1670 // No flags, all regions. 1671 idxs[idx] = 0; 1672 c = 0; 1673 } 1674 else if (c != BY_INDEX) 1675 { 1676 if (prefixtree) 1677 { 1678 // Read the optional pflags byte, the prefix ID and the 1679 // condition nr. In idxs[] store the prefix ID in the low 1680 // byte, the condition index shifted up 8 bits, the flags 1681 // shifted up 24 bits. 1682 if (c == BY_FLAGS) 1683 c = getc(fd) << 24; // <pflags> 1684 else 1685 c = 0; 1686 1687 c |= getc(fd); // <affixID> 1688 1689 n = get2c(fd); // <prefcondnr> 1690 if (n >= maxprefcondnr) 1691 return SP_FORMERROR; 1692 c |= (n << 8); 1693 } 1694 else // c must be BY_FLAGS or BY_FLAGS2 1695 { 1696 // Read flags and optional region and prefix ID. In 1697 // idxs[] the flags go in the low two bytes, region above 1698 // that and prefix ID above the region. 1699 c2 = c; 1700 c = getc(fd); // <flags> 1701 if (c2 == BY_FLAGS2) 1702 c = (getc(fd) << 8) + c; // <flags2> 1703 if (c & WF_REGION) 1704 c = (getc(fd) << 16) + c; // <region> 1705 if (c & WF_AFX) 1706 c = (getc(fd) << 24) + c; // <affixID> 1707 } 1708 1709 idxs[idx] = c; 1710 c = 0; 1711 } 1712 else // c == BY_INDEX 1713 { 1714 // <nodeidx> 1715 n = get3c(fd); 1716 if (n < 0 || n >= maxidx) 1717 return SP_FORMERROR; 1718 idxs[idx] = n + SHARED_MASK; 1719 c = getc(fd); // <xbyte> 1720 } 1721 } 1722 byts[idx++] = c; 1723 } 1724 1725 // Recursively read the children for non-shared siblings. 1726 // Skip the end-of-word ones (zero byte value) and the shared ones (and 1727 // remove SHARED_MASK) 1728 for (i = 1; i <= len; ++i) 1729 if (byts[startidx + i] != 0) 1730 { 1731 if (idxs[startidx + i] & SHARED_MASK) 1732 idxs[startidx + i] &= ~SHARED_MASK; 1733 else 1734 { 1735 idxs[startidx + i] = idx; 1736 idx = read_tree_node(fd, byts, idxs, maxidx, idx, 1737 prefixtree, maxprefcondnr); 1738 if (idx < 0) 1739 break; 1740 } 1741 } 1742 1743 return idx; 1744 } 1745 1746 /* 1747 * Reload the spell file "fname" if it's loaded. 1748 */ 1749 static void 1750 spell_reload_one( 1751 char_u *fname, 1752 int added_word) // invoked through "zg" 1753 { 1754 slang_T *slang; 1755 int didit = FALSE; 1756 1757 FOR_ALL_SPELL_LANGS(slang) 1758 { 1759 if (fullpathcmp(fname, slang->sl_fname, FALSE, TRUE) == FPC_SAME) 1760 { 1761 slang_clear(slang); 1762 if (spell_load_file(fname, NULL, slang, FALSE) == NULL) 1763 // reloading failed, clear the language 1764 slang_clear(slang); 1765 redraw_all_later(SOME_VALID); 1766 didit = TRUE; 1767 } 1768 } 1769 1770 // When "zg" was used and the file wasn't loaded yet, should redo 1771 // 'spelllang' to load it now. 1772 if (added_word && !didit) 1773 did_set_spelllang(curwin); 1774 } 1775 1776 1777 /* 1778 * Functions for ":mkspell". 1779 */ 1780 1781 #define MAXLINELEN 500 // Maximum length in bytes of a line in a .aff 1782 // and .dic file. 1783 /* 1784 * Main structure to store the contents of a ".aff" file. 1785 */ 1786 typedef struct afffile_S 1787 { 1788 char_u *af_enc; // "SET", normalized, alloc'ed string or NULL 1789 int af_flagtype; // AFT_CHAR, AFT_LONG, AFT_NUM or AFT_CAPLONG 1790 unsigned af_rare; // RARE ID for rare word 1791 unsigned af_keepcase; // KEEPCASE ID for keep-case word 1792 unsigned af_bad; // BAD ID for banned word 1793 unsigned af_needaffix; // NEEDAFFIX ID 1794 unsigned af_circumfix; // CIRCUMFIX ID 1795 unsigned af_needcomp; // NEEDCOMPOUND ID 1796 unsigned af_comproot; // COMPOUNDROOT ID 1797 unsigned af_compforbid; // COMPOUNDFORBIDFLAG ID 1798 unsigned af_comppermit; // COMPOUNDPERMITFLAG ID 1799 unsigned af_nosuggest; // NOSUGGEST ID 1800 int af_pfxpostpone; // postpone prefixes without chop string and 1801 // without flags 1802 int af_ignoreextra; // IGNOREEXTRA present 1803 hashtab_T af_pref; // hashtable for prefixes, affheader_T 1804 hashtab_T af_suff; // hashtable for suffixes, affheader_T 1805 hashtab_T af_comp; // hashtable for compound flags, compitem_T 1806 } afffile_T; 1807 1808 #define AFT_CHAR 0 // flags are one character 1809 #define AFT_LONG 1 // flags are two characters 1810 #define AFT_CAPLONG 2 // flags are one or two characters 1811 #define AFT_NUM 3 // flags are numbers, comma separated 1812 1813 typedef struct affentry_S affentry_T; 1814 // Affix entry from ".aff" file. Used for prefixes and suffixes. 1815 struct affentry_S 1816 { 1817 affentry_T *ae_next; // next affix with same name/number 1818 char_u *ae_chop; // text to chop off basic word (can be NULL) 1819 char_u *ae_add; // text to add to basic word (can be NULL) 1820 char_u *ae_flags; // flags on the affix (can be NULL) 1821 char_u *ae_cond; // condition (NULL for ".") 1822 regprog_T *ae_prog; // regexp program for ae_cond or NULL 1823 char ae_compforbid; // COMPOUNDFORBIDFLAG found 1824 char ae_comppermit; // COMPOUNDPERMITFLAG found 1825 }; 1826 1827 #define AH_KEY_LEN 17 // 2 x 8 bytes + NUL 1828 1829 // Affix header from ".aff" file. Used for af_pref and af_suff. 1830 typedef struct affheader_S 1831 { 1832 char_u ah_key[AH_KEY_LEN]; // key for hashtab == name of affix 1833 unsigned ah_flag; // affix name as number, uses "af_flagtype" 1834 int ah_newID; // prefix ID after renumbering; 0 if not used 1835 int ah_combine; // suffix may combine with prefix 1836 int ah_follows; // another affix block should be following 1837 affentry_T *ah_first; // first affix entry 1838 } affheader_T; 1839 1840 #define HI2AH(hi) ((affheader_T *)(hi)->hi_key) 1841 1842 // Flag used in compound items. 1843 typedef struct compitem_S 1844 { 1845 char_u ci_key[AH_KEY_LEN]; // key for hashtab == name of compound 1846 unsigned ci_flag; // affix name as number, uses "af_flagtype" 1847 int ci_newID; // affix ID after renumbering. 1848 } compitem_T; 1849 1850 #define HI2CI(hi) ((compitem_T *)(hi)->hi_key) 1851 1852 /* 1853 * Structure that is used to store the items in the word tree. This avoids 1854 * the need to keep track of each allocated thing, everything is freed all at 1855 * once after ":mkspell" is done. 1856 * Note: "sb_next" must be just before "sb_data" to make sure the alignment of 1857 * "sb_data" is correct for systems where pointers must be aligned on 1858 * pointer-size boundaries and sizeof(pointer) > sizeof(int) (e.g., Sparc). 1859 */ 1860 #define SBLOCKSIZE 16000 // size of sb_data 1861 typedef struct sblock_S sblock_T; 1862 struct sblock_S 1863 { 1864 int sb_used; // nr of bytes already in use 1865 sblock_T *sb_next; // next block in list 1866 char_u sb_data[1]; // data, actually longer 1867 }; 1868 1869 /* 1870 * A node in the tree. 1871 */ 1872 typedef struct wordnode_S wordnode_T; 1873 struct wordnode_S 1874 { 1875 union // shared to save space 1876 { 1877 char_u hashkey[6]; // the hash key, only used while compressing 1878 int index; // index in written nodes (valid after first 1879 // round) 1880 } wn_u1; 1881 union // shared to save space 1882 { 1883 wordnode_T *next; // next node with same hash key 1884 wordnode_T *wnode; // parent node that will write this node 1885 } wn_u2; 1886 wordnode_T *wn_child; // child (next byte in word) 1887 wordnode_T *wn_sibling; // next sibling (alternate byte in word, 1888 // always sorted) 1889 int wn_refs; // Nr. of references to this node. Only 1890 // relevant for first node in a list of 1891 // siblings, in following siblings it is 1892 // always one. 1893 char_u wn_byte; // Byte for this node. NUL for word end 1894 1895 // Info for when "wn_byte" is NUL. 1896 // In PREFIXTREE "wn_region" is used for the prefcondnr. 1897 // In the soundfolded word tree "wn_flags" has the MSW of the wordnr and 1898 // "wn_region" the LSW of the wordnr. 1899 char_u wn_affixID; // supported/required prefix ID or 0 1900 short_u wn_flags; // WF_ flags 1901 short wn_region; // region mask 1902 1903 #ifdef SPELL_PRINTTREE 1904 int wn_nr; // sequence nr for printing 1905 #endif 1906 }; 1907 1908 #define WN_MASK 0xffff // mask relevant bits of "wn_flags" 1909 1910 #define HI2WN(hi) (wordnode_T *)((hi)->hi_key) 1911 1912 /* 1913 * Info used while reading the spell files. 1914 */ 1915 typedef struct spellinfo_S 1916 { 1917 wordnode_T *si_foldroot; // tree with case-folded words 1918 long si_foldwcount; // nr of words in si_foldroot 1919 1920 wordnode_T *si_keeproot; // tree with keep-case words 1921 long si_keepwcount; // nr of words in si_keeproot 1922 1923 wordnode_T *si_prefroot; // tree with postponed prefixes 1924 1925 long si_sugtree; // creating the soundfolding trie 1926 1927 sblock_T *si_blocks; // memory blocks used 1928 long si_blocks_cnt; // memory blocks allocated 1929 int si_did_emsg; // TRUE when ran out of memory 1930 1931 long si_compress_cnt; // words to add before lowering 1932 // compression limit 1933 wordnode_T *si_first_free; // List of nodes that have been freed during 1934 // compression, linked by "wn_child" field. 1935 long si_free_count; // number of nodes in si_first_free 1936 #ifdef SPELL_PRINTTREE 1937 int si_wordnode_nr; // sequence nr for nodes 1938 #endif 1939 buf_T *si_spellbuf; // buffer used to store soundfold word table 1940 1941 int si_ascii; // handling only ASCII words 1942 int si_add; // addition file 1943 int si_clear_chartab; // when TRUE clear char tables 1944 int si_region; // region mask 1945 vimconv_T si_conv; // for conversion to 'encoding' 1946 int si_memtot; // runtime memory used 1947 int si_verbose; // verbose messages 1948 int si_msg_count; // number of words added since last message 1949 char_u *si_info; // info text chars or NULL 1950 int si_region_count; // number of regions supported (1 when there 1951 // are no regions) 1952 char_u si_region_name[MAXREGIONS * 2 + 1]; 1953 // region names; used only if 1954 // si_region_count > 1) 1955 1956 garray_T si_rep; // list of fromto_T entries from REP lines 1957 garray_T si_repsal; // list of fromto_T entries from REPSAL lines 1958 garray_T si_sal; // list of fromto_T entries from SAL lines 1959 char_u *si_sofofr; // SOFOFROM text 1960 char_u *si_sofoto; // SOFOTO text 1961 int si_nosugfile; // NOSUGFILE item found 1962 int si_nosplitsugs; // NOSPLITSUGS item found 1963 int si_nocompoundsugs; // NOCOMPOUNDSUGS item found 1964 int si_followup; // soundsalike: ? 1965 int si_collapse; // soundsalike: ? 1966 hashtab_T si_commonwords; // hashtable for common words 1967 time_t si_sugtime; // timestamp for .sug file 1968 int si_rem_accents; // soundsalike: remove accents 1969 garray_T si_map; // MAP info concatenated 1970 char_u *si_midword; // MIDWORD chars or NULL 1971 int si_compmax; // max nr of words for compounding 1972 int si_compminlen; // minimal length for compounding 1973 int si_compsylmax; // max nr of syllables for compounding 1974 int si_compoptions; // COMP_ flags 1975 garray_T si_comppat; // CHECKCOMPOUNDPATTERN items, each stored as 1976 // a string 1977 char_u *si_compflags; // flags used for compounding 1978 char_u si_nobreak; // NOBREAK 1979 char_u *si_syllable; // syllable string 1980 garray_T si_prefcond; // table with conditions for postponed 1981 // prefixes, each stored as a string 1982 int si_newprefID; // current value for ah_newID 1983 int si_newcompID; // current value for compound ID 1984 } spellinfo_T; 1985 1986 static int is_aff_rule(char_u **items, int itemcnt, char *rulename, int mincount); 1987 static void aff_process_flags(afffile_T *affile, affentry_T *entry); 1988 static int spell_info_item(char_u *s); 1989 static unsigned affitem2flag(int flagtype, char_u *item, char_u *fname, int lnum); 1990 static unsigned get_affitem(int flagtype, char_u **pp); 1991 static void process_compflags(spellinfo_T *spin, afffile_T *aff, char_u *compflags); 1992 static void check_renumber(spellinfo_T *spin); 1993 static void aff_check_number(int spinval, int affval, char *name); 1994 static void aff_check_string(char_u *spinval, char_u *affval, char *name); 1995 static int str_equal(char_u *s1, char_u *s2); 1996 static void add_fromto(spellinfo_T *spin, garray_T *gap, char_u *from, char_u *to); 1997 static int sal_to_bool(char_u *s); 1998 static int get_affix_flags(afffile_T *affile, char_u *afflist); 1999 static int get_pfxlist(afffile_T *affile, char_u *afflist, char_u *store_afflist); 2000 static void get_compflags(afffile_T *affile, char_u *afflist, char_u *store_afflist); 2001 static int store_aff_word(spellinfo_T *spin, char_u *word, char_u *afflist, afffile_T *affile, hashtab_T *ht, hashtab_T *xht, int condit, int flags, char_u *pfxlist, int pfxlen); 2002 static void *getroom(spellinfo_T *spin, size_t len, int align); 2003 static char_u *getroom_save(spellinfo_T *spin, char_u *s); 2004 static int store_word(spellinfo_T *spin, char_u *word, int flags, int region, char_u *pfxlist, int need_affix); 2005 static int tree_add_word(spellinfo_T *spin, char_u *word, wordnode_T *tree, int flags, int region, int affixID); 2006 static wordnode_T *get_wordnode(spellinfo_T *spin); 2007 static void free_wordnode(spellinfo_T *spin, wordnode_T *n); 2008 static void wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name); 2009 static long node_compress(spellinfo_T *spin, wordnode_T *node, hashtab_T *ht, long *tot); 2010 static int node_equal(wordnode_T *n1, wordnode_T *n2); 2011 static void clear_node(wordnode_T *node); 2012 static int put_node(FILE *fd, wordnode_T *node, int idx, int regionmask, int prefixtree); 2013 static int sug_filltree(spellinfo_T *spin, slang_T *slang); 2014 static int sug_maketable(spellinfo_T *spin); 2015 static int sug_filltable(spellinfo_T *spin, wordnode_T *node, int startwordnr, garray_T *gap); 2016 static int offset2bytes(int nr, char_u *buf); 2017 static void sug_write(spellinfo_T *spin, char_u *fname); 2018 static void spell_message(spellinfo_T *spin, char_u *str); 2019 static void init_spellfile(void); 2020 2021 // In the postponed prefixes tree wn_flags is used to store the WFP_ flags, 2022 // but it must be negative to indicate the prefix tree to tree_add_word(). 2023 // Use a negative number with the lower 8 bits zero. 2024 #define PFX_FLAGS -256 2025 2026 // flags for "condit" argument of store_aff_word() 2027 #define CONDIT_COMB 1 // affix must combine 2028 #define CONDIT_CFIX 2 // affix must have CIRCUMFIX flag 2029 #define CONDIT_SUF 4 // add a suffix for matching flags 2030 #define CONDIT_AFF 8 // word already has an affix 2031 2032 /* 2033 * Tunable parameters for when the tree is compressed. Filled from the 2034 * 'mkspellmem' option. 2035 */ 2036 static long compress_start = 30000; // memory / SBLOCKSIZE 2037 static long compress_inc = 100; // memory / SBLOCKSIZE 2038 static long compress_added = 500000; // word count 2039 2040 /* 2041 * Check the 'mkspellmem' option. Return FAIL if it's wrong. 2042 * Sets "sps_flags". 2043 */ 2044 int 2045 spell_check_msm(void) 2046 { 2047 char_u *p = p_msm; 2048 long start = 0; 2049 long incr = 0; 2050 long added = 0; 2051 2052 if (!VIM_ISDIGIT(*p)) 2053 return FAIL; 2054 // block count = (value * 1024) / SBLOCKSIZE (but avoid overflow) 2055 start = (getdigits(&p) * 10) / (SBLOCKSIZE / 102); 2056 if (*p != ',') 2057 return FAIL; 2058 ++p; 2059 if (!VIM_ISDIGIT(*p)) 2060 return FAIL; 2061 incr = (getdigits(&p) * 102) / (SBLOCKSIZE / 10); 2062 if (*p != ',') 2063 return FAIL; 2064 ++p; 2065 if (!VIM_ISDIGIT(*p)) 2066 return FAIL; 2067 added = getdigits(&p) * 1024; 2068 if (*p != NUL) 2069 return FAIL; 2070 2071 if (start == 0 || incr == 0 || added == 0 || incr > start) 2072 return FAIL; 2073 2074 compress_start = start; 2075 compress_inc = incr; 2076 compress_added = added; 2077 return OK; 2078 } 2079 2080 #ifdef SPELL_PRINTTREE 2081 /* 2082 * For debugging the tree code: print the current tree in a (more or less) 2083 * readable format, so that we can see what happens when adding a word and/or 2084 * compressing the tree. 2085 * Based on code from Olaf Seibert. 2086 */ 2087 #define PRINTLINESIZE 1000 2088 #define PRINTWIDTH 6 2089 2090 #define PRINTSOME(l, depth, fmt, a1, a2) vim_snprintf(l + depth * PRINTWIDTH, \ 2091 PRINTLINESIZE - PRINTWIDTH * depth, fmt, a1, a2) 2092 2093 static char line1[PRINTLINESIZE]; 2094 static char line2[PRINTLINESIZE]; 2095 static char line3[PRINTLINESIZE]; 2096 2097 static void 2098 spell_clear_flags(wordnode_T *node) 2099 { 2100 wordnode_T *np; 2101 2102 FOR_ALL_NODE_SIBLINGS(node, np) 2103 { 2104 np->wn_u1.index = FALSE; 2105 spell_clear_flags(np->wn_child); 2106 } 2107 } 2108 2109 static void 2110 spell_print_node(wordnode_T *node, int depth) 2111 { 2112 if (node->wn_u1.index) 2113 { 2114 // Done this node before, print the reference. 2115 PRINTSOME(line1, depth, "(%d)", node->wn_nr, 0); 2116 PRINTSOME(line2, depth, " ", 0, 0); 2117 PRINTSOME(line3, depth, " ", 0, 0); 2118 msg(line1); 2119 msg(line2); 2120 msg(line3); 2121 } 2122 else 2123 { 2124 node->wn_u1.index = TRUE; 2125 2126 if (node->wn_byte != NUL) 2127 { 2128 if (node->wn_child != NULL) 2129 PRINTSOME(line1, depth, " %c -> ", node->wn_byte, 0); 2130 else 2131 // Cannot happen? 2132 PRINTSOME(line1, depth, " %c ???", node->wn_byte, 0); 2133 } 2134 else 2135 PRINTSOME(line1, depth, " $ ", 0, 0); 2136 2137 PRINTSOME(line2, depth, "%d/%d ", node->wn_nr, node->wn_refs); 2138 2139 if (node->wn_sibling != NULL) 2140 PRINTSOME(line3, depth, " | ", 0, 0); 2141 else 2142 PRINTSOME(line3, depth, " ", 0, 0); 2143 2144 if (node->wn_byte == NUL) 2145 { 2146 msg(line1); 2147 msg(line2); 2148 msg(line3); 2149 } 2150 2151 // do the children 2152 if (node->wn_byte != NUL && node->wn_child != NULL) 2153 spell_print_node(node->wn_child, depth + 1); 2154 2155 // do the siblings 2156 if (node->wn_sibling != NULL) 2157 { 2158 // get rid of all parent details except | 2159 STRCPY(line1, line3); 2160 STRCPY(line2, line3); 2161 spell_print_node(node->wn_sibling, depth); 2162 } 2163 } 2164 } 2165 2166 static void 2167 spell_print_tree(wordnode_T *root) 2168 { 2169 if (root != NULL) 2170 { 2171 // Clear the "wn_u1.index" fields, used to remember what has been 2172 // done. 2173 spell_clear_flags(root); 2174 2175 // Recursively print the tree. 2176 spell_print_node(root, 0); 2177 } 2178 } 2179 #endif // SPELL_PRINTTREE 2180 2181 /* 2182 * Read the affix file "fname". 2183 * Returns an afffile_T, NULL for complete failure. 2184 */ 2185 static afffile_T * 2186 spell_read_aff(spellinfo_T *spin, char_u *fname) 2187 { 2188 FILE *fd; 2189 afffile_T *aff; 2190 char_u rline[MAXLINELEN]; 2191 char_u *line; 2192 char_u *pc = NULL; 2193 #define MAXITEMCNT 30 2194 char_u *(items[MAXITEMCNT]); 2195 int itemcnt; 2196 char_u *p; 2197 int lnum = 0; 2198 affheader_T *cur_aff = NULL; 2199 int did_postpone_prefix = FALSE; 2200 int aff_todo = 0; 2201 hashtab_T *tp; 2202 char_u *low = NULL; 2203 char_u *fol = NULL; 2204 char_u *upp = NULL; 2205 int do_rep; 2206 int do_repsal; 2207 int do_sal; 2208 int do_mapline; 2209 int found_map = FALSE; 2210 hashitem_T *hi; 2211 int l; 2212 int compminlen = 0; // COMPOUNDMIN value 2213 int compsylmax = 0; // COMPOUNDSYLMAX value 2214 int compoptions = 0; // COMP_ flags 2215 int compmax = 0; // COMPOUNDWORDMAX value 2216 char_u *compflags = NULL; // COMPOUNDFLAG and COMPOUNDRULE 2217 // concatenated 2218 char_u *midword = NULL; // MIDWORD value 2219 char_u *syllable = NULL; // SYLLABLE value 2220 char_u *sofofrom = NULL; // SOFOFROM value 2221 char_u *sofoto = NULL; // SOFOTO value 2222 2223 /* 2224 * Open the file. 2225 */ 2226 fd = mch_fopen((char *)fname, "r"); 2227 if (fd == NULL) 2228 { 2229 semsg(_(e_notopen), fname); 2230 return NULL; 2231 } 2232 2233 vim_snprintf((char *)IObuff, IOSIZE, _("Reading affix file %s..."), fname); 2234 spell_message(spin, IObuff); 2235 2236 // Only do REP lines when not done in another .aff file already. 2237 do_rep = spin->si_rep.ga_len == 0; 2238 2239 // Only do REPSAL lines when not done in another .aff file already. 2240 do_repsal = spin->si_repsal.ga_len == 0; 2241 2242 // Only do SAL lines when not done in another .aff file already. 2243 do_sal = spin->si_sal.ga_len == 0; 2244 2245 // Only do MAP lines when not done in another .aff file already. 2246 do_mapline = spin->si_map.ga_len == 0; 2247 2248 /* 2249 * Allocate and init the afffile_T structure. 2250 */ 2251 aff = (afffile_T *)getroom(spin, sizeof(afffile_T), TRUE); 2252 if (aff == NULL) 2253 { 2254 fclose(fd); 2255 return NULL; 2256 } 2257 hash_init(&aff->af_pref); 2258 hash_init(&aff->af_suff); 2259 hash_init(&aff->af_comp); 2260 2261 /* 2262 * Read all the lines in the file one by one. 2263 */ 2264 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 2265 { 2266 line_breakcheck(); 2267 ++lnum; 2268 2269 // Skip comment lines. 2270 if (*rline == '#') 2271 continue; 2272 2273 // Convert from "SET" to 'encoding' when needed. 2274 vim_free(pc); 2275 if (spin->si_conv.vc_type != CONV_NONE) 2276 { 2277 pc = string_convert(&spin->si_conv, rline, NULL); 2278 if (pc == NULL) 2279 { 2280 smsg(_("Conversion failure for word in %s line %d: %s"), 2281 fname, lnum, rline); 2282 continue; 2283 } 2284 line = pc; 2285 } 2286 else 2287 { 2288 pc = NULL; 2289 line = rline; 2290 } 2291 2292 // Split the line up in white separated items. Put a NUL after each 2293 // item. 2294 itemcnt = 0; 2295 for (p = line; ; ) 2296 { 2297 while (*p != NUL && *p <= ' ') // skip white space and CR/NL 2298 ++p; 2299 if (*p == NUL) 2300 break; 2301 if (itemcnt == MAXITEMCNT) // too many items 2302 break; 2303 items[itemcnt++] = p; 2304 // A few items have arbitrary text argument, don't split them. 2305 if (itemcnt == 2 && spell_info_item(items[0])) 2306 while (*p >= ' ' || *p == TAB) // skip until CR/NL 2307 ++p; 2308 else 2309 while (*p > ' ') // skip until white space or CR/NL 2310 ++p; 2311 if (*p == NUL) 2312 break; 2313 *p++ = NUL; 2314 } 2315 2316 // Handle non-empty lines. 2317 if (itemcnt > 0) 2318 { 2319 if (is_aff_rule(items, itemcnt, "SET", 2) && aff->af_enc == NULL) 2320 { 2321 // Setup for conversion from "ENC" to 'encoding'. 2322 aff->af_enc = enc_canonize(items[1]); 2323 if (aff->af_enc != NULL && !spin->si_ascii 2324 && convert_setup(&spin->si_conv, aff->af_enc, 2325 p_enc) == FAIL) 2326 smsg(_("Conversion in %s not supported: from %s to %s"), 2327 fname, aff->af_enc, p_enc); 2328 spin->si_conv.vc_fail = TRUE; 2329 } 2330 else if (is_aff_rule(items, itemcnt, "FLAG", 2) 2331 && aff->af_flagtype == AFT_CHAR) 2332 { 2333 if (STRCMP(items[1], "long") == 0) 2334 aff->af_flagtype = AFT_LONG; 2335 else if (STRCMP(items[1], "num") == 0) 2336 aff->af_flagtype = AFT_NUM; 2337 else if (STRCMP(items[1], "caplong") == 0) 2338 aff->af_flagtype = AFT_CAPLONG; 2339 else 2340 smsg(_("Invalid value for FLAG in %s line %d: %s"), 2341 fname, lnum, items[1]); 2342 if (aff->af_rare != 0 2343 || aff->af_keepcase != 0 2344 || aff->af_bad != 0 2345 || aff->af_needaffix != 0 2346 || aff->af_circumfix != 0 2347 || aff->af_needcomp != 0 2348 || aff->af_comproot != 0 2349 || aff->af_nosuggest != 0 2350 || compflags != NULL 2351 || aff->af_suff.ht_used > 0 2352 || aff->af_pref.ht_used > 0) 2353 smsg(_("FLAG after using flags in %s line %d: %s"), 2354 fname, lnum, items[1]); 2355 } 2356 else if (spell_info_item(items[0])) 2357 { 2358 p = (char_u *)getroom(spin, 2359 (spin->si_info == NULL ? 0 : STRLEN(spin->si_info)) 2360 + STRLEN(items[0]) 2361 + STRLEN(items[1]) + 3, FALSE); 2362 if (p != NULL) 2363 { 2364 if (spin->si_info != NULL) 2365 { 2366 STRCPY(p, spin->si_info); 2367 STRCAT(p, "\n"); 2368 } 2369 STRCAT(p, items[0]); 2370 STRCAT(p, " "); 2371 STRCAT(p, items[1]); 2372 spin->si_info = p; 2373 } 2374 } 2375 else if (is_aff_rule(items, itemcnt, "MIDWORD", 2) 2376 && midword == NULL) 2377 { 2378 midword = getroom_save(spin, items[1]); 2379 } 2380 else if (is_aff_rule(items, itemcnt, "TRY", 2)) 2381 { 2382 // ignored, we look in the tree for what chars may appear 2383 } 2384 // TODO: remove "RAR" later 2385 else if ((is_aff_rule(items, itemcnt, "RAR", 2) 2386 || is_aff_rule(items, itemcnt, "RARE", 2)) 2387 && aff->af_rare == 0) 2388 { 2389 aff->af_rare = affitem2flag(aff->af_flagtype, items[1], 2390 fname, lnum); 2391 } 2392 // TODO: remove "KEP" later 2393 else if ((is_aff_rule(items, itemcnt, "KEP", 2) 2394 || is_aff_rule(items, itemcnt, "KEEPCASE", 2)) 2395 && aff->af_keepcase == 0) 2396 { 2397 aff->af_keepcase = affitem2flag(aff->af_flagtype, items[1], 2398 fname, lnum); 2399 } 2400 else if ((is_aff_rule(items, itemcnt, "BAD", 2) 2401 || is_aff_rule(items, itemcnt, "FORBIDDENWORD", 2)) 2402 && aff->af_bad == 0) 2403 { 2404 aff->af_bad = affitem2flag(aff->af_flagtype, items[1], 2405 fname, lnum); 2406 } 2407 else if (is_aff_rule(items, itemcnt, "NEEDAFFIX", 2) 2408 && aff->af_needaffix == 0) 2409 { 2410 aff->af_needaffix = affitem2flag(aff->af_flagtype, items[1], 2411 fname, lnum); 2412 } 2413 else if (is_aff_rule(items, itemcnt, "CIRCUMFIX", 2) 2414 && aff->af_circumfix == 0) 2415 { 2416 aff->af_circumfix = affitem2flag(aff->af_flagtype, items[1], 2417 fname, lnum); 2418 } 2419 else if (is_aff_rule(items, itemcnt, "NOSUGGEST", 2) 2420 && aff->af_nosuggest == 0) 2421 { 2422 aff->af_nosuggest = affitem2flag(aff->af_flagtype, items[1], 2423 fname, lnum); 2424 } 2425 else if ((is_aff_rule(items, itemcnt, "NEEDCOMPOUND", 2) 2426 || is_aff_rule(items, itemcnt, "ONLYINCOMPOUND", 2)) 2427 && aff->af_needcomp == 0) 2428 { 2429 aff->af_needcomp = affitem2flag(aff->af_flagtype, items[1], 2430 fname, lnum); 2431 } 2432 else if (is_aff_rule(items, itemcnt, "COMPOUNDROOT", 2) 2433 && aff->af_comproot == 0) 2434 { 2435 aff->af_comproot = affitem2flag(aff->af_flagtype, items[1], 2436 fname, lnum); 2437 } 2438 else if (is_aff_rule(items, itemcnt, "COMPOUNDFORBIDFLAG", 2) 2439 && aff->af_compforbid == 0) 2440 { 2441 aff->af_compforbid = affitem2flag(aff->af_flagtype, items[1], 2442 fname, lnum); 2443 if (aff->af_pref.ht_used > 0) 2444 smsg(_("Defining COMPOUNDFORBIDFLAG after PFX item may give wrong results in %s line %d"), 2445 fname, lnum); 2446 } 2447 else if (is_aff_rule(items, itemcnt, "COMPOUNDPERMITFLAG", 2) 2448 && aff->af_comppermit == 0) 2449 { 2450 aff->af_comppermit = affitem2flag(aff->af_flagtype, items[1], 2451 fname, lnum); 2452 if (aff->af_pref.ht_used > 0) 2453 smsg(_("Defining COMPOUNDPERMITFLAG after PFX item may give wrong results in %s line %d"), 2454 fname, lnum); 2455 } 2456 else if (is_aff_rule(items, itemcnt, "COMPOUNDFLAG", 2) 2457 && compflags == NULL) 2458 { 2459 // Turn flag "c" into COMPOUNDRULE compatible string "c+", 2460 // "Na" into "Na+", "1234" into "1234+". 2461 p = getroom(spin, STRLEN(items[1]) + 2, FALSE); 2462 if (p != NULL) 2463 { 2464 STRCPY(p, items[1]); 2465 STRCAT(p, "+"); 2466 compflags = p; 2467 } 2468 } 2469 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULES", 2)) 2470 { 2471 // We don't use the count, but do check that it's a number and 2472 // not COMPOUNDRULE mistyped. 2473 if (atoi((char *)items[1]) == 0) 2474 smsg(_("Wrong COMPOUNDRULES value in %s line %d: %s"), 2475 fname, lnum, items[1]); 2476 } 2477 else if (is_aff_rule(items, itemcnt, "COMPOUNDRULE", 2)) 2478 { 2479 // Don't use the first rule if it is a number. 2480 if (compflags != NULL || *skipdigits(items[1]) != NUL) 2481 { 2482 // Concatenate this string to previously defined ones, 2483 // using a slash to separate them. 2484 l = (int)STRLEN(items[1]) + 1; 2485 if (compflags != NULL) 2486 l += (int)STRLEN(compflags) + 1; 2487 p = getroom(spin, l, FALSE); 2488 if (p != NULL) 2489 { 2490 if (compflags != NULL) 2491 { 2492 STRCPY(p, compflags); 2493 STRCAT(p, "/"); 2494 } 2495 STRCAT(p, items[1]); 2496 compflags = p; 2497 } 2498 } 2499 } 2500 else if (is_aff_rule(items, itemcnt, "COMPOUNDWORDMAX", 2) 2501 && compmax == 0) 2502 { 2503 compmax = atoi((char *)items[1]); 2504 if (compmax == 0) 2505 smsg(_("Wrong COMPOUNDWORDMAX value in %s line %d: %s"), 2506 fname, lnum, items[1]); 2507 } 2508 else if (is_aff_rule(items, itemcnt, "COMPOUNDMIN", 2) 2509 && compminlen == 0) 2510 { 2511 compminlen = atoi((char *)items[1]); 2512 if (compminlen == 0) 2513 smsg(_("Wrong COMPOUNDMIN value in %s line %d: %s"), 2514 fname, lnum, items[1]); 2515 } 2516 else if (is_aff_rule(items, itemcnt, "COMPOUNDSYLMAX", 2) 2517 && compsylmax == 0) 2518 { 2519 compsylmax = atoi((char *)items[1]); 2520 if (compsylmax == 0) 2521 smsg(_("Wrong COMPOUNDSYLMAX value in %s line %d: %s"), 2522 fname, lnum, items[1]); 2523 } 2524 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDDUP", 1)) 2525 { 2526 compoptions |= COMP_CHECKDUP; 2527 } 2528 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDREP", 1)) 2529 { 2530 compoptions |= COMP_CHECKREP; 2531 } 2532 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDCASE", 1)) 2533 { 2534 compoptions |= COMP_CHECKCASE; 2535 } 2536 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDTRIPLE", 1)) 2537 { 2538 compoptions |= COMP_CHECKTRIPLE; 2539 } 2540 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 2)) 2541 { 2542 if (atoi((char *)items[1]) == 0) 2543 smsg(_("Wrong CHECKCOMPOUNDPATTERN value in %s line %d: %s"), 2544 fname, lnum, items[1]); 2545 } 2546 else if (is_aff_rule(items, itemcnt, "CHECKCOMPOUNDPATTERN", 3)) 2547 { 2548 garray_T *gap = &spin->si_comppat; 2549 int i; 2550 2551 // Only add the couple if it isn't already there. 2552 for (i = 0; i < gap->ga_len - 1; i += 2) 2553 if (STRCMP(((char_u **)(gap->ga_data))[i], items[1]) == 0 2554 && STRCMP(((char_u **)(gap->ga_data))[i + 1], 2555 items[2]) == 0) 2556 break; 2557 if (i >= gap->ga_len && ga_grow(gap, 2) == OK) 2558 { 2559 ((char_u **)(gap->ga_data))[gap->ga_len++] 2560 = getroom_save(spin, items[1]); 2561 ((char_u **)(gap->ga_data))[gap->ga_len++] 2562 = getroom_save(spin, items[2]); 2563 } 2564 } 2565 else if (is_aff_rule(items, itemcnt, "SYLLABLE", 2) 2566 && syllable == NULL) 2567 { 2568 syllable = getroom_save(spin, items[1]); 2569 } 2570 else if (is_aff_rule(items, itemcnt, "NOBREAK", 1)) 2571 { 2572 spin->si_nobreak = TRUE; 2573 } 2574 else if (is_aff_rule(items, itemcnt, "NOSPLITSUGS", 1)) 2575 { 2576 spin->si_nosplitsugs = TRUE; 2577 } 2578 else if (is_aff_rule(items, itemcnt, "NOCOMPOUNDSUGS", 1)) 2579 { 2580 spin->si_nocompoundsugs = TRUE; 2581 } 2582 else if (is_aff_rule(items, itemcnt, "NOSUGFILE", 1)) 2583 { 2584 spin->si_nosugfile = TRUE; 2585 } 2586 else if (is_aff_rule(items, itemcnt, "PFXPOSTPONE", 1)) 2587 { 2588 aff->af_pfxpostpone = TRUE; 2589 } 2590 else if (is_aff_rule(items, itemcnt, "IGNOREEXTRA", 1)) 2591 { 2592 aff->af_ignoreextra = TRUE; 2593 } 2594 else if ((STRCMP(items[0], "PFX") == 0 2595 || STRCMP(items[0], "SFX") == 0) 2596 && aff_todo == 0 2597 && itemcnt >= 4) 2598 { 2599 int lasti = 4; 2600 char_u key[AH_KEY_LEN]; 2601 2602 if (*items[0] == 'P') 2603 tp = &aff->af_pref; 2604 else 2605 tp = &aff->af_suff; 2606 2607 // Myspell allows the same affix name to be used multiple 2608 // times. The affix files that do this have an undocumented 2609 // "S" flag on all but the last block, thus we check for that 2610 // and store it in ah_follows. 2611 vim_strncpy(key, items[1], AH_KEY_LEN - 1); 2612 hi = hash_find(tp, key); 2613 if (!HASHITEM_EMPTY(hi)) 2614 { 2615 cur_aff = HI2AH(hi); 2616 if (cur_aff->ah_combine != (*items[2] == 'Y')) 2617 smsg(_("Different combining flag in continued affix block in %s line %d: %s"), 2618 fname, lnum, items[1]); 2619 if (!cur_aff->ah_follows) 2620 smsg(_("Duplicate affix in %s line %d: %s"), 2621 fname, lnum, items[1]); 2622 } 2623 else 2624 { 2625 // New affix letter. 2626 cur_aff = (affheader_T *)getroom(spin, 2627 sizeof(affheader_T), TRUE); 2628 if (cur_aff == NULL) 2629 break; 2630 cur_aff->ah_flag = affitem2flag(aff->af_flagtype, items[1], 2631 fname, lnum); 2632 if (cur_aff->ah_flag == 0 || STRLEN(items[1]) >= AH_KEY_LEN) 2633 break; 2634 if (cur_aff->ah_flag == aff->af_bad 2635 || cur_aff->ah_flag == aff->af_rare 2636 || cur_aff->ah_flag == aff->af_keepcase 2637 || cur_aff->ah_flag == aff->af_needaffix 2638 || cur_aff->ah_flag == aff->af_circumfix 2639 || cur_aff->ah_flag == aff->af_nosuggest 2640 || cur_aff->ah_flag == aff->af_needcomp 2641 || cur_aff->ah_flag == aff->af_comproot) 2642 smsg(_("Affix also used for BAD/RARE/KEEPCASE/NEEDAFFIX/NEEDCOMPOUND/NOSUGGEST in %s line %d: %s"), 2643 fname, lnum, items[1]); 2644 STRCPY(cur_aff->ah_key, items[1]); 2645 hash_add(tp, cur_aff->ah_key); 2646 2647 cur_aff->ah_combine = (*items[2] == 'Y'); 2648 } 2649 2650 // Check for the "S" flag, which apparently means that another 2651 // block with the same affix name is following. 2652 if (itemcnt > lasti && STRCMP(items[lasti], "S") == 0) 2653 { 2654 ++lasti; 2655 cur_aff->ah_follows = TRUE; 2656 } 2657 else 2658 cur_aff->ah_follows = FALSE; 2659 2660 // Myspell allows extra text after the item, but that might 2661 // mean mistakes go unnoticed. Require a comment-starter. 2662 if (itemcnt > lasti && *items[lasti] != '#') 2663 smsg(_(e_afftrailing), fname, lnum, items[lasti]); 2664 2665 if (STRCMP(items[2], "Y") != 0 && STRCMP(items[2], "N") != 0) 2666 smsg(_("Expected Y or N in %s line %d: %s"), 2667 fname, lnum, items[2]); 2668 2669 if (*items[0] == 'P' && aff->af_pfxpostpone) 2670 { 2671 if (cur_aff->ah_newID == 0) 2672 { 2673 // Use a new number in the .spl file later, to be able 2674 // to handle multiple .aff files. 2675 check_renumber(spin); 2676 cur_aff->ah_newID = ++spin->si_newprefID; 2677 2678 // We only really use ah_newID if the prefix is 2679 // postponed. We know that only after handling all 2680 // the items. 2681 did_postpone_prefix = FALSE; 2682 } 2683 else 2684 // Did use the ID in a previous block. 2685 did_postpone_prefix = TRUE; 2686 } 2687 2688 aff_todo = atoi((char *)items[3]); 2689 } 2690 else if ((STRCMP(items[0], "PFX") == 0 2691 || STRCMP(items[0], "SFX") == 0) 2692 && aff_todo > 0 2693 && STRCMP(cur_aff->ah_key, items[1]) == 0 2694 && itemcnt >= 5) 2695 { 2696 affentry_T *aff_entry; 2697 int upper = FALSE; 2698 int lasti = 5; 2699 2700 // Myspell allows extra text after the item, but that might 2701 // mean mistakes go unnoticed. Require a comment-starter, 2702 // unless IGNOREEXTRA is used. Hunspell uses a "-" item. 2703 if (itemcnt > lasti 2704 && !aff->af_ignoreextra 2705 && *items[lasti] != '#' 2706 && (STRCMP(items[lasti], "-") != 0 2707 || itemcnt != lasti + 1)) 2708 smsg(_(e_afftrailing), fname, lnum, items[lasti]); 2709 2710 // New item for an affix letter. 2711 --aff_todo; 2712 aff_entry = (affentry_T *)getroom(spin, 2713 sizeof(affentry_T), TRUE); 2714 if (aff_entry == NULL) 2715 break; 2716 2717 if (STRCMP(items[2], "0") != 0) 2718 aff_entry->ae_chop = getroom_save(spin, items[2]); 2719 if (STRCMP(items[3], "0") != 0) 2720 { 2721 aff_entry->ae_add = getroom_save(spin, items[3]); 2722 2723 // Recognize flags on the affix: abcd/XYZ 2724 aff_entry->ae_flags = vim_strchr(aff_entry->ae_add, '/'); 2725 if (aff_entry->ae_flags != NULL) 2726 { 2727 *aff_entry->ae_flags++ = NUL; 2728 aff_process_flags(aff, aff_entry); 2729 } 2730 } 2731 2732 // Don't use an affix entry with non-ASCII characters when 2733 // "spin->si_ascii" is TRUE. 2734 if (!spin->si_ascii || !(has_non_ascii(aff_entry->ae_chop) 2735 || has_non_ascii(aff_entry->ae_add))) 2736 { 2737 aff_entry->ae_next = cur_aff->ah_first; 2738 cur_aff->ah_first = aff_entry; 2739 2740 if (STRCMP(items[4], ".") != 0) 2741 { 2742 char_u buf[MAXLINELEN]; 2743 2744 aff_entry->ae_cond = getroom_save(spin, items[4]); 2745 if (*items[0] == 'P') 2746 sprintf((char *)buf, "^%s", items[4]); 2747 else 2748 sprintf((char *)buf, "%s$", items[4]); 2749 aff_entry->ae_prog = vim_regcomp(buf, 2750 RE_MAGIC + RE_STRING + RE_STRICT); 2751 if (aff_entry->ae_prog == NULL) 2752 smsg(_("Broken condition in %s line %d: %s"), 2753 fname, lnum, items[4]); 2754 } 2755 2756 // For postponed prefixes we need an entry in si_prefcond 2757 // for the condition. Use an existing one if possible. 2758 // Can't be done for an affix with flags, ignoring 2759 // COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG. 2760 if (*items[0] == 'P' && aff->af_pfxpostpone 2761 && aff_entry->ae_flags == NULL) 2762 { 2763 // When the chop string is one lower-case letter and 2764 // the add string ends in the upper-case letter we set 2765 // the "upper" flag, clear "ae_chop" and remove the 2766 // letters from "ae_add". The condition must either 2767 // be empty or start with the same letter. 2768 if (aff_entry->ae_chop != NULL 2769 && aff_entry->ae_add != NULL 2770 && aff_entry->ae_chop[(*mb_ptr2len)( 2771 aff_entry->ae_chop)] == NUL) 2772 { 2773 int c, c_up; 2774 2775 c = PTR2CHAR(aff_entry->ae_chop); 2776 c_up = SPELL_TOUPPER(c); 2777 if (c_up != c 2778 && (aff_entry->ae_cond == NULL 2779 || PTR2CHAR(aff_entry->ae_cond) == c)) 2780 { 2781 p = aff_entry->ae_add 2782 + STRLEN(aff_entry->ae_add); 2783 MB_PTR_BACK(aff_entry->ae_add, p); 2784 if (PTR2CHAR(p) == c_up) 2785 { 2786 upper = TRUE; 2787 aff_entry->ae_chop = NULL; 2788 *p = NUL; 2789 2790 // The condition is matched with the 2791 // actual word, thus must check for the 2792 // upper-case letter. 2793 if (aff_entry->ae_cond != NULL) 2794 { 2795 char_u buf[MAXLINELEN]; 2796 2797 if (has_mbyte) 2798 { 2799 onecap_copy(items[4], buf, TRUE); 2800 aff_entry->ae_cond = getroom_save( 2801 spin, buf); 2802 } 2803 else 2804 *aff_entry->ae_cond = c_up; 2805 if (aff_entry->ae_cond != NULL) 2806 { 2807 sprintf((char *)buf, "^%s", 2808 aff_entry->ae_cond); 2809 vim_regfree(aff_entry->ae_prog); 2810 aff_entry->ae_prog = vim_regcomp( 2811 buf, RE_MAGIC + RE_STRING); 2812 } 2813 } 2814 } 2815 } 2816 } 2817 2818 if (aff_entry->ae_chop == NULL 2819 && aff_entry->ae_flags == NULL) 2820 { 2821 int idx; 2822 char_u **pp; 2823 int n; 2824 2825 // Find a previously used condition. 2826 for (idx = spin->si_prefcond.ga_len - 1; idx >= 0; 2827 --idx) 2828 { 2829 p = ((char_u **)spin->si_prefcond.ga_data)[idx]; 2830 if (str_equal(p, aff_entry->ae_cond)) 2831 break; 2832 } 2833 if (idx < 0 && ga_grow(&spin->si_prefcond, 1) == OK) 2834 { 2835 // Not found, add a new condition. 2836 idx = spin->si_prefcond.ga_len++; 2837 pp = ((char_u **)spin->si_prefcond.ga_data) 2838 + idx; 2839 if (aff_entry->ae_cond == NULL) 2840 *pp = NULL; 2841 else 2842 *pp = getroom_save(spin, 2843 aff_entry->ae_cond); 2844 } 2845 2846 // Add the prefix to the prefix tree. 2847 if (aff_entry->ae_add == NULL) 2848 p = (char_u *)""; 2849 else 2850 p = aff_entry->ae_add; 2851 2852 // PFX_FLAGS is a negative number, so that 2853 // tree_add_word() knows this is the prefix tree. 2854 n = PFX_FLAGS; 2855 if (!cur_aff->ah_combine) 2856 n |= WFP_NC; 2857 if (upper) 2858 n |= WFP_UP; 2859 if (aff_entry->ae_comppermit) 2860 n |= WFP_COMPPERMIT; 2861 if (aff_entry->ae_compforbid) 2862 n |= WFP_COMPFORBID; 2863 tree_add_word(spin, p, spin->si_prefroot, n, 2864 idx, cur_aff->ah_newID); 2865 did_postpone_prefix = TRUE; 2866 } 2867 2868 // Didn't actually use ah_newID, backup si_newprefID. 2869 if (aff_todo == 0 && !did_postpone_prefix) 2870 { 2871 --spin->si_newprefID; 2872 cur_aff->ah_newID = 0; 2873 } 2874 } 2875 } 2876 } 2877 else if (is_aff_rule(items, itemcnt, "FOL", 2) && fol == NULL) 2878 { 2879 fol = vim_strsave(items[1]); 2880 } 2881 else if (is_aff_rule(items, itemcnt, "LOW", 2) && low == NULL) 2882 { 2883 low = vim_strsave(items[1]); 2884 } 2885 else if (is_aff_rule(items, itemcnt, "UPP", 2) && upp == NULL) 2886 { 2887 upp = vim_strsave(items[1]); 2888 } 2889 else if (is_aff_rule(items, itemcnt, "REP", 2) 2890 || is_aff_rule(items, itemcnt, "REPSAL", 2)) 2891 { 2892 // Ignore REP/REPSAL count 2893 if (!isdigit(*items[1])) 2894 smsg(_("Expected REP(SAL) count in %s line %d"), 2895 fname, lnum); 2896 } 2897 else if ((STRCMP(items[0], "REP") == 0 2898 || STRCMP(items[0], "REPSAL") == 0) 2899 && itemcnt >= 3) 2900 { 2901 // REP/REPSAL item 2902 // Myspell ignores extra arguments, we require it starts with 2903 // # to detect mistakes. 2904 if (itemcnt > 3 && items[3][0] != '#') 2905 smsg(_(e_afftrailing), fname, lnum, items[3]); 2906 if (items[0][3] == 'S' ? do_repsal : do_rep) 2907 { 2908 // Replace underscore with space (can't include a space 2909 // directly). 2910 for (p = items[1]; *p != NUL; MB_PTR_ADV(p)) 2911 if (*p == '_') 2912 *p = ' '; 2913 for (p = items[2]; *p != NUL; MB_PTR_ADV(p)) 2914 if (*p == '_') 2915 *p = ' '; 2916 add_fromto(spin, items[0][3] == 'S' 2917 ? &spin->si_repsal 2918 : &spin->si_rep, items[1], items[2]); 2919 } 2920 } 2921 else if (is_aff_rule(items, itemcnt, "MAP", 2)) 2922 { 2923 // MAP item or count 2924 if (!found_map) 2925 { 2926 // First line contains the count. 2927 found_map = TRUE; 2928 if (!isdigit(*items[1])) 2929 smsg(_("Expected MAP count in %s line %d"), 2930 fname, lnum); 2931 } 2932 else if (do_mapline) 2933 { 2934 int c; 2935 2936 // Check that every character appears only once. 2937 for (p = items[1]; *p != NUL; ) 2938 { 2939 c = mb_ptr2char_adv(&p); 2940 if ((spin->si_map.ga_len > 0 2941 && vim_strchr(spin->si_map.ga_data, c) 2942 != NULL) 2943 || vim_strchr(p, c) != NULL) 2944 smsg(_("Duplicate character in MAP in %s line %d"), 2945 fname, lnum); 2946 } 2947 2948 // We simply concatenate all the MAP strings, separated by 2949 // slashes. 2950 ga_concat(&spin->si_map, items[1]); 2951 ga_append(&spin->si_map, '/'); 2952 } 2953 } 2954 // Accept "SAL from to" and "SAL from to #comment". 2955 else if (is_aff_rule(items, itemcnt, "SAL", 3)) 2956 { 2957 if (do_sal) 2958 { 2959 // SAL item (sounds-a-like) 2960 // Either one of the known keys or a from-to pair. 2961 if (STRCMP(items[1], "followup") == 0) 2962 spin->si_followup = sal_to_bool(items[2]); 2963 else if (STRCMP(items[1], "collapse_result") == 0) 2964 spin->si_collapse = sal_to_bool(items[2]); 2965 else if (STRCMP(items[1], "remove_accents") == 0) 2966 spin->si_rem_accents = sal_to_bool(items[2]); 2967 else 2968 // when "to" is "_" it means empty 2969 add_fromto(spin, &spin->si_sal, items[1], 2970 STRCMP(items[2], "_") == 0 ? (char_u *)"" 2971 : items[2]); 2972 } 2973 } 2974 else if (is_aff_rule(items, itemcnt, "SOFOFROM", 2) 2975 && sofofrom == NULL) 2976 { 2977 sofofrom = getroom_save(spin, items[1]); 2978 } 2979 else if (is_aff_rule(items, itemcnt, "SOFOTO", 2) 2980 && sofoto == NULL) 2981 { 2982 sofoto = getroom_save(spin, items[1]); 2983 } 2984 else if (STRCMP(items[0], "COMMON") == 0) 2985 { 2986 int i; 2987 2988 for (i = 1; i < itemcnt; ++i) 2989 { 2990 if (HASHITEM_EMPTY(hash_find(&spin->si_commonwords, 2991 items[i]))) 2992 { 2993 p = vim_strsave(items[i]); 2994 if (p == NULL) 2995 break; 2996 hash_add(&spin->si_commonwords, p); 2997 } 2998 } 2999 } 3000 else 3001 smsg(_("Unrecognized or duplicate item in %s line %d: %s"), 3002 fname, lnum, items[0]); 3003 } 3004 } 3005 3006 if (fol != NULL || low != NULL || upp != NULL) 3007 { 3008 if (spin->si_clear_chartab) 3009 { 3010 // Clear the char type tables, don't want to use any of the 3011 // currently used spell properties. 3012 init_spell_chartab(); 3013 spin->si_clear_chartab = FALSE; 3014 } 3015 3016 /* 3017 * Don't write a word table for an ASCII file, so that we don't check 3018 * for conflicts with a word table that matches 'encoding'. 3019 * Don't write one for utf-8 either, we use utf_*() and 3020 * mb_get_class(), the list of chars in the file will be incomplete. 3021 */ 3022 if (!spin->si_ascii && !enc_utf8) 3023 { 3024 if (fol == NULL || low == NULL || upp == NULL) 3025 smsg(_("Missing FOL/LOW/UPP line in %s"), fname); 3026 else 3027 (void)set_spell_chartab(fol, low, upp); 3028 } 3029 3030 vim_free(fol); 3031 vim_free(low); 3032 vim_free(upp); 3033 } 3034 3035 // Use compound specifications of the .aff file for the spell info. 3036 if (compmax != 0) 3037 { 3038 aff_check_number(spin->si_compmax, compmax, "COMPOUNDWORDMAX"); 3039 spin->si_compmax = compmax; 3040 } 3041 3042 if (compminlen != 0) 3043 { 3044 aff_check_number(spin->si_compminlen, compminlen, "COMPOUNDMIN"); 3045 spin->si_compminlen = compminlen; 3046 } 3047 3048 if (compsylmax != 0) 3049 { 3050 if (syllable == NULL) 3051 smsg(_("COMPOUNDSYLMAX used without SYLLABLE")); 3052 aff_check_number(spin->si_compsylmax, compsylmax, "COMPOUNDSYLMAX"); 3053 spin->si_compsylmax = compsylmax; 3054 } 3055 3056 if (compoptions != 0) 3057 { 3058 aff_check_number(spin->si_compoptions, compoptions, "COMPOUND options"); 3059 spin->si_compoptions |= compoptions; 3060 } 3061 3062 if (compflags != NULL) 3063 process_compflags(spin, aff, compflags); 3064 3065 // Check that we didn't use too many renumbered flags. 3066 if (spin->si_newcompID < spin->si_newprefID) 3067 { 3068 if (spin->si_newcompID == 127 || spin->si_newcompID == 255) 3069 msg(_("Too many postponed prefixes")); 3070 else if (spin->si_newprefID == 0 || spin->si_newprefID == 127) 3071 msg(_("Too many compound flags")); 3072 else 3073 msg(_("Too many postponed prefixes and/or compound flags")); 3074 } 3075 3076 if (syllable != NULL) 3077 { 3078 aff_check_string(spin->si_syllable, syllable, "SYLLABLE"); 3079 spin->si_syllable = syllable; 3080 } 3081 3082 if (sofofrom != NULL || sofoto != NULL) 3083 { 3084 if (sofofrom == NULL || sofoto == NULL) 3085 smsg(_("Missing SOFO%s line in %s"), 3086 sofofrom == NULL ? "FROM" : "TO", fname); 3087 else if (spin->si_sal.ga_len > 0) 3088 smsg(_("Both SAL and SOFO lines in %s"), fname); 3089 else 3090 { 3091 aff_check_string(spin->si_sofofr, sofofrom, "SOFOFROM"); 3092 aff_check_string(spin->si_sofoto, sofoto, "SOFOTO"); 3093 spin->si_sofofr = sofofrom; 3094 spin->si_sofoto = sofoto; 3095 } 3096 } 3097 3098 if (midword != NULL) 3099 { 3100 aff_check_string(spin->si_midword, midword, "MIDWORD"); 3101 spin->si_midword = midword; 3102 } 3103 3104 vim_free(pc); 3105 fclose(fd); 3106 return aff; 3107 } 3108 3109 /* 3110 * Return TRUE when items[0] equals "rulename", there are "mincount" items or 3111 * a comment is following after item "mincount". 3112 */ 3113 static int 3114 is_aff_rule( 3115 char_u **items, 3116 int itemcnt, 3117 char *rulename, 3118 int mincount) 3119 { 3120 return (STRCMP(items[0], rulename) == 0 3121 && (itemcnt == mincount 3122 || (itemcnt > mincount && items[mincount][0] == '#'))); 3123 } 3124 3125 /* 3126 * For affix "entry" move COMPOUNDFORBIDFLAG and COMPOUNDPERMITFLAG from 3127 * ae_flags to ae_comppermit and ae_compforbid. 3128 */ 3129 static void 3130 aff_process_flags(afffile_T *affile, affentry_T *entry) 3131 { 3132 char_u *p; 3133 char_u *prevp; 3134 unsigned flag; 3135 3136 if (entry->ae_flags != NULL 3137 && (affile->af_compforbid != 0 || affile->af_comppermit != 0)) 3138 { 3139 for (p = entry->ae_flags; *p != NUL; ) 3140 { 3141 prevp = p; 3142 flag = get_affitem(affile->af_flagtype, &p); 3143 if (flag == affile->af_comppermit || flag == affile->af_compforbid) 3144 { 3145 STRMOVE(prevp, p); 3146 p = prevp; 3147 if (flag == affile->af_comppermit) 3148 entry->ae_comppermit = TRUE; 3149 else 3150 entry->ae_compforbid = TRUE; 3151 } 3152 if (affile->af_flagtype == AFT_NUM && *p == ',') 3153 ++p; 3154 } 3155 if (*entry->ae_flags == NUL) 3156 entry->ae_flags = NULL; // nothing left 3157 } 3158 } 3159 3160 /* 3161 * Return TRUE if "s" is the name of an info item in the affix file. 3162 */ 3163 static int 3164 spell_info_item(char_u *s) 3165 { 3166 return STRCMP(s, "NAME") == 0 3167 || STRCMP(s, "HOME") == 0 3168 || STRCMP(s, "VERSION") == 0 3169 || STRCMP(s, "AUTHOR") == 0 3170 || STRCMP(s, "EMAIL") == 0 3171 || STRCMP(s, "COPYRIGHT") == 0; 3172 } 3173 3174 /* 3175 * Turn an affix flag name into a number, according to the FLAG type. 3176 * returns zero for failure. 3177 */ 3178 static unsigned 3179 affitem2flag( 3180 int flagtype, 3181 char_u *item, 3182 char_u *fname, 3183 int lnum) 3184 { 3185 unsigned res; 3186 char_u *p = item; 3187 3188 res = get_affitem(flagtype, &p); 3189 if (res == 0) 3190 { 3191 if (flagtype == AFT_NUM) 3192 smsg(_("Flag is not a number in %s line %d: %s"), 3193 fname, lnum, item); 3194 else 3195 smsg(_("Illegal flag in %s line %d: %s"), 3196 fname, lnum, item); 3197 } 3198 if (*p != NUL) 3199 { 3200 smsg(_(e_affname), fname, lnum, item); 3201 return 0; 3202 } 3203 3204 return res; 3205 } 3206 3207 /* 3208 * Get one affix name from "*pp" and advance the pointer. 3209 * Returns ZERO_FLAG for "0". 3210 * Returns zero for an error, still advances the pointer then. 3211 */ 3212 static unsigned 3213 get_affitem(int flagtype, char_u **pp) 3214 { 3215 int res; 3216 3217 if (flagtype == AFT_NUM) 3218 { 3219 if (!VIM_ISDIGIT(**pp)) 3220 { 3221 ++*pp; // always advance, avoid getting stuck 3222 return 0; 3223 } 3224 res = getdigits(pp); 3225 if (res == 0) 3226 res = ZERO_FLAG; 3227 } 3228 else 3229 { 3230 res = mb_ptr2char_adv(pp); 3231 if (flagtype == AFT_LONG || (flagtype == AFT_CAPLONG 3232 && res >= 'A' && res <= 'Z')) 3233 { 3234 if (**pp == NUL) 3235 return 0; 3236 res = mb_ptr2char_adv(pp) + (res << 16); 3237 } 3238 } 3239 return res; 3240 } 3241 3242 /* 3243 * Process the "compflags" string used in an affix file and append it to 3244 * spin->si_compflags. 3245 * The processing involves changing the affix names to ID numbers, so that 3246 * they fit in one byte. 3247 */ 3248 static void 3249 process_compflags( 3250 spellinfo_T *spin, 3251 afffile_T *aff, 3252 char_u *compflags) 3253 { 3254 char_u *p; 3255 char_u *prevp; 3256 unsigned flag; 3257 compitem_T *ci; 3258 int id; 3259 int len; 3260 char_u *tp; 3261 char_u key[AH_KEY_LEN]; 3262 hashitem_T *hi; 3263 3264 // Make room for the old and the new compflags, concatenated with a / in 3265 // between. Processing it makes it shorter, but we don't know by how 3266 // much, thus allocate the maximum. 3267 len = (int)STRLEN(compflags) + 1; 3268 if (spin->si_compflags != NULL) 3269 len += (int)STRLEN(spin->si_compflags) + 1; 3270 p = getroom(spin, len, FALSE); 3271 if (p == NULL) 3272 return; 3273 if (spin->si_compflags != NULL) 3274 { 3275 STRCPY(p, spin->si_compflags); 3276 STRCAT(p, "/"); 3277 } 3278 spin->si_compflags = p; 3279 tp = p + STRLEN(p); 3280 3281 for (p = compflags; *p != NUL; ) 3282 { 3283 if (vim_strchr((char_u *)"/?*+[]", *p) != NULL) 3284 // Copy non-flag characters directly. 3285 *tp++ = *p++; 3286 else 3287 { 3288 // First get the flag number, also checks validity. 3289 prevp = p; 3290 flag = get_affitem(aff->af_flagtype, &p); 3291 if (flag != 0) 3292 { 3293 // Find the flag in the hashtable. If it was used before, use 3294 // the existing ID. Otherwise add a new entry. 3295 vim_strncpy(key, prevp, p - prevp); 3296 hi = hash_find(&aff->af_comp, key); 3297 if (!HASHITEM_EMPTY(hi)) 3298 id = HI2CI(hi)->ci_newID; 3299 else 3300 { 3301 ci = (compitem_T *)getroom(spin, sizeof(compitem_T), TRUE); 3302 if (ci == NULL) 3303 break; 3304 STRCPY(ci->ci_key, key); 3305 ci->ci_flag = flag; 3306 // Avoid using a flag ID that has a special meaning in a 3307 // regexp (also inside []). 3308 do 3309 { 3310 check_renumber(spin); 3311 id = spin->si_newcompID--; 3312 } while (vim_strchr((char_u *)"/?*+[]\\-^", id) != NULL); 3313 ci->ci_newID = id; 3314 hash_add(&aff->af_comp, ci->ci_key); 3315 } 3316 *tp++ = id; 3317 } 3318 if (aff->af_flagtype == AFT_NUM && *p == ',') 3319 ++p; 3320 } 3321 } 3322 3323 *tp = NUL; 3324 } 3325 3326 /* 3327 * Check that the new IDs for postponed affixes and compounding don't overrun 3328 * each other. We have almost 255 available, but start at 0-127 to avoid 3329 * using two bytes for utf-8. When the 0-127 range is used up go to 128-255. 3330 * When that is used up an error message is given. 3331 */ 3332 static void 3333 check_renumber(spellinfo_T *spin) 3334 { 3335 if (spin->si_newprefID == spin->si_newcompID && spin->si_newcompID < 128) 3336 { 3337 spin->si_newprefID = 127; 3338 spin->si_newcompID = 255; 3339 } 3340 } 3341 3342 /* 3343 * Return TRUE if flag "flag" appears in affix list "afflist". 3344 */ 3345 static int 3346 flag_in_afflist(int flagtype, char_u *afflist, unsigned flag) 3347 { 3348 char_u *p; 3349 unsigned n; 3350 3351 switch (flagtype) 3352 { 3353 case AFT_CHAR: 3354 return vim_strchr(afflist, flag) != NULL; 3355 3356 case AFT_CAPLONG: 3357 case AFT_LONG: 3358 for (p = afflist; *p != NUL; ) 3359 { 3360 n = mb_ptr2char_adv(&p); 3361 if ((flagtype == AFT_LONG || (n >= 'A' && n <= 'Z')) 3362 && *p != NUL) 3363 n = mb_ptr2char_adv(&p) + (n << 16); 3364 if (n == flag) 3365 return TRUE; 3366 } 3367 break; 3368 3369 case AFT_NUM: 3370 for (p = afflist; *p != NUL; ) 3371 { 3372 n = getdigits(&p); 3373 if (n == 0) 3374 n = ZERO_FLAG; 3375 if (n == flag) 3376 return TRUE; 3377 if (*p != NUL) // skip over comma 3378 ++p; 3379 } 3380 break; 3381 } 3382 return FALSE; 3383 } 3384 3385 /* 3386 * Give a warning when "spinval" and "affval" numbers are set and not the same. 3387 */ 3388 static void 3389 aff_check_number(int spinval, int affval, char *name) 3390 { 3391 if (spinval != 0 && spinval != affval) 3392 smsg(_("%s value differs from what is used in another .aff file"), name); 3393 } 3394 3395 /* 3396 * Give a warning when "spinval" and "affval" strings are set and not the same. 3397 */ 3398 static void 3399 aff_check_string(char_u *spinval, char_u *affval, char *name) 3400 { 3401 if (spinval != NULL && STRCMP(spinval, affval) != 0) 3402 smsg(_("%s value differs from what is used in another .aff file"), name); 3403 } 3404 3405 /* 3406 * Return TRUE if strings "s1" and "s2" are equal. Also consider both being 3407 * NULL as equal. 3408 */ 3409 static int 3410 str_equal(char_u *s1, char_u *s2) 3411 { 3412 if (s1 == NULL || s2 == NULL) 3413 return s1 == s2; 3414 return STRCMP(s1, s2) == 0; 3415 } 3416 3417 /* 3418 * Add a from-to item to "gap". Used for REP and SAL items. 3419 * They are stored case-folded. 3420 */ 3421 static void 3422 add_fromto( 3423 spellinfo_T *spin, 3424 garray_T *gap, 3425 char_u *from, 3426 char_u *to) 3427 { 3428 fromto_T *ftp; 3429 char_u word[MAXWLEN]; 3430 3431 if (ga_grow(gap, 1) == OK) 3432 { 3433 ftp = ((fromto_T *)gap->ga_data) + gap->ga_len; 3434 (void)spell_casefold(curwin, from, (int)STRLEN(from), word, MAXWLEN); 3435 ftp->ft_from = getroom_save(spin, word); 3436 (void)spell_casefold(curwin, to, (int)STRLEN(to), word, MAXWLEN); 3437 ftp->ft_to = getroom_save(spin, word); 3438 ++gap->ga_len; 3439 } 3440 } 3441 3442 /* 3443 * Convert a boolean argument in a SAL line to TRUE or FALSE; 3444 */ 3445 static int 3446 sal_to_bool(char_u *s) 3447 { 3448 return STRCMP(s, "1") == 0 || STRCMP(s, "true") == 0; 3449 } 3450 3451 /* 3452 * Free the structure filled by spell_read_aff(). 3453 */ 3454 static void 3455 spell_free_aff(afffile_T *aff) 3456 { 3457 hashtab_T *ht; 3458 hashitem_T *hi; 3459 int todo; 3460 affheader_T *ah; 3461 affentry_T *ae; 3462 3463 vim_free(aff->af_enc); 3464 3465 // All this trouble to free the "ae_prog" items... 3466 for (ht = &aff->af_pref; ; ht = &aff->af_suff) 3467 { 3468 todo = (int)ht->ht_used; 3469 for (hi = ht->ht_array; todo > 0; ++hi) 3470 { 3471 if (!HASHITEM_EMPTY(hi)) 3472 { 3473 --todo; 3474 ah = HI2AH(hi); 3475 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 3476 vim_regfree(ae->ae_prog); 3477 } 3478 } 3479 if (ht == &aff->af_suff) 3480 break; 3481 } 3482 3483 hash_clear(&aff->af_pref); 3484 hash_clear(&aff->af_suff); 3485 hash_clear(&aff->af_comp); 3486 } 3487 3488 /* 3489 * Read dictionary file "fname". 3490 * Returns OK or FAIL; 3491 */ 3492 static int 3493 spell_read_dic(spellinfo_T *spin, char_u *fname, afffile_T *affile) 3494 { 3495 hashtab_T ht; 3496 char_u line[MAXLINELEN]; 3497 char_u *p; 3498 char_u *afflist; 3499 char_u store_afflist[MAXWLEN]; 3500 int pfxlen; 3501 int need_affix; 3502 char_u *dw; 3503 char_u *pc; 3504 char_u *w; 3505 int l; 3506 hash_T hash; 3507 hashitem_T *hi; 3508 FILE *fd; 3509 int lnum = 1; 3510 int non_ascii = 0; 3511 int retval = OK; 3512 char_u message[MAXLINELEN + MAXWLEN]; 3513 int flags; 3514 int duplicate = 0; 3515 time_T last_msg_time = 0; 3516 3517 /* 3518 * Open the file. 3519 */ 3520 fd = mch_fopen((char *)fname, "r"); 3521 if (fd == NULL) 3522 { 3523 semsg(_(e_notopen), fname); 3524 return FAIL; 3525 } 3526 3527 // The hashtable is only used to detect duplicated words. 3528 hash_init(&ht); 3529 3530 vim_snprintf((char *)IObuff, IOSIZE, 3531 _("Reading dictionary file %s..."), fname); 3532 spell_message(spin, IObuff); 3533 3534 // start with a message for the first line 3535 spin->si_msg_count = 999999; 3536 3537 // Read and ignore the first line: word count. 3538 if (vim_fgets(line, MAXLINELEN, fd) || !vim_isdigit(*skipwhite(line))) 3539 semsg(_("E760: No word count in %s"), fname); 3540 3541 /* 3542 * Read all the lines in the file one by one. 3543 * The words are converted to 'encoding' here, before being added to 3544 * the hashtable. 3545 */ 3546 while (!vim_fgets(line, MAXLINELEN, fd) && !got_int) 3547 { 3548 line_breakcheck(); 3549 ++lnum; 3550 if (line[0] == '#' || line[0] == '/') 3551 continue; // comment line 3552 3553 // Remove CR, LF and white space from the end. White space halfway 3554 // the word is kept to allow e.g., "et al.". 3555 l = (int)STRLEN(line); 3556 while (l > 0 && line[l - 1] <= ' ') 3557 --l; 3558 if (l == 0) 3559 continue; // empty line 3560 line[l] = NUL; 3561 3562 // Convert from "SET" to 'encoding' when needed. 3563 if (spin->si_conv.vc_type != CONV_NONE) 3564 { 3565 pc = string_convert(&spin->si_conv, line, NULL); 3566 if (pc == NULL) 3567 { 3568 smsg(_("Conversion failure for word in %s line %d: %s"), 3569 fname, lnum, line); 3570 continue; 3571 } 3572 w = pc; 3573 } 3574 else 3575 { 3576 pc = NULL; 3577 w = line; 3578 } 3579 3580 // Truncate the word at the "/", set "afflist" to what follows. 3581 // Replace "\/" by "/" and "\\" by "\". 3582 afflist = NULL; 3583 for (p = w; *p != NUL; MB_PTR_ADV(p)) 3584 { 3585 if (*p == '\\' && (p[1] == '\\' || p[1] == '/')) 3586 STRMOVE(p, p + 1); 3587 else if (*p == '/') 3588 { 3589 *p = NUL; 3590 afflist = p + 1; 3591 break; 3592 } 3593 } 3594 3595 // Skip non-ASCII words when "spin->si_ascii" is TRUE. 3596 if (spin->si_ascii && has_non_ascii(w)) 3597 { 3598 ++non_ascii; 3599 vim_free(pc); 3600 continue; 3601 } 3602 3603 // This takes time, print a message every 10000 words, but not more 3604 // often than once per second. 3605 if (spin->si_verbose && spin->si_msg_count > 10000) 3606 { 3607 spin->si_msg_count = 0; 3608 if (vim_time() > last_msg_time) 3609 { 3610 last_msg_time = vim_time(); 3611 vim_snprintf((char *)message, sizeof(message), 3612 _("line %6d, word %6ld - %s"), 3613 lnum, spin->si_foldwcount + spin->si_keepwcount, w); 3614 msg_start(); 3615 msg_outtrans_long_attr(message, 0); 3616 msg_clr_eos(); 3617 msg_didout = FALSE; 3618 msg_col = 0; 3619 out_flush(); 3620 } 3621 } 3622 3623 // Store the word in the hashtable to be able to find duplicates. 3624 dw = (char_u *)getroom_save(spin, w); 3625 if (dw == NULL) 3626 { 3627 retval = FAIL; 3628 vim_free(pc); 3629 break; 3630 } 3631 3632 hash = hash_hash(dw); 3633 hi = hash_lookup(&ht, dw, hash); 3634 if (!HASHITEM_EMPTY(hi)) 3635 { 3636 if (p_verbose > 0) 3637 smsg(_("Duplicate word in %s line %d: %s"), 3638 fname, lnum, dw); 3639 else if (duplicate == 0) 3640 smsg(_("First duplicate word in %s line %d: %s"), 3641 fname, lnum, dw); 3642 ++duplicate; 3643 } 3644 else 3645 hash_add_item(&ht, hi, dw, hash); 3646 3647 flags = 0; 3648 store_afflist[0] = NUL; 3649 pfxlen = 0; 3650 need_affix = FALSE; 3651 if (afflist != NULL) 3652 { 3653 // Extract flags from the affix list. 3654 flags |= get_affix_flags(affile, afflist); 3655 3656 if (affile->af_needaffix != 0 && flag_in_afflist( 3657 affile->af_flagtype, afflist, affile->af_needaffix)) 3658 need_affix = TRUE; 3659 3660 if (affile->af_pfxpostpone) 3661 // Need to store the list of prefix IDs with the word. 3662 pfxlen = get_pfxlist(affile, afflist, store_afflist); 3663 3664 if (spin->si_compflags != NULL) 3665 // Need to store the list of compound flags with the word. 3666 // Concatenate them to the list of prefix IDs. 3667 get_compflags(affile, afflist, store_afflist + pfxlen); 3668 } 3669 3670 // Add the word to the word tree(s). 3671 if (store_word(spin, dw, flags, spin->si_region, 3672 store_afflist, need_affix) == FAIL) 3673 retval = FAIL; 3674 3675 if (afflist != NULL) 3676 { 3677 // Find all matching suffixes and add the resulting words. 3678 // Additionally do matching prefixes that combine. 3679 if (store_aff_word(spin, dw, afflist, affile, 3680 &affile->af_suff, &affile->af_pref, 3681 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 3682 retval = FAIL; 3683 3684 // Find all matching prefixes and add the resulting words. 3685 if (store_aff_word(spin, dw, afflist, affile, 3686 &affile->af_pref, NULL, 3687 CONDIT_SUF, flags, store_afflist, pfxlen) == FAIL) 3688 retval = FAIL; 3689 } 3690 3691 vim_free(pc); 3692 } 3693 3694 if (duplicate > 0) 3695 smsg(_("%d duplicate word(s) in %s"), duplicate, fname); 3696 if (spin->si_ascii && non_ascii > 0) 3697 smsg(_("Ignored %d word(s) with non-ASCII characters in %s"), 3698 non_ascii, fname); 3699 hash_clear(&ht); 3700 3701 fclose(fd); 3702 return retval; 3703 } 3704 3705 /* 3706 * Check for affix flags in "afflist" that are turned into word flags. 3707 * Return WF_ flags. 3708 */ 3709 static int 3710 get_affix_flags(afffile_T *affile, char_u *afflist) 3711 { 3712 int flags = 0; 3713 3714 if (affile->af_keepcase != 0 && flag_in_afflist( 3715 affile->af_flagtype, afflist, affile->af_keepcase)) 3716 flags |= WF_KEEPCAP | WF_FIXCAP; 3717 if (affile->af_rare != 0 && flag_in_afflist( 3718 affile->af_flagtype, afflist, affile->af_rare)) 3719 flags |= WF_RARE; 3720 if (affile->af_bad != 0 && flag_in_afflist( 3721 affile->af_flagtype, afflist, affile->af_bad)) 3722 flags |= WF_BANNED; 3723 if (affile->af_needcomp != 0 && flag_in_afflist( 3724 affile->af_flagtype, afflist, affile->af_needcomp)) 3725 flags |= WF_NEEDCOMP; 3726 if (affile->af_comproot != 0 && flag_in_afflist( 3727 affile->af_flagtype, afflist, affile->af_comproot)) 3728 flags |= WF_COMPROOT; 3729 if (affile->af_nosuggest != 0 && flag_in_afflist( 3730 affile->af_flagtype, afflist, affile->af_nosuggest)) 3731 flags |= WF_NOSUGGEST; 3732 return flags; 3733 } 3734 3735 /* 3736 * Get the list of prefix IDs from the affix list "afflist". 3737 * Used for PFXPOSTPONE. 3738 * Put the resulting flags in "store_afflist[MAXWLEN]" with a terminating NUL 3739 * and return the number of affixes. 3740 */ 3741 static int 3742 get_pfxlist( 3743 afffile_T *affile, 3744 char_u *afflist, 3745 char_u *store_afflist) 3746 { 3747 char_u *p; 3748 char_u *prevp; 3749 int cnt = 0; 3750 int id; 3751 char_u key[AH_KEY_LEN]; 3752 hashitem_T *hi; 3753 3754 for (p = afflist; *p != NUL; ) 3755 { 3756 prevp = p; 3757 if (get_affitem(affile->af_flagtype, &p) != 0) 3758 { 3759 // A flag is a postponed prefix flag if it appears in "af_pref" 3760 // and its ID is not zero. 3761 vim_strncpy(key, prevp, p - prevp); 3762 hi = hash_find(&affile->af_pref, key); 3763 if (!HASHITEM_EMPTY(hi)) 3764 { 3765 id = HI2AH(hi)->ah_newID; 3766 if (id != 0) 3767 store_afflist[cnt++] = id; 3768 } 3769 } 3770 if (affile->af_flagtype == AFT_NUM && *p == ',') 3771 ++p; 3772 } 3773 3774 store_afflist[cnt] = NUL; 3775 return cnt; 3776 } 3777 3778 /* 3779 * Get the list of compound IDs from the affix list "afflist" that are used 3780 * for compound words. 3781 * Puts the flags in "store_afflist[]". 3782 */ 3783 static void 3784 get_compflags( 3785 afffile_T *affile, 3786 char_u *afflist, 3787 char_u *store_afflist) 3788 { 3789 char_u *p; 3790 char_u *prevp; 3791 int cnt = 0; 3792 char_u key[AH_KEY_LEN]; 3793 hashitem_T *hi; 3794 3795 for (p = afflist; *p != NUL; ) 3796 { 3797 prevp = p; 3798 if (get_affitem(affile->af_flagtype, &p) != 0) 3799 { 3800 // A flag is a compound flag if it appears in "af_comp". 3801 vim_strncpy(key, prevp, p - prevp); 3802 hi = hash_find(&affile->af_comp, key); 3803 if (!HASHITEM_EMPTY(hi)) 3804 store_afflist[cnt++] = HI2CI(hi)->ci_newID; 3805 } 3806 if (affile->af_flagtype == AFT_NUM && *p == ',') 3807 ++p; 3808 } 3809 3810 store_afflist[cnt] = NUL; 3811 } 3812 3813 /* 3814 * Apply affixes to a word and store the resulting words. 3815 * "ht" is the hashtable with affentry_T that need to be applied, either 3816 * prefixes or suffixes. 3817 * "xht", when not NULL, is the prefix hashtable, to be used additionally on 3818 * the resulting words for combining affixes. 3819 * 3820 * Returns FAIL when out of memory. 3821 */ 3822 static int 3823 store_aff_word( 3824 spellinfo_T *spin, // spell info 3825 char_u *word, // basic word start 3826 char_u *afflist, // list of names of supported affixes 3827 afffile_T *affile, 3828 hashtab_T *ht, 3829 hashtab_T *xht, 3830 int condit, // CONDIT_SUF et al. 3831 int flags, // flags for the word 3832 char_u *pfxlist, // list of prefix IDs 3833 int pfxlen) // nr of flags in "pfxlist" for prefixes, rest 3834 // is compound flags 3835 { 3836 int todo; 3837 hashitem_T *hi; 3838 affheader_T *ah; 3839 affentry_T *ae; 3840 char_u newword[MAXWLEN]; 3841 int retval = OK; 3842 int i, j; 3843 char_u *p; 3844 int use_flags; 3845 char_u *use_pfxlist; 3846 int use_pfxlen; 3847 int need_affix; 3848 char_u store_afflist[MAXWLEN]; 3849 char_u pfx_pfxlist[MAXWLEN]; 3850 size_t wordlen = STRLEN(word); 3851 int use_condit; 3852 3853 todo = (int)ht->ht_used; 3854 for (hi = ht->ht_array; todo > 0 && retval == OK; ++hi) 3855 { 3856 if (!HASHITEM_EMPTY(hi)) 3857 { 3858 --todo; 3859 ah = HI2AH(hi); 3860 3861 // Check that the affix combines, if required, and that the word 3862 // supports this affix. 3863 if (((condit & CONDIT_COMB) == 0 || ah->ah_combine) 3864 && flag_in_afflist(affile->af_flagtype, afflist, 3865 ah->ah_flag)) 3866 { 3867 // Loop over all affix entries with this name. 3868 for (ae = ah->ah_first; ae != NULL; ae = ae->ae_next) 3869 { 3870 // Check the condition. It's not logical to match case 3871 // here, but it is required for compatibility with 3872 // Myspell. 3873 // Another requirement from Myspell is that the chop 3874 // string is shorter than the word itself. 3875 // For prefixes, when "PFXPOSTPONE" was used, only do 3876 // prefixes with a chop string and/or flags. 3877 // When a previously added affix had CIRCUMFIX this one 3878 // must have it too, if it had not then this one must not 3879 // have one either. 3880 if ((xht != NULL || !affile->af_pfxpostpone 3881 || ae->ae_chop != NULL 3882 || ae->ae_flags != NULL) 3883 && (ae->ae_chop == NULL 3884 || STRLEN(ae->ae_chop) < wordlen) 3885 && (ae->ae_prog == NULL 3886 || vim_regexec_prog(&ae->ae_prog, FALSE, 3887 word, (colnr_T)0)) 3888 && (((condit & CONDIT_CFIX) == 0) 3889 == ((condit & CONDIT_AFF) == 0 3890 || ae->ae_flags == NULL 3891 || !flag_in_afflist(affile->af_flagtype, 3892 ae->ae_flags, affile->af_circumfix)))) 3893 { 3894 // Match. Remove the chop and add the affix. 3895 if (xht == NULL) 3896 { 3897 // prefix: chop/add at the start of the word 3898 if (ae->ae_add == NULL) 3899 *newword = NUL; 3900 else 3901 vim_strncpy(newword, ae->ae_add, MAXWLEN - 1); 3902 p = word; 3903 if (ae->ae_chop != NULL) 3904 { 3905 // Skip chop string. 3906 if (has_mbyte) 3907 { 3908 i = mb_charlen(ae->ae_chop); 3909 for ( ; i > 0; --i) 3910 MB_PTR_ADV(p); 3911 } 3912 else 3913 p += STRLEN(ae->ae_chop); 3914 } 3915 STRCAT(newword, p); 3916 } 3917 else 3918 { 3919 // suffix: chop/add at the end of the word 3920 vim_strncpy(newword, word, MAXWLEN - 1); 3921 if (ae->ae_chop != NULL) 3922 { 3923 // Remove chop string. 3924 p = newword + STRLEN(newword); 3925 i = (int)MB_CHARLEN(ae->ae_chop); 3926 for ( ; i > 0; --i) 3927 MB_PTR_BACK(newword, p); 3928 *p = NUL; 3929 } 3930 if (ae->ae_add != NULL) 3931 STRCAT(newword, ae->ae_add); 3932 } 3933 3934 use_flags = flags; 3935 use_pfxlist = pfxlist; 3936 use_pfxlen = pfxlen; 3937 need_affix = FALSE; 3938 use_condit = condit | CONDIT_COMB | CONDIT_AFF; 3939 if (ae->ae_flags != NULL) 3940 { 3941 // Extract flags from the affix list. 3942 use_flags |= get_affix_flags(affile, ae->ae_flags); 3943 3944 if (affile->af_needaffix != 0 && flag_in_afflist( 3945 affile->af_flagtype, ae->ae_flags, 3946 affile->af_needaffix)) 3947 need_affix = TRUE; 3948 3949 // When there is a CIRCUMFIX flag the other affix 3950 // must also have it and we don't add the word 3951 // with one affix. 3952 if (affile->af_circumfix != 0 && flag_in_afflist( 3953 affile->af_flagtype, ae->ae_flags, 3954 affile->af_circumfix)) 3955 { 3956 use_condit |= CONDIT_CFIX; 3957 if ((condit & CONDIT_CFIX) == 0) 3958 need_affix = TRUE; 3959 } 3960 3961 if (affile->af_pfxpostpone 3962 || spin->si_compflags != NULL) 3963 { 3964 if (affile->af_pfxpostpone) 3965 // Get prefix IDS from the affix list. 3966 use_pfxlen = get_pfxlist(affile, 3967 ae->ae_flags, store_afflist); 3968 else 3969 use_pfxlen = 0; 3970 use_pfxlist = store_afflist; 3971 3972 // Combine the prefix IDs. Avoid adding the 3973 // same ID twice. 3974 for (i = 0; i < pfxlen; ++i) 3975 { 3976 for (j = 0; j < use_pfxlen; ++j) 3977 if (pfxlist[i] == use_pfxlist[j]) 3978 break; 3979 if (j == use_pfxlen) 3980 use_pfxlist[use_pfxlen++] = pfxlist[i]; 3981 } 3982 3983 if (spin->si_compflags != NULL) 3984 // Get compound IDS from the affix list. 3985 get_compflags(affile, ae->ae_flags, 3986 use_pfxlist + use_pfxlen); 3987 3988 // Combine the list of compound flags. 3989 // Concatenate them to the prefix IDs list. 3990 // Avoid adding the same ID twice. 3991 for (i = pfxlen; pfxlist[i] != NUL; ++i) 3992 { 3993 for (j = use_pfxlen; 3994 use_pfxlist[j] != NUL; ++j) 3995 if (pfxlist[i] == use_pfxlist[j]) 3996 break; 3997 if (use_pfxlist[j] == NUL) 3998 { 3999 use_pfxlist[j++] = pfxlist[i]; 4000 use_pfxlist[j] = NUL; 4001 } 4002 } 4003 } 4004 } 4005 4006 // Obey a "COMPOUNDFORBIDFLAG" of the affix: don't 4007 // use the compound flags. 4008 if (use_pfxlist != NULL && ae->ae_compforbid) 4009 { 4010 vim_strncpy(pfx_pfxlist, use_pfxlist, use_pfxlen); 4011 use_pfxlist = pfx_pfxlist; 4012 } 4013 4014 // When there are postponed prefixes... 4015 if (spin->si_prefroot != NULL 4016 && spin->si_prefroot->wn_sibling != NULL) 4017 { 4018 // ... add a flag to indicate an affix was used. 4019 use_flags |= WF_HAS_AFF; 4020 4021 // ... don't use a prefix list if combining 4022 // affixes is not allowed. But do use the 4023 // compound flags after them. 4024 if (!ah->ah_combine && use_pfxlist != NULL) 4025 use_pfxlist += use_pfxlen; 4026 } 4027 4028 // When compounding is supported and there is no 4029 // "COMPOUNDPERMITFLAG" then forbid compounding on the 4030 // side where the affix is applied. 4031 if (spin->si_compflags != NULL && !ae->ae_comppermit) 4032 { 4033 if (xht != NULL) 4034 use_flags |= WF_NOCOMPAFT; 4035 else 4036 use_flags |= WF_NOCOMPBEF; 4037 } 4038 4039 // Store the modified word. 4040 if (store_word(spin, newword, use_flags, 4041 spin->si_region, use_pfxlist, 4042 need_affix) == FAIL) 4043 retval = FAIL; 4044 4045 // When added a prefix or a first suffix and the affix 4046 // has flags may add a(nother) suffix. RECURSIVE! 4047 if ((condit & CONDIT_SUF) && ae->ae_flags != NULL) 4048 if (store_aff_word(spin, newword, ae->ae_flags, 4049 affile, &affile->af_suff, xht, 4050 use_condit & (xht == NULL 4051 ? ~0 : ~CONDIT_SUF), 4052 use_flags, use_pfxlist, pfxlen) == FAIL) 4053 retval = FAIL; 4054 4055 // When added a suffix and combining is allowed also 4056 // try adding a prefix additionally. Both for the 4057 // word flags and for the affix flags. RECURSIVE! 4058 if (xht != NULL && ah->ah_combine) 4059 { 4060 if (store_aff_word(spin, newword, 4061 afflist, affile, 4062 xht, NULL, use_condit, 4063 use_flags, use_pfxlist, 4064 pfxlen) == FAIL 4065 || (ae->ae_flags != NULL 4066 && store_aff_word(spin, newword, 4067 ae->ae_flags, affile, 4068 xht, NULL, use_condit, 4069 use_flags, use_pfxlist, 4070 pfxlen) == FAIL)) 4071 retval = FAIL; 4072 } 4073 } 4074 } 4075 } 4076 } 4077 } 4078 4079 return retval; 4080 } 4081 4082 /* 4083 * Read a file with a list of words. 4084 */ 4085 static int 4086 spell_read_wordfile(spellinfo_T *spin, char_u *fname) 4087 { 4088 FILE *fd; 4089 long lnum = 0; 4090 char_u rline[MAXLINELEN]; 4091 char_u *line; 4092 char_u *pc = NULL; 4093 char_u *p; 4094 int l; 4095 int retval = OK; 4096 int did_word = FALSE; 4097 int non_ascii = 0; 4098 int flags; 4099 int regionmask; 4100 4101 /* 4102 * Open the file. 4103 */ 4104 fd = mch_fopen((char *)fname, "r"); 4105 if (fd == NULL) 4106 { 4107 semsg(_(e_notopen), fname); 4108 return FAIL; 4109 } 4110 4111 vim_snprintf((char *)IObuff, IOSIZE, _("Reading word file %s..."), fname); 4112 spell_message(spin, IObuff); 4113 4114 /* 4115 * Read all the lines in the file one by one. 4116 */ 4117 while (!vim_fgets(rline, MAXLINELEN, fd) && !got_int) 4118 { 4119 line_breakcheck(); 4120 ++lnum; 4121 4122 // Skip comment lines. 4123 if (*rline == '#') 4124 continue; 4125 4126 // Remove CR, LF and white space from the end. 4127 l = (int)STRLEN(rline); 4128 while (l > 0 && rline[l - 1] <= ' ') 4129 --l; 4130 if (l == 0) 4131 continue; // empty or blank line 4132 rline[l] = NUL; 4133 4134 // Convert from "/encoding={encoding}" to 'encoding' when needed. 4135 vim_free(pc); 4136 if (spin->si_conv.vc_type != CONV_NONE) 4137 { 4138 pc = string_convert(&spin->si_conv, rline, NULL); 4139 if (pc == NULL) 4140 { 4141 smsg(_("Conversion failure for word in %s line %ld: %s"), 4142 fname, lnum, rline); 4143 continue; 4144 } 4145 line = pc; 4146 } 4147 else 4148 { 4149 pc = NULL; 4150 line = rline; 4151 } 4152 4153 if (*line == '/') 4154 { 4155 ++line; 4156 if (STRNCMP(line, "encoding=", 9) == 0) 4157 { 4158 if (spin->si_conv.vc_type != CONV_NONE) 4159 smsg(_("Duplicate /encoding= line ignored in %s line %ld: %s"), 4160 fname, lnum, line - 1); 4161 else if (did_word) 4162 smsg(_("/encoding= line after word ignored in %s line %ld: %s"), 4163 fname, lnum, line - 1); 4164 else 4165 { 4166 char_u *enc; 4167 4168 // Setup for conversion to 'encoding'. 4169 line += 9; 4170 enc = enc_canonize(line); 4171 if (enc != NULL && !spin->si_ascii 4172 && convert_setup(&spin->si_conv, enc, 4173 p_enc) == FAIL) 4174 smsg(_("Conversion in %s not supported: from %s to %s"), 4175 fname, line, p_enc); 4176 vim_free(enc); 4177 spin->si_conv.vc_fail = TRUE; 4178 } 4179 continue; 4180 } 4181 4182 if (STRNCMP(line, "regions=", 8) == 0) 4183 { 4184 if (spin->si_region_count > 1) 4185 smsg(_("Duplicate /regions= line ignored in %s line %ld: %s"), 4186 fname, lnum, line); 4187 else 4188 { 4189 line += 8; 4190 if (STRLEN(line) > MAXREGIONS * 2) 4191 smsg(_("Too many regions in %s line %ld: %s"), 4192 fname, lnum, line); 4193 else 4194 { 4195 spin->si_region_count = (int)STRLEN(line) / 2; 4196 STRCPY(spin->si_region_name, line); 4197 4198 // Adjust the mask for a word valid in all regions. 4199 spin->si_region = (1 << spin->si_region_count) - 1; 4200 } 4201 } 4202 continue; 4203 } 4204 4205 smsg(_("/ line ignored in %s line %ld: %s"), 4206 fname, lnum, line - 1); 4207 continue; 4208 } 4209 4210 flags = 0; 4211 regionmask = spin->si_region; 4212 4213 // Check for flags and region after a slash. 4214 p = vim_strchr(line, '/'); 4215 if (p != NULL) 4216 { 4217 *p++ = NUL; 4218 while (*p != NUL) 4219 { 4220 if (*p == '=') // keep-case word 4221 flags |= WF_KEEPCAP | WF_FIXCAP; 4222 else if (*p == '!') // Bad, bad, wicked word. 4223 flags |= WF_BANNED; 4224 else if (*p == '?') // Rare word. 4225 flags |= WF_RARE; 4226 else if (VIM_ISDIGIT(*p)) // region number(s) 4227 { 4228 if ((flags & WF_REGION) == 0) // first one 4229 regionmask = 0; 4230 flags |= WF_REGION; 4231 4232 l = *p - '0'; 4233 if (l == 0 || l > spin->si_region_count) 4234 { 4235 smsg(_("Invalid region nr in %s line %ld: %s"), 4236 fname, lnum, p); 4237 break; 4238 } 4239 regionmask |= 1 << (l - 1); 4240 } 4241 else 4242 { 4243 smsg(_("Unrecognized flags in %s line %ld: %s"), 4244 fname, lnum, p); 4245 break; 4246 } 4247 ++p; 4248 } 4249 } 4250 4251 // Skip non-ASCII words when "spin->si_ascii" is TRUE. 4252 if (spin->si_ascii && has_non_ascii(line)) 4253 { 4254 ++non_ascii; 4255 continue; 4256 } 4257 4258 // Normal word: store it. 4259 if (store_word(spin, line, flags, regionmask, NULL, FALSE) == FAIL) 4260 { 4261 retval = FAIL; 4262 break; 4263 } 4264 did_word = TRUE; 4265 } 4266 4267 vim_free(pc); 4268 fclose(fd); 4269 4270 if (spin->si_ascii && non_ascii > 0) 4271 { 4272 vim_snprintf((char *)IObuff, IOSIZE, 4273 _("Ignored %d words with non-ASCII characters"), non_ascii); 4274 spell_message(spin, IObuff); 4275 } 4276 4277 return retval; 4278 } 4279 4280 /* 4281 * Get part of an sblock_T, "len" bytes long. 4282 * This avoids calling free() for every little struct we use (and keeping 4283 * track of them). 4284 * The memory is cleared to all zeros. 4285 * Returns NULL when out of memory. 4286 */ 4287 static void * 4288 getroom( 4289 spellinfo_T *spin, 4290 size_t len, // length needed 4291 int align) // align for pointer 4292 { 4293 char_u *p; 4294 sblock_T *bl = spin->si_blocks; 4295 4296 if (align && bl != NULL) 4297 // Round size up for alignment. On some systems structures need to be 4298 // aligned to the size of a pointer (e.g., SPARC). 4299 bl->sb_used = (bl->sb_used + sizeof(char *) - 1) 4300 & ~(sizeof(char *) - 1); 4301 4302 if (bl == NULL || bl->sb_used + len > SBLOCKSIZE) 4303 { 4304 if (len >= SBLOCKSIZE) 4305 bl = NULL; 4306 else 4307 // Allocate a block of memory. It is not freed until much later. 4308 bl = alloc_clear(sizeof(sblock_T) + SBLOCKSIZE); 4309 if (bl == NULL) 4310 { 4311 if (!spin->si_did_emsg) 4312 { 4313 emsg(_("E845: Insufficient memory, word list will be incomplete")); 4314 spin->si_did_emsg = TRUE; 4315 } 4316 return NULL; 4317 } 4318 bl->sb_next = spin->si_blocks; 4319 spin->si_blocks = bl; 4320 bl->sb_used = 0; 4321 ++spin->si_blocks_cnt; 4322 } 4323 4324 p = bl->sb_data + bl->sb_used; 4325 bl->sb_used += (int)len; 4326 4327 return p; 4328 } 4329 4330 /* 4331 * Make a copy of a string into memory allocated with getroom(). 4332 * Returns NULL when out of memory. 4333 */ 4334 static char_u * 4335 getroom_save(spellinfo_T *spin, char_u *s) 4336 { 4337 char_u *sc; 4338 4339 sc = (char_u *)getroom(spin, STRLEN(s) + 1, FALSE); 4340 if (sc != NULL) 4341 STRCPY(sc, s); 4342 return sc; 4343 } 4344 4345 4346 /* 4347 * Free the list of allocated sblock_T. 4348 */ 4349 static void 4350 free_blocks(sblock_T *bl) 4351 { 4352 sblock_T *next; 4353 4354 while (bl != NULL) 4355 { 4356 next = bl->sb_next; 4357 vim_free(bl); 4358 bl = next; 4359 } 4360 } 4361 4362 /* 4363 * Allocate the root of a word tree. 4364 * Returns NULL when out of memory. 4365 */ 4366 static wordnode_T * 4367 wordtree_alloc(spellinfo_T *spin) 4368 { 4369 return (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 4370 } 4371 4372 /* 4373 * Store a word in the tree(s). 4374 * Always store it in the case-folded tree. For a keep-case word this is 4375 * useful when the word can also be used with all caps (no WF_FIXCAP flag) and 4376 * used to find suggestions. 4377 * For a keep-case word also store it in the keep-case tree. 4378 * When "pfxlist" is not NULL store the word for each postponed prefix ID and 4379 * compound flag. 4380 */ 4381 static int 4382 store_word( 4383 spellinfo_T *spin, 4384 char_u *word, 4385 int flags, // extra flags, WF_BANNED 4386 int region, // supported region(s) 4387 char_u *pfxlist, // list of prefix IDs or NULL 4388 int need_affix) // only store word with affix ID 4389 { 4390 int len = (int)STRLEN(word); 4391 int ct = captype(word, word + len); 4392 char_u foldword[MAXWLEN]; 4393 int res = OK; 4394 char_u *p; 4395 4396 (void)spell_casefold(curwin, word, len, foldword, MAXWLEN); 4397 for (p = pfxlist; res == OK; ++p) 4398 { 4399 if (!need_affix || (p != NULL && *p != NUL)) 4400 res = tree_add_word(spin, foldword, spin->si_foldroot, ct | flags, 4401 region, p == NULL ? 0 : *p); 4402 if (p == NULL || *p == NUL) 4403 break; 4404 } 4405 ++spin->si_foldwcount; 4406 4407 if (res == OK && (ct == WF_KEEPCAP || (flags & WF_KEEPCAP))) 4408 { 4409 for (p = pfxlist; res == OK; ++p) 4410 { 4411 if (!need_affix || (p != NULL && *p != NUL)) 4412 res = tree_add_word(spin, word, spin->si_keeproot, flags, 4413 region, p == NULL ? 0 : *p); 4414 if (p == NULL || *p == NUL) 4415 break; 4416 } 4417 ++spin->si_keepwcount; 4418 } 4419 return res; 4420 } 4421 4422 /* 4423 * Add word "word" to a word tree at "root". 4424 * When "flags" < 0 we are adding to the prefix tree where "flags" is used for 4425 * "rare" and "region" is the condition nr. 4426 * Returns FAIL when out of memory. 4427 */ 4428 static int 4429 tree_add_word( 4430 spellinfo_T *spin, 4431 char_u *word, 4432 wordnode_T *root, 4433 int flags, 4434 int region, 4435 int affixID) 4436 { 4437 wordnode_T *node = root; 4438 wordnode_T *np; 4439 wordnode_T *copyp, **copyprev; 4440 wordnode_T **prev = NULL; 4441 int i; 4442 4443 // Add each byte of the word to the tree, including the NUL at the end. 4444 for (i = 0; ; ++i) 4445 { 4446 // When there is more than one reference to this node we need to make 4447 // a copy, so that we can modify it. Copy the whole list of siblings 4448 // (we don't optimize for a partly shared list of siblings). 4449 if (node != NULL && node->wn_refs > 1) 4450 { 4451 --node->wn_refs; 4452 copyprev = prev; 4453 FOR_ALL_NODE_SIBLINGS(node, copyp) 4454 { 4455 // Allocate a new node and copy the info. 4456 np = get_wordnode(spin); 4457 if (np == NULL) 4458 return FAIL; 4459 np->wn_child = copyp->wn_child; 4460 if (np->wn_child != NULL) 4461 ++np->wn_child->wn_refs; // child gets extra ref 4462 np->wn_byte = copyp->wn_byte; 4463 if (np->wn_byte == NUL) 4464 { 4465 np->wn_flags = copyp->wn_flags; 4466 np->wn_region = copyp->wn_region; 4467 np->wn_affixID = copyp->wn_affixID; 4468 } 4469 4470 // Link the new node in the list, there will be one ref. 4471 np->wn_refs = 1; 4472 if (copyprev != NULL) 4473 *copyprev = np; 4474 copyprev = &np->wn_sibling; 4475 4476 // Let "node" point to the head of the copied list. 4477 if (copyp == node) 4478 node = np; 4479 } 4480 } 4481 4482 // Look for the sibling that has the same character. They are sorted 4483 // on byte value, thus stop searching when a sibling is found with a 4484 // higher byte value. For zero bytes (end of word) the sorting is 4485 // done on flags and then on affixID. 4486 while (node != NULL 4487 && (node->wn_byte < word[i] 4488 || (node->wn_byte == NUL 4489 && (flags < 0 4490 ? node->wn_affixID < (unsigned)affixID 4491 : (node->wn_flags < (unsigned)(flags & WN_MASK) 4492 || (node->wn_flags == (flags & WN_MASK) 4493 && (spin->si_sugtree 4494 ? (node->wn_region & 0xffff) < region 4495 : node->wn_affixID 4496 < (unsigned)affixID))))))) 4497 { 4498 prev = &node->wn_sibling; 4499 node = *prev; 4500 } 4501 if (node == NULL 4502 || node->wn_byte != word[i] 4503 || (word[i] == NUL 4504 && (flags < 0 4505 || spin->si_sugtree 4506 || node->wn_flags != (flags & WN_MASK) 4507 || node->wn_affixID != affixID))) 4508 { 4509 // Allocate a new node. 4510 np = get_wordnode(spin); 4511 if (np == NULL) 4512 return FAIL; 4513 np->wn_byte = word[i]; 4514 4515 // If "node" is NULL this is a new child or the end of the sibling 4516 // list: ref count is one. Otherwise use ref count of sibling and 4517 // make ref count of sibling one (matters when inserting in front 4518 // of the list of siblings). 4519 if (node == NULL) 4520 np->wn_refs = 1; 4521 else 4522 { 4523 np->wn_refs = node->wn_refs; 4524 node->wn_refs = 1; 4525 } 4526 if (prev != NULL) 4527 *prev = np; 4528 np->wn_sibling = node; 4529 node = np; 4530 } 4531 4532 if (word[i] == NUL) 4533 { 4534 node->wn_flags = flags; 4535 node->wn_region |= region; 4536 node->wn_affixID = affixID; 4537 break; 4538 } 4539 prev = &node->wn_child; 4540 node = *prev; 4541 } 4542 #ifdef SPELL_PRINTTREE 4543 smsg("Added \"%s\"", word); 4544 spell_print_tree(root->wn_sibling); 4545 #endif 4546 4547 // count nr of words added since last message 4548 ++spin->si_msg_count; 4549 4550 if (spin->si_compress_cnt > 1) 4551 { 4552 if (--spin->si_compress_cnt == 1) 4553 // Did enough words to lower the block count limit. 4554 spin->si_blocks_cnt += compress_inc; 4555 } 4556 4557 /* 4558 * When we have allocated lots of memory we need to compress the word tree 4559 * to free up some room. But compression is slow, and we might actually 4560 * need that room, thus only compress in the following situations: 4561 * 1. When not compressed before (si_compress_cnt == 0): when using 4562 * "compress_start" blocks. 4563 * 2. When compressed before and used "compress_inc" blocks before 4564 * adding "compress_added" words (si_compress_cnt > 1). 4565 * 3. When compressed before, added "compress_added" words 4566 * (si_compress_cnt == 1) and the number of free nodes drops below the 4567 * maximum word length. 4568 */ 4569 #ifndef SPELL_COMPRESS_ALLWAYS 4570 if (spin->si_compress_cnt == 1 4571 ? spin->si_free_count < MAXWLEN 4572 : spin->si_blocks_cnt >= compress_start) 4573 #endif 4574 { 4575 // Decrement the block counter. The effect is that we compress again 4576 // when the freed up room has been used and another "compress_inc" 4577 // blocks have been allocated. Unless "compress_added" words have 4578 // been added, then the limit is put back again. 4579 spin->si_blocks_cnt -= compress_inc; 4580 spin->si_compress_cnt = compress_added; 4581 4582 if (spin->si_verbose) 4583 { 4584 msg_start(); 4585 msg_puts(_(msg_compressing)); 4586 msg_clr_eos(); 4587 msg_didout = FALSE; 4588 msg_col = 0; 4589 out_flush(); 4590 } 4591 4592 // Compress both trees. Either they both have many nodes, which makes 4593 // compression useful, or one of them is small, which means 4594 // compression goes fast. But when filling the soundfold word tree 4595 // there is no keep-case tree. 4596 wordtree_compress(spin, spin->si_foldroot, "case-folded"); 4597 if (affixID >= 0) 4598 wordtree_compress(spin, spin->si_keeproot, "keep-case"); 4599 } 4600 4601 return OK; 4602 } 4603 4604 /* 4605 * Get a wordnode_T, either from the list of previously freed nodes or 4606 * allocate a new one. 4607 * Returns NULL when out of memory. 4608 */ 4609 static wordnode_T * 4610 get_wordnode(spellinfo_T *spin) 4611 { 4612 wordnode_T *n; 4613 4614 if (spin->si_first_free == NULL) 4615 n = (wordnode_T *)getroom(spin, sizeof(wordnode_T), TRUE); 4616 else 4617 { 4618 n = spin->si_first_free; 4619 spin->si_first_free = n->wn_child; 4620 CLEAR_POINTER(n); 4621 --spin->si_free_count; 4622 } 4623 #ifdef SPELL_PRINTTREE 4624 if (n != NULL) 4625 n->wn_nr = ++spin->si_wordnode_nr; 4626 #endif 4627 return n; 4628 } 4629 4630 /* 4631 * Decrement the reference count on a node (which is the head of a list of 4632 * siblings). If the reference count becomes zero free the node and its 4633 * siblings. 4634 * Returns the number of nodes actually freed. 4635 */ 4636 static int 4637 deref_wordnode(spellinfo_T *spin, wordnode_T *node) 4638 { 4639 wordnode_T *np; 4640 int cnt = 0; 4641 4642 if (--node->wn_refs == 0) 4643 { 4644 FOR_ALL_NODE_SIBLINGS(node, np) 4645 { 4646 if (np->wn_child != NULL) 4647 cnt += deref_wordnode(spin, np->wn_child); 4648 free_wordnode(spin, np); 4649 ++cnt; 4650 } 4651 ++cnt; // length field 4652 } 4653 return cnt; 4654 } 4655 4656 /* 4657 * Free a wordnode_T for re-use later. 4658 * Only the "wn_child" field becomes invalid. 4659 */ 4660 static void 4661 free_wordnode(spellinfo_T *spin, wordnode_T *n) 4662 { 4663 n->wn_child = spin->si_first_free; 4664 spin->si_first_free = n; 4665 ++spin->si_free_count; 4666 } 4667 4668 /* 4669 * Compress a tree: find tails that are identical and can be shared. 4670 */ 4671 static void 4672 wordtree_compress(spellinfo_T *spin, wordnode_T *root, char *name) 4673 { 4674 hashtab_T ht; 4675 long n; 4676 long tot = 0; 4677 long perc; 4678 4679 // Skip the root itself, it's not actually used. The first sibling is the 4680 // start of the tree. 4681 if (root->wn_sibling != NULL) 4682 { 4683 hash_init(&ht); 4684 n = node_compress(spin, root->wn_sibling, &ht, &tot); 4685 4686 #ifndef SPELL_PRINTTREE 4687 if (spin->si_verbose || p_verbose > 2) 4688 #endif 4689 { 4690 if (tot > 1000000) 4691 perc = (tot - n) / (tot / 100); 4692 else if (tot == 0) 4693 perc = 0; 4694 else 4695 perc = (tot - n) * 100 / tot; 4696 vim_snprintf((char *)IObuff, IOSIZE, 4697 _("Compressed %s: %ld of %ld nodes; %ld (%ld%%) remaining"), 4698 name, n, tot, tot - n, perc); 4699 spell_message(spin, IObuff); 4700 } 4701 #ifdef SPELL_PRINTTREE 4702 spell_print_tree(root->wn_sibling); 4703 #endif 4704 hash_clear(&ht); 4705 } 4706 } 4707 4708 /* 4709 * Compress a node, its siblings and its children, depth first. 4710 * Returns the number of compressed nodes. 4711 */ 4712 static long 4713 node_compress( 4714 spellinfo_T *spin, 4715 wordnode_T *node, 4716 hashtab_T *ht, 4717 long *tot) // total count of nodes before compressing, 4718 // incremented while going through the tree 4719 { 4720 wordnode_T *np; 4721 wordnode_T *tp; 4722 wordnode_T *child; 4723 hash_T hash; 4724 hashitem_T *hi; 4725 long len = 0; 4726 unsigned nr, n; 4727 long compressed = 0; 4728 4729 /* 4730 * Go through the list of siblings. Compress each child and then try 4731 * finding an identical child to replace it. 4732 * Note that with "child" we mean not just the node that is pointed to, 4733 * but the whole list of siblings of which the child node is the first. 4734 */ 4735 for (np = node; np != NULL && !got_int; np = np->wn_sibling) 4736 { 4737 ++len; 4738 if ((child = np->wn_child) != NULL) 4739 { 4740 // Compress the child first. This fills hashkey. 4741 compressed += node_compress(spin, child, ht, tot); 4742 4743 // Try to find an identical child. 4744 hash = hash_hash(child->wn_u1.hashkey); 4745 hi = hash_lookup(ht, child->wn_u1.hashkey, hash); 4746 if (!HASHITEM_EMPTY(hi)) 4747 { 4748 // There are children we encountered before with a hash value 4749 // identical to the current child. Now check if there is one 4750 // that is really identical. 4751 for (tp = HI2WN(hi); tp != NULL; tp = tp->wn_u2.next) 4752 if (node_equal(child, tp)) 4753 { 4754 // Found one! Now use that child in place of the 4755 // current one. This means the current child and all 4756 // its siblings is unlinked from the tree. 4757 ++tp->wn_refs; 4758 compressed += deref_wordnode(spin, child); 4759 np->wn_child = tp; 4760 break; 4761 } 4762 if (tp == NULL) 4763 { 4764 // No other child with this hash value equals the child of 4765 // the node, add it to the linked list after the first 4766 // item. 4767 tp = HI2WN(hi); 4768 child->wn_u2.next = tp->wn_u2.next; 4769 tp->wn_u2.next = child; 4770 } 4771 } 4772 else 4773 // No other child has this hash value, add it to the 4774 // hashtable. 4775 hash_add_item(ht, hi, child->wn_u1.hashkey, hash); 4776 } 4777 } 4778 *tot += len + 1; // add one for the node that stores the length 4779 4780 /* 4781 * Make a hash key for the node and its siblings, so that we can quickly 4782 * find a lookalike node. This must be done after compressing the sibling 4783 * list, otherwise the hash key would become invalid by the compression. 4784 */ 4785 node->wn_u1.hashkey[0] = len; 4786 nr = 0; 4787 FOR_ALL_NODE_SIBLINGS(node, np) 4788 { 4789 if (np->wn_byte == NUL) 4790 // end node: use wn_flags, wn_region and wn_affixID 4791 n = np->wn_flags + (np->wn_region << 8) + (np->wn_affixID << 16); 4792 else 4793 // byte node: use the byte value and the child pointer 4794 n = (unsigned)(np->wn_byte + ((long_u)np->wn_child << 8)); 4795 nr = nr * 101 + n; 4796 } 4797 4798 // Avoid NUL bytes, it terminates the hash key. 4799 n = nr & 0xff; 4800 node->wn_u1.hashkey[1] = n == 0 ? 1 : n; 4801 n = (nr >> 8) & 0xff; 4802 node->wn_u1.hashkey[2] = n == 0 ? 1 : n; 4803 n = (nr >> 16) & 0xff; 4804 node->wn_u1.hashkey[3] = n == 0 ? 1 : n; 4805 n = (nr >> 24) & 0xff; 4806 node->wn_u1.hashkey[4] = n == 0 ? 1 : n; 4807 node->wn_u1.hashkey[5] = NUL; 4808 4809 // Check for CTRL-C pressed now and then. 4810 veryfast_breakcheck(); 4811 4812 return compressed; 4813 } 4814 4815 /* 4816 * Return TRUE when two nodes have identical siblings and children. 4817 */ 4818 static int 4819 node_equal(wordnode_T *n1, wordnode_T *n2) 4820 { 4821 wordnode_T *p1; 4822 wordnode_T *p2; 4823 4824 for (p1 = n1, p2 = n2; p1 != NULL && p2 != NULL; 4825 p1 = p1->wn_sibling, p2 = p2->wn_sibling) 4826 if (p1->wn_byte != p2->wn_byte 4827 || (p1->wn_byte == NUL 4828 ? (p1->wn_flags != p2->wn_flags 4829 || p1->wn_region != p2->wn_region 4830 || p1->wn_affixID != p2->wn_affixID) 4831 : (p1->wn_child != p2->wn_child))) 4832 break; 4833 4834 return p1 == NULL && p2 == NULL; 4835 } 4836 4837 static int rep_compare(const void *s1, const void *s2); 4838 4839 /* 4840 * Function given to qsort() to sort the REP items on "from" string. 4841 */ 4842 static int 4843 rep_compare(const void *s1, const void *s2) 4844 { 4845 fromto_T *p1 = (fromto_T *)s1; 4846 fromto_T *p2 = (fromto_T *)s2; 4847 4848 return STRCMP(p1->ft_from, p2->ft_from); 4849 } 4850 4851 /* 4852 * Write the Vim .spl file "fname". 4853 * Return FAIL or OK; 4854 */ 4855 static int 4856 write_vim_spell(spellinfo_T *spin, char_u *fname) 4857 { 4858 FILE *fd; 4859 int regionmask; 4860 int round; 4861 wordnode_T *tree; 4862 int nodecount; 4863 int i; 4864 int l; 4865 garray_T *gap; 4866 fromto_T *ftp; 4867 char_u *p; 4868 int rr; 4869 int retval = OK; 4870 size_t fwv = 1; // collect return value of fwrite() to avoid 4871 // warnings from picky compiler 4872 4873 fd = mch_fopen((char *)fname, "w"); 4874 if (fd == NULL) 4875 { 4876 semsg(_(e_notopen), fname); 4877 return FAIL; 4878 } 4879 4880 // <HEADER>: <fileID> <versionnr> 4881 // <fileID> 4882 fwv &= fwrite(VIMSPELLMAGIC, VIMSPELLMAGICL, (size_t)1, fd); 4883 if (fwv != (size_t)1) 4884 // Catch first write error, don't try writing more. 4885 goto theend; 4886 4887 putc(VIMSPELLVERSION, fd); // <versionnr> 4888 4889 /* 4890 * <SECTIONS>: <section> ... <sectionend> 4891 */ 4892 4893 // SN_INFO: <infotext> 4894 if (spin->si_info != NULL) 4895 { 4896 putc(SN_INFO, fd); // <sectionID> 4897 putc(0, fd); // <sectionflags> 4898 4899 i = (int)STRLEN(spin->si_info); 4900 put_bytes(fd, (long_u)i, 4); // <sectionlen> 4901 fwv &= fwrite(spin->si_info, (size_t)i, (size_t)1, fd); // <infotext> 4902 } 4903 4904 // SN_REGION: <regionname> ... 4905 // Write the region names only if there is more than one. 4906 if (spin->si_region_count > 1) 4907 { 4908 putc(SN_REGION, fd); // <sectionID> 4909 putc(SNF_REQUIRED, fd); // <sectionflags> 4910 l = spin->si_region_count * 2; 4911 put_bytes(fd, (long_u)l, 4); // <sectionlen> 4912 fwv &= fwrite(spin->si_region_name, (size_t)l, (size_t)1, fd); 4913 // <regionname> ... 4914 regionmask = (1 << spin->si_region_count) - 1; 4915 } 4916 else 4917 regionmask = 0; 4918 4919 // SN_CHARFLAGS: <charflagslen> <charflags> <folcharslen> <folchars> 4920 // 4921 // The table with character flags and the table for case folding. 4922 // This makes sure the same characters are recognized as word characters 4923 // when generating an when using a spell file. 4924 // Skip this for ASCII, the table may conflict with the one used for 4925 // 'encoding'. 4926 // Also skip this for an .add.spl file, the main spell file must contain 4927 // the table (avoids that it conflicts). File is shorter too. 4928 if (!spin->si_ascii && !spin->si_add) 4929 { 4930 char_u folchars[128 * 8]; 4931 int flags; 4932 4933 putc(SN_CHARFLAGS, fd); // <sectionID> 4934 putc(SNF_REQUIRED, fd); // <sectionflags> 4935 4936 // Form the <folchars> string first, we need to know its length. 4937 l = 0; 4938 for (i = 128; i < 256; ++i) 4939 { 4940 if (has_mbyte) 4941 l += mb_char2bytes(spelltab.st_fold[i], folchars + l); 4942 else 4943 folchars[l++] = spelltab.st_fold[i]; 4944 } 4945 put_bytes(fd, (long_u)(1 + 128 + 2 + l), 4); // <sectionlen> 4946 4947 fputc(128, fd); // <charflagslen> 4948 for (i = 128; i < 256; ++i) 4949 { 4950 flags = 0; 4951 if (spelltab.st_isw[i]) 4952 flags |= CF_WORD; 4953 if (spelltab.st_isu[i]) 4954 flags |= CF_UPPER; 4955 fputc(flags, fd); // <charflags> 4956 } 4957 4958 put_bytes(fd, (long_u)l, 2); // <folcharslen> 4959 fwv &= fwrite(folchars, (size_t)l, (size_t)1, fd); // <folchars> 4960 } 4961 4962 // SN_MIDWORD: <midword> 4963 if (spin->si_midword != NULL) 4964 { 4965 putc(SN_MIDWORD, fd); // <sectionID> 4966 putc(SNF_REQUIRED, fd); // <sectionflags> 4967 4968 i = (int)STRLEN(spin->si_midword); 4969 put_bytes(fd, (long_u)i, 4); // <sectionlen> 4970 fwv &= fwrite(spin->si_midword, (size_t)i, (size_t)1, fd); 4971 // <midword> 4972 } 4973 4974 // SN_PREFCOND: <prefcondcnt> <prefcond> ... 4975 if (spin->si_prefcond.ga_len > 0) 4976 { 4977 putc(SN_PREFCOND, fd); // <sectionID> 4978 putc(SNF_REQUIRED, fd); // <sectionflags> 4979 4980 l = write_spell_prefcond(NULL, &spin->si_prefcond); 4981 put_bytes(fd, (long_u)l, 4); // <sectionlen> 4982 4983 write_spell_prefcond(fd, &spin->si_prefcond); 4984 } 4985 4986 // SN_REP: <repcount> <rep> ... 4987 // SN_SAL: <salflags> <salcount> <sal> ... 4988 // SN_REPSAL: <repcount> <rep> ... 4989 4990 // round 1: SN_REP section 4991 // round 2: SN_SAL section (unless SN_SOFO is used) 4992 // round 3: SN_REPSAL section 4993 for (round = 1; round <= 3; ++round) 4994 { 4995 if (round == 1) 4996 gap = &spin->si_rep; 4997 else if (round == 2) 4998 { 4999 // Don't write SN_SAL when using a SN_SOFO section 5000 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 5001 continue; 5002 gap = &spin->si_sal; 5003 } 5004 else 5005 gap = &spin->si_repsal; 5006 5007 // Don't write the section if there are no items. 5008 if (gap->ga_len == 0) 5009 continue; 5010 5011 // Sort the REP/REPSAL items. 5012 if (round != 2) 5013 qsort(gap->ga_data, (size_t)gap->ga_len, 5014 sizeof(fromto_T), rep_compare); 5015 5016 i = round == 1 ? SN_REP : (round == 2 ? SN_SAL : SN_REPSAL); 5017 putc(i, fd); // <sectionID> 5018 5019 // This is for making suggestions, section is not required. 5020 putc(0, fd); // <sectionflags> 5021 5022 // Compute the length of what follows. 5023 l = 2; // count <repcount> or <salcount> 5024 for (i = 0; i < gap->ga_len; ++i) 5025 { 5026 ftp = &((fromto_T *)gap->ga_data)[i]; 5027 l += 1 + (int)STRLEN(ftp->ft_from); // count <*fromlen> and <*from> 5028 l += 1 + (int)STRLEN(ftp->ft_to); // count <*tolen> and <*to> 5029 } 5030 if (round == 2) 5031 ++l; // count <salflags> 5032 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5033 5034 if (round == 2) 5035 { 5036 i = 0; 5037 if (spin->si_followup) 5038 i |= SAL_F0LLOWUP; 5039 if (spin->si_collapse) 5040 i |= SAL_COLLAPSE; 5041 if (spin->si_rem_accents) 5042 i |= SAL_REM_ACCENTS; 5043 putc(i, fd); // <salflags> 5044 } 5045 5046 put_bytes(fd, (long_u)gap->ga_len, 2); // <repcount> or <salcount> 5047 for (i = 0; i < gap->ga_len; ++i) 5048 { 5049 // <rep> : <repfromlen> <repfrom> <reptolen> <repto> 5050 // <sal> : <salfromlen> <salfrom> <saltolen> <salto> 5051 ftp = &((fromto_T *)gap->ga_data)[i]; 5052 for (rr = 1; rr <= 2; ++rr) 5053 { 5054 p = rr == 1 ? ftp->ft_from : ftp->ft_to; 5055 l = (int)STRLEN(p); 5056 putc(l, fd); 5057 if (l > 0) 5058 fwv &= fwrite(p, l, (size_t)1, fd); 5059 } 5060 } 5061 5062 } 5063 5064 // SN_SOFO: <sofofromlen> <sofofrom> <sofotolen> <sofoto> 5065 // This is for making suggestions, section is not required. 5066 if (spin->si_sofofr != NULL && spin->si_sofoto != NULL) 5067 { 5068 putc(SN_SOFO, fd); // <sectionID> 5069 putc(0, fd); // <sectionflags> 5070 5071 l = (int)STRLEN(spin->si_sofofr); 5072 put_bytes(fd, (long_u)(l + STRLEN(spin->si_sofoto) + 4), 4); 5073 // <sectionlen> 5074 5075 put_bytes(fd, (long_u)l, 2); // <sofofromlen> 5076 fwv &= fwrite(spin->si_sofofr, l, (size_t)1, fd); // <sofofrom> 5077 5078 l = (int)STRLEN(spin->si_sofoto); 5079 put_bytes(fd, (long_u)l, 2); // <sofotolen> 5080 fwv &= fwrite(spin->si_sofoto, l, (size_t)1, fd); // <sofoto> 5081 } 5082 5083 // SN_WORDS: <word> ... 5084 // This is for making suggestions, section is not required. 5085 if (spin->si_commonwords.ht_used > 0) 5086 { 5087 putc(SN_WORDS, fd); // <sectionID> 5088 putc(0, fd); // <sectionflags> 5089 5090 // round 1: count the bytes 5091 // round 2: write the bytes 5092 for (round = 1; round <= 2; ++round) 5093 { 5094 int todo; 5095 int len = 0; 5096 hashitem_T *hi; 5097 5098 todo = (int)spin->si_commonwords.ht_used; 5099 for (hi = spin->si_commonwords.ht_array; todo > 0; ++hi) 5100 if (!HASHITEM_EMPTY(hi)) 5101 { 5102 l = (int)STRLEN(hi->hi_key) + 1; 5103 len += l; 5104 if (round == 2) // <word> 5105 fwv &= fwrite(hi->hi_key, (size_t)l, (size_t)1, fd); 5106 --todo; 5107 } 5108 if (round == 1) 5109 put_bytes(fd, (long_u)len, 4); // <sectionlen> 5110 } 5111 } 5112 5113 // SN_MAP: <mapstr> 5114 // This is for making suggestions, section is not required. 5115 if (spin->si_map.ga_len > 0) 5116 { 5117 putc(SN_MAP, fd); // <sectionID> 5118 putc(0, fd); // <sectionflags> 5119 l = spin->si_map.ga_len; 5120 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5121 fwv &= fwrite(spin->si_map.ga_data, (size_t)l, (size_t)1, fd); 5122 // <mapstr> 5123 } 5124 5125 // SN_SUGFILE: <timestamp> 5126 // This is used to notify that a .sug file may be available and at the 5127 // same time allows for checking that a .sug file that is found matches 5128 // with this .spl file. That's because the word numbers must be exactly 5129 // right. 5130 if (!spin->si_nosugfile 5131 && (spin->si_sal.ga_len > 0 5132 || (spin->si_sofofr != NULL && spin->si_sofoto != NULL))) 5133 { 5134 putc(SN_SUGFILE, fd); // <sectionID> 5135 putc(0, fd); // <sectionflags> 5136 put_bytes(fd, (long_u)8, 4); // <sectionlen> 5137 5138 // Set si_sugtime and write it to the file. 5139 spin->si_sugtime = time(NULL); 5140 put_time(fd, spin->si_sugtime); // <timestamp> 5141 } 5142 5143 // SN_NOSPLITSUGS: nothing 5144 // This is used to notify that no suggestions with word splits are to be 5145 // made. 5146 if (spin->si_nosplitsugs) 5147 { 5148 putc(SN_NOSPLITSUGS, fd); // <sectionID> 5149 putc(0, fd); // <sectionflags> 5150 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5151 } 5152 5153 // SN_NOCOMPUNDSUGS: nothing 5154 // This is used to notify that no suggestions with compounds are to be 5155 // made. 5156 if (spin->si_nocompoundsugs) 5157 { 5158 putc(SN_NOCOMPOUNDSUGS, fd); // <sectionID> 5159 putc(0, fd); // <sectionflags> 5160 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5161 } 5162 5163 // SN_COMPOUND: compound info. 5164 // We don't mark it required, when not supported all compound words will 5165 // be bad words. 5166 if (spin->si_compflags != NULL) 5167 { 5168 putc(SN_COMPOUND, fd); // <sectionID> 5169 putc(0, fd); // <sectionflags> 5170 5171 l = (int)STRLEN(spin->si_compflags); 5172 for (i = 0; i < spin->si_comppat.ga_len; ++i) 5173 l += (int)STRLEN(((char_u **)(spin->si_comppat.ga_data))[i]) + 1; 5174 put_bytes(fd, (long_u)(l + 7), 4); // <sectionlen> 5175 5176 putc(spin->si_compmax, fd); // <compmax> 5177 putc(spin->si_compminlen, fd); // <compminlen> 5178 putc(spin->si_compsylmax, fd); // <compsylmax> 5179 putc(0, fd); // for Vim 7.0b compatibility 5180 putc(spin->si_compoptions, fd); // <compoptions> 5181 put_bytes(fd, (long_u)spin->si_comppat.ga_len, 2); 5182 // <comppatcount> 5183 for (i = 0; i < spin->si_comppat.ga_len; ++i) 5184 { 5185 p = ((char_u **)(spin->si_comppat.ga_data))[i]; 5186 putc((int)STRLEN(p), fd); // <comppatlen> 5187 fwv &= fwrite(p, (size_t)STRLEN(p), (size_t)1, fd); 5188 // <comppattext> 5189 } 5190 // <compflags> 5191 fwv &= fwrite(spin->si_compflags, (size_t)STRLEN(spin->si_compflags), 5192 (size_t)1, fd); 5193 } 5194 5195 // SN_NOBREAK: NOBREAK flag 5196 if (spin->si_nobreak) 5197 { 5198 putc(SN_NOBREAK, fd); // <sectionID> 5199 putc(0, fd); // <sectionflags> 5200 5201 // It's empty, the presence of the section flags the feature. 5202 put_bytes(fd, (long_u)0, 4); // <sectionlen> 5203 } 5204 5205 // SN_SYLLABLE: syllable info. 5206 // We don't mark it required, when not supported syllables will not be 5207 // counted. 5208 if (spin->si_syllable != NULL) 5209 { 5210 putc(SN_SYLLABLE, fd); // <sectionID> 5211 putc(0, fd); // <sectionflags> 5212 5213 l = (int)STRLEN(spin->si_syllable); 5214 put_bytes(fd, (long_u)l, 4); // <sectionlen> 5215 fwv &= fwrite(spin->si_syllable, (size_t)l, (size_t)1, fd); 5216 // <syllable> 5217 } 5218 5219 // end of <SECTIONS> 5220 putc(SN_END, fd); // <sectionend> 5221 5222 5223 /* 5224 * <LWORDTREE> <KWORDTREE> <PREFIXTREE> 5225 */ 5226 spin->si_memtot = 0; 5227 for (round = 1; round <= 3; ++round) 5228 { 5229 if (round == 1) 5230 tree = spin->si_foldroot->wn_sibling; 5231 else if (round == 2) 5232 tree = spin->si_keeproot->wn_sibling; 5233 else 5234 tree = spin->si_prefroot->wn_sibling; 5235 5236 // Clear the index and wnode fields in the tree. 5237 clear_node(tree); 5238 5239 // Count the number of nodes. Needed to be able to allocate the 5240 // memory when reading the nodes. Also fills in index for shared 5241 // nodes. 5242 nodecount = put_node(NULL, tree, 0, regionmask, round == 3); 5243 5244 // number of nodes in 4 bytes 5245 put_bytes(fd, (long_u)nodecount, 4); // <nodecount> 5246 spin->si_memtot += nodecount + nodecount * sizeof(int); 5247 5248 // Write the nodes. 5249 (void)put_node(fd, tree, 0, regionmask, round == 3); 5250 } 5251 5252 // Write another byte to check for errors (file system full). 5253 if (putc(0, fd) == EOF) 5254 retval = FAIL; 5255 theend: 5256 if (fclose(fd) == EOF) 5257 retval = FAIL; 5258 5259 if (fwv != (size_t)1) 5260 retval = FAIL; 5261 if (retval == FAIL) 5262 emsg(_(e_write)); 5263 5264 return retval; 5265 } 5266 5267 /* 5268 * Clear the index and wnode fields of "node", it siblings and its 5269 * children. This is needed because they are a union with other items to save 5270 * space. 5271 */ 5272 static void 5273 clear_node(wordnode_T *node) 5274 { 5275 wordnode_T *np; 5276 5277 if (node != NULL) 5278 FOR_ALL_NODE_SIBLINGS(node, np) 5279 { 5280 np->wn_u1.index = 0; 5281 np->wn_u2.wnode = NULL; 5282 5283 if (np->wn_byte != NUL) 5284 clear_node(np->wn_child); 5285 } 5286 } 5287 5288 5289 /* 5290 * Dump a word tree at node "node". 5291 * 5292 * This first writes the list of possible bytes (siblings). Then for each 5293 * byte recursively write the children. 5294 * 5295 * NOTE: The code here must match the code in read_tree_node(), since 5296 * assumptions are made about the indexes (so that we don't have to write them 5297 * in the file). 5298 * 5299 * Returns the number of nodes used. 5300 */ 5301 static int 5302 put_node( 5303 FILE *fd, // NULL when only counting 5304 wordnode_T *node, 5305 int idx, 5306 int regionmask, 5307 int prefixtree) // TRUE for PREFIXTREE 5308 { 5309 int newindex = idx; 5310 int siblingcount = 0; 5311 wordnode_T *np; 5312 int flags; 5313 5314 // If "node" is zero the tree is empty. 5315 if (node == NULL) 5316 return 0; 5317 5318 // Store the index where this node is written. 5319 node->wn_u1.index = idx; 5320 5321 // Count the number of siblings. 5322 FOR_ALL_NODE_SIBLINGS(node, np) 5323 ++siblingcount; 5324 5325 // Write the sibling count. 5326 if (fd != NULL) 5327 putc(siblingcount, fd); // <siblingcount> 5328 5329 // Write each sibling byte and optionally extra info. 5330 FOR_ALL_NODE_SIBLINGS(node, np) 5331 { 5332 if (np->wn_byte == 0) 5333 { 5334 if (fd != NULL) 5335 { 5336 // For a NUL byte (end of word) write the flags etc. 5337 if (prefixtree) 5338 { 5339 // In PREFIXTREE write the required affixID and the 5340 // associated condition nr (stored in wn_region). The 5341 // byte value is misused to store the "rare" and "not 5342 // combining" flags 5343 if (np->wn_flags == (short_u)PFX_FLAGS) 5344 putc(BY_NOFLAGS, fd); // <byte> 5345 else 5346 { 5347 putc(BY_FLAGS, fd); // <byte> 5348 putc(np->wn_flags, fd); // <pflags> 5349 } 5350 putc(np->wn_affixID, fd); // <affixID> 5351 put_bytes(fd, (long_u)np->wn_region, 2); // <prefcondnr> 5352 } 5353 else 5354 { 5355 // For word trees we write the flag/region items. 5356 flags = np->wn_flags; 5357 if (regionmask != 0 && np->wn_region != regionmask) 5358 flags |= WF_REGION; 5359 if (np->wn_affixID != 0) 5360 flags |= WF_AFX; 5361 if (flags == 0) 5362 { 5363 // word without flags or region 5364 putc(BY_NOFLAGS, fd); // <byte> 5365 } 5366 else 5367 { 5368 if (np->wn_flags >= 0x100) 5369 { 5370 putc(BY_FLAGS2, fd); // <byte> 5371 putc(flags, fd); // <flags> 5372 putc((unsigned)flags >> 8, fd); // <flags2> 5373 } 5374 else 5375 { 5376 putc(BY_FLAGS, fd); // <byte> 5377 putc(flags, fd); // <flags> 5378 } 5379 if (flags & WF_REGION) 5380 putc(np->wn_region, fd); // <region> 5381 if (flags & WF_AFX) 5382 putc(np->wn_affixID, fd); // <affixID> 5383 } 5384 } 5385 } 5386 } 5387 else 5388 { 5389 if (np->wn_child->wn_u1.index != 0 5390 && np->wn_child->wn_u2.wnode != node) 5391 { 5392 // The child is written elsewhere, write the reference. 5393 if (fd != NULL) 5394 { 5395 putc(BY_INDEX, fd); // <byte> 5396 // <nodeidx> 5397 put_bytes(fd, (long_u)np->wn_child->wn_u1.index, 3); 5398 } 5399 } 5400 else if (np->wn_child->wn_u2.wnode == NULL) 5401 // We will write the child below and give it an index. 5402 np->wn_child->wn_u2.wnode = node; 5403 5404 if (fd != NULL) 5405 if (putc(np->wn_byte, fd) == EOF) // <byte> or <xbyte> 5406 { 5407 emsg(_(e_write)); 5408 return 0; 5409 } 5410 } 5411 } 5412 5413 // Space used in the array when reading: one for each sibling and one for 5414 // the count. 5415 newindex += siblingcount + 1; 5416 5417 // Recursively dump the children of each sibling. 5418 FOR_ALL_NODE_SIBLINGS(node, np) 5419 if (np->wn_byte != 0 && np->wn_child->wn_u2.wnode == node) 5420 newindex = put_node(fd, np->wn_child, newindex, regionmask, 5421 prefixtree); 5422 5423 return newindex; 5424 } 5425 5426 5427 /* 5428 * ":mkspell [-ascii] outfile infile ..." 5429 * ":mkspell [-ascii] addfile" 5430 */ 5431 void 5432 ex_mkspell(exarg_T *eap) 5433 { 5434 int fcount; 5435 char_u **fnames; 5436 char_u *arg = eap->arg; 5437 int ascii = FALSE; 5438 5439 if (STRNCMP(arg, "-ascii", 6) == 0) 5440 { 5441 ascii = TRUE; 5442 arg = skipwhite(arg + 6); 5443 } 5444 5445 // Expand all the remaining arguments (e.g., $VIMRUNTIME). 5446 if (get_arglist_exp(arg, &fcount, &fnames, FALSE) == OK) 5447 { 5448 mkspell(fcount, fnames, ascii, eap->forceit, FALSE); 5449 FreeWild(fcount, fnames); 5450 } 5451 } 5452 5453 /* 5454 * Create the .sug file. 5455 * Uses the soundfold info in "spin". 5456 * Writes the file with the name "wfname", with ".spl" changed to ".sug". 5457 */ 5458 static void 5459 spell_make_sugfile(spellinfo_T *spin, char_u *wfname) 5460 { 5461 char_u *fname = NULL; 5462 int len; 5463 slang_T *slang; 5464 int free_slang = FALSE; 5465 5466 /* 5467 * Read back the .spl file that was written. This fills the required 5468 * info for soundfolding. This also uses less memory than the 5469 * pointer-linked version of the trie. And it avoids having two versions 5470 * of the code for the soundfolding stuff. 5471 * It might have been done already by spell_reload_one(). 5472 */ 5473 FOR_ALL_SPELL_LANGS(slang) 5474 if (fullpathcmp(wfname, slang->sl_fname, FALSE, TRUE) == FPC_SAME) 5475 break; 5476 if (slang == NULL) 5477 { 5478 spell_message(spin, (char_u *)_("Reading back spell file...")); 5479 slang = spell_load_file(wfname, NULL, NULL, FALSE); 5480 if (slang == NULL) 5481 return; 5482 free_slang = TRUE; 5483 } 5484 5485 /* 5486 * Clear the info in "spin" that is used. 5487 */ 5488 spin->si_blocks = NULL; 5489 spin->si_blocks_cnt = 0; 5490 spin->si_compress_cnt = 0; // will stay at 0 all the time 5491 spin->si_free_count = 0; 5492 spin->si_first_free = NULL; 5493 spin->si_foldwcount = 0; 5494 5495 /* 5496 * Go through the trie of good words, soundfold each word and add it to 5497 * the soundfold trie. 5498 */ 5499 spell_message(spin, (char_u *)_("Performing soundfolding...")); 5500 if (sug_filltree(spin, slang) == FAIL) 5501 goto theend; 5502 5503 /* 5504 * Create the table which links each soundfold word with a list of the 5505 * good words it may come from. Creates buffer "spin->si_spellbuf". 5506 * This also removes the wordnr from the NUL byte entries to make 5507 * compression possible. 5508 */ 5509 if (sug_maketable(spin) == FAIL) 5510 goto theend; 5511 5512 smsg(_("Number of words after soundfolding: %ld"), 5513 (long)spin->si_spellbuf->b_ml.ml_line_count); 5514 5515 /* 5516 * Compress the soundfold trie. 5517 */ 5518 spell_message(spin, (char_u *)_(msg_compressing)); 5519 wordtree_compress(spin, spin->si_foldroot, "case-folded"); 5520 5521 /* 5522 * Write the .sug file. 5523 * Make the file name by changing ".spl" to ".sug". 5524 */ 5525 fname = alloc(MAXPATHL); 5526 if (fname == NULL) 5527 goto theend; 5528 vim_strncpy(fname, wfname, MAXPATHL - 1); 5529 len = (int)STRLEN(fname); 5530 fname[len - 2] = 'u'; 5531 fname[len - 1] = 'g'; 5532 sug_write(spin, fname); 5533 5534 theend: 5535 vim_free(fname); 5536 if (free_slang) 5537 slang_free(slang); 5538 free_blocks(spin->si_blocks); 5539 close_spellbuf(spin->si_spellbuf); 5540 } 5541 5542 /* 5543 * Build the soundfold trie for language "slang". 5544 */ 5545 static int 5546 sug_filltree(spellinfo_T *spin, slang_T *slang) 5547 { 5548 char_u *byts; 5549 idx_T *idxs; 5550 int depth; 5551 idx_T arridx[MAXWLEN]; 5552 int curi[MAXWLEN]; 5553 char_u tword[MAXWLEN]; 5554 char_u tsalword[MAXWLEN]; 5555 int c; 5556 idx_T n; 5557 unsigned words_done = 0; 5558 int wordcount[MAXWLEN]; 5559 5560 // We use si_foldroot for the soundfolded trie. 5561 spin->si_foldroot = wordtree_alloc(spin); 5562 if (spin->si_foldroot == NULL) 5563 return FAIL; 5564 5565 // let tree_add_word() know we're adding to the soundfolded tree 5566 spin->si_sugtree = TRUE; 5567 5568 /* 5569 * Go through the whole case-folded tree, soundfold each word and put it 5570 * in the trie. 5571 */ 5572 byts = slang->sl_fbyts; 5573 idxs = slang->sl_fidxs; 5574 5575 arridx[0] = 0; 5576 curi[0] = 1; 5577 wordcount[0] = 0; 5578 5579 depth = 0; 5580 while (depth >= 0 && !got_int) 5581 { 5582 if (curi[depth] > byts[arridx[depth]]) 5583 { 5584 // Done all bytes at this node, go up one level. 5585 idxs[arridx[depth]] = wordcount[depth]; 5586 if (depth > 0) 5587 wordcount[depth - 1] += wordcount[depth]; 5588 5589 --depth; 5590 line_breakcheck(); 5591 } 5592 else 5593 { 5594 5595 // Do one more byte at this node. 5596 n = arridx[depth] + curi[depth]; 5597 ++curi[depth]; 5598 5599 c = byts[n]; 5600 if (c == 0) 5601 { 5602 // Sound-fold the word. 5603 tword[depth] = NUL; 5604 spell_soundfold(slang, tword, TRUE, tsalword); 5605 5606 // We use the "flags" field for the MSB of the wordnr, 5607 // "region" for the LSB of the wordnr. 5608 if (tree_add_word(spin, tsalword, spin->si_foldroot, 5609 words_done >> 16, words_done & 0xffff, 5610 0) == FAIL) 5611 return FAIL; 5612 5613 ++words_done; 5614 ++wordcount[depth]; 5615 5616 // Reset the block count each time to avoid compression 5617 // kicking in. 5618 spin->si_blocks_cnt = 0; 5619 5620 // Skip over any other NUL bytes (same word with different 5621 // flags). But don't go over the end. 5622 while (n + 1 < slang->sl_fbyts_len && byts[n + 1] == 0) 5623 { 5624 ++n; 5625 ++curi[depth]; 5626 } 5627 } 5628 else 5629 { 5630 // Normal char, go one level deeper. 5631 tword[depth++] = c; 5632 arridx[depth] = idxs[n]; 5633 curi[depth] = 1; 5634 wordcount[depth] = 0; 5635 } 5636 } 5637 } 5638 5639 smsg(_("Total number of words: %d"), words_done); 5640 5641 return OK; 5642 } 5643 5644 /* 5645 * Make the table that links each word in the soundfold trie to the words it 5646 * can be produced from. 5647 * This is not unlike lines in a file, thus use a memfile to be able to access 5648 * the table efficiently. 5649 * Returns FAIL when out of memory. 5650 */ 5651 static int 5652 sug_maketable(spellinfo_T *spin) 5653 { 5654 garray_T ga; 5655 int res = OK; 5656 5657 // Allocate a buffer, open a memline for it and create the swap file 5658 // (uses a temp file, not a .swp file). 5659 spin->si_spellbuf = open_spellbuf(); 5660 if (spin->si_spellbuf == NULL) 5661 return FAIL; 5662 5663 // Use a buffer to store the line info, avoids allocating many small 5664 // pieces of memory. 5665 ga_init2(&ga, 1, 100); 5666 5667 // recursively go through the tree 5668 if (sug_filltable(spin, spin->si_foldroot->wn_sibling, 0, &ga) == -1) 5669 res = FAIL; 5670 5671 ga_clear(&ga); 5672 return res; 5673 } 5674 5675 /* 5676 * Fill the table for one node and its children. 5677 * Returns the wordnr at the start of the node. 5678 * Returns -1 when out of memory. 5679 */ 5680 static int 5681 sug_filltable( 5682 spellinfo_T *spin, 5683 wordnode_T *node, 5684 int startwordnr, 5685 garray_T *gap) // place to store line of numbers 5686 { 5687 wordnode_T *p, *np; 5688 int wordnr = startwordnr; 5689 int nr; 5690 int prev_nr; 5691 5692 FOR_ALL_NODE_SIBLINGS(node, p) 5693 { 5694 if (p->wn_byte == NUL) 5695 { 5696 gap->ga_len = 0; 5697 prev_nr = 0; 5698 for (np = p; np != NULL && np->wn_byte == NUL; np = np->wn_sibling) 5699 { 5700 if (ga_grow(gap, 10) == FAIL) 5701 return -1; 5702 5703 nr = (np->wn_flags << 16) + (np->wn_region & 0xffff); 5704 // Compute the offset from the previous nr and store the 5705 // offset in a way that it takes a minimum number of bytes. 5706 // It's a bit like utf-8, but without the need to mark 5707 // following bytes. 5708 nr -= prev_nr; 5709 prev_nr += nr; 5710 gap->ga_len += offset2bytes(nr, 5711 (char_u *)gap->ga_data + gap->ga_len); 5712 } 5713 5714 // add the NUL byte 5715 ((char_u *)gap->ga_data)[gap->ga_len++] = NUL; 5716 5717 if (ml_append_buf(spin->si_spellbuf, (linenr_T)wordnr, 5718 gap->ga_data, gap->ga_len, TRUE) == FAIL) 5719 return -1; 5720 ++wordnr; 5721 5722 // Remove extra NUL entries, we no longer need them. We don't 5723 // bother freeing the nodes, the won't be reused anyway. 5724 while (p->wn_sibling != NULL && p->wn_sibling->wn_byte == NUL) 5725 p->wn_sibling = p->wn_sibling->wn_sibling; 5726 5727 // Clear the flags on the remaining NUL node, so that compression 5728 // works a lot better. 5729 p->wn_flags = 0; 5730 p->wn_region = 0; 5731 } 5732 else 5733 { 5734 wordnr = sug_filltable(spin, p->wn_child, wordnr, gap); 5735 if (wordnr == -1) 5736 return -1; 5737 } 5738 } 5739 return wordnr; 5740 } 5741 5742 /* 5743 * Convert an offset into a minimal number of bytes. 5744 * Similar to utf_char2byters, but use 8 bits in followup bytes and avoid NUL 5745 * bytes. 5746 */ 5747 static int 5748 offset2bytes(int nr, char_u *buf) 5749 { 5750 int rem; 5751 int b1, b2, b3, b4; 5752 5753 // Split the number in parts of base 255. We need to avoid NUL bytes. 5754 b1 = nr % 255 + 1; 5755 rem = nr / 255; 5756 b2 = rem % 255 + 1; 5757 rem = rem / 255; 5758 b3 = rem % 255 + 1; 5759 b4 = rem / 255 + 1; 5760 5761 if (b4 > 1 || b3 > 0x1f) // 4 bytes 5762 { 5763 buf[0] = 0xe0 + b4; 5764 buf[1] = b3; 5765 buf[2] = b2; 5766 buf[3] = b1; 5767 return 4; 5768 } 5769 if (b3 > 1 || b2 > 0x3f ) // 3 bytes 5770 { 5771 buf[0] = 0xc0 + b3; 5772 buf[1] = b2; 5773 buf[2] = b1; 5774 return 3; 5775 } 5776 if (b2 > 1 || b1 > 0x7f ) // 2 bytes 5777 { 5778 buf[0] = 0x80 + b2; 5779 buf[1] = b1; 5780 return 2; 5781 } 5782 // 1 byte 5783 buf[0] = b1; 5784 return 1; 5785 } 5786 5787 /* 5788 * Write the .sug file in "fname". 5789 */ 5790 static void 5791 sug_write(spellinfo_T *spin, char_u *fname) 5792 { 5793 FILE *fd; 5794 wordnode_T *tree; 5795 int nodecount; 5796 int wcount; 5797 char_u *line; 5798 linenr_T lnum; 5799 int len; 5800 5801 // Create the file. Note that an existing file is silently overwritten! 5802 fd = mch_fopen((char *)fname, "w"); 5803 if (fd == NULL) 5804 { 5805 semsg(_(e_notopen), fname); 5806 return; 5807 } 5808 5809 vim_snprintf((char *)IObuff, IOSIZE, 5810 _("Writing suggestion file %s..."), fname); 5811 spell_message(spin, IObuff); 5812 5813 /* 5814 * <SUGHEADER>: <fileID> <versionnr> <timestamp> 5815 */ 5816 if (fwrite(VIMSUGMAGIC, VIMSUGMAGICL, (size_t)1, fd) != 1) // <fileID> 5817 { 5818 emsg(_(e_write)); 5819 goto theend; 5820 } 5821 putc(VIMSUGVERSION, fd); // <versionnr> 5822 5823 // Write si_sugtime to the file. 5824 put_time(fd, spin->si_sugtime); // <timestamp> 5825 5826 /* 5827 * <SUGWORDTREE> 5828 */ 5829 spin->si_memtot = 0; 5830 tree = spin->si_foldroot->wn_sibling; 5831 5832 // Clear the index and wnode fields in the tree. 5833 clear_node(tree); 5834 5835 // Count the number of nodes. Needed to be able to allocate the 5836 // memory when reading the nodes. Also fills in index for shared 5837 // nodes. 5838 nodecount = put_node(NULL, tree, 0, 0, FALSE); 5839 5840 // number of nodes in 4 bytes 5841 put_bytes(fd, (long_u)nodecount, 4); // <nodecount> 5842 spin->si_memtot += nodecount + nodecount * sizeof(int); 5843 5844 // Write the nodes. 5845 (void)put_node(fd, tree, 0, 0, FALSE); 5846 5847 /* 5848 * <SUGTABLE>: <sugwcount> <sugline> ... 5849 */ 5850 wcount = spin->si_spellbuf->b_ml.ml_line_count; 5851 put_bytes(fd, (long_u)wcount, 4); // <sugwcount> 5852 5853 for (lnum = 1; lnum <= (linenr_T)wcount; ++lnum) 5854 { 5855 // <sugline>: <sugnr> ... NUL 5856 line = ml_get_buf(spin->si_spellbuf, lnum, FALSE); 5857 len = (int)STRLEN(line) + 1; 5858 if (fwrite(line, (size_t)len, (size_t)1, fd) == 0) 5859 { 5860 emsg(_(e_write)); 5861 goto theend; 5862 } 5863 spin->si_memtot += len; 5864 } 5865 5866 // Write another byte to check for errors. 5867 if (putc(0, fd) == EOF) 5868 emsg(_(e_write)); 5869 5870 vim_snprintf((char *)IObuff, IOSIZE, 5871 _("Estimated runtime memory use: %d bytes"), spin->si_memtot); 5872 spell_message(spin, IObuff); 5873 5874 theend: 5875 // close the file 5876 fclose(fd); 5877 } 5878 5879 5880 /* 5881 * Create a Vim spell file from one or more word lists. 5882 * "fnames[0]" is the output file name. 5883 * "fnames[fcount - 1]" is the last input file name. 5884 * Exception: when "fnames[0]" ends in ".add" it's used as the input file name 5885 * and ".spl" is appended to make the output file name. 5886 */ 5887 void 5888 mkspell( 5889 int fcount, 5890 char_u **fnames, 5891 int ascii, // -ascii argument given 5892 int over_write, // overwrite existing output file 5893 int added_word) // invoked through "zg" 5894 { 5895 char_u *fname = NULL; 5896 char_u *wfname; 5897 char_u **innames; 5898 int incount; 5899 afffile_T *(afile[MAXREGIONS]); 5900 int i; 5901 int len; 5902 stat_T st; 5903 int error = FALSE; 5904 spellinfo_T spin; 5905 5906 CLEAR_FIELD(spin); 5907 spin.si_verbose = !added_word; 5908 spin.si_ascii = ascii; 5909 spin.si_followup = TRUE; 5910 spin.si_rem_accents = TRUE; 5911 ga_init2(&spin.si_rep, (int)sizeof(fromto_T), 20); 5912 ga_init2(&spin.si_repsal, (int)sizeof(fromto_T), 20); 5913 ga_init2(&spin.si_sal, (int)sizeof(fromto_T), 20); 5914 ga_init2(&spin.si_map, (int)sizeof(char_u), 100); 5915 ga_init2(&spin.si_comppat, (int)sizeof(char_u *), 20); 5916 ga_init2(&spin.si_prefcond, (int)sizeof(char_u *), 50); 5917 hash_init(&spin.si_commonwords); 5918 spin.si_newcompID = 127; // start compound ID at first maximum 5919 5920 // default: fnames[0] is output file, following are input files 5921 // When "fcount" is 1 there is only one file. 5922 innames = &fnames[fcount == 1 ? 0 : 1]; 5923 incount = fcount - 1; 5924 5925 wfname = alloc(MAXPATHL); 5926 if (wfname == NULL) 5927 return; 5928 5929 if (fcount >= 1) 5930 { 5931 len = (int)STRLEN(fnames[0]); 5932 if (fcount == 1 && len > 4 && STRCMP(fnames[0] + len - 4, ".add") == 0) 5933 { 5934 // For ":mkspell path/en.latin1.add" output file is 5935 // "path/en.latin1.add.spl". 5936 incount = 1; 5937 vim_snprintf((char *)wfname, MAXPATHL, "%s.spl", fnames[0]); 5938 } 5939 else if (fcount == 1) 5940 { 5941 // For ":mkspell path/vim" output file is "path/vim.latin1.spl". 5942 incount = 1; 5943 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL, 5944 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 5945 } 5946 else if (len > 4 && STRCMP(fnames[0] + len - 4, ".spl") == 0) 5947 { 5948 // Name ends in ".spl", use as the file name. 5949 vim_strncpy(wfname, fnames[0], MAXPATHL - 1); 5950 } 5951 else 5952 // Name should be language, make the file name from it. 5953 vim_snprintf((char *)wfname, MAXPATHL, SPL_FNAME_TMPL, 5954 fnames[0], spin.si_ascii ? (char_u *)"ascii" : spell_enc()); 5955 5956 // Check for .ascii.spl. 5957 if (strstr((char *)gettail(wfname), SPL_FNAME_ASCII) != NULL) 5958 spin.si_ascii = TRUE; 5959 5960 // Check for .add.spl. 5961 if (strstr((char *)gettail(wfname), SPL_FNAME_ADD) != NULL) 5962 spin.si_add = TRUE; 5963 } 5964 5965 if (incount <= 0) 5966 emsg(_(e_invarg)); // need at least output and input names 5967 else if (vim_strchr(gettail(wfname), '_') != NULL) 5968 emsg(_("E751: Output file name must not have region name")); 5969 else if (incount > MAXREGIONS) 5970 semsg(_("E754: Only up to %d regions supported"), MAXREGIONS); 5971 else 5972 { 5973 // Check for overwriting before doing things that may take a lot of 5974 // time. 5975 if (!over_write && mch_stat((char *)wfname, &st) >= 0) 5976 { 5977 emsg(_(e_file_exists)); 5978 goto theend; 5979 } 5980 if (mch_isdir(wfname)) 5981 { 5982 semsg(_(e_src_is_directory), wfname); 5983 goto theend; 5984 } 5985 5986 fname = alloc(MAXPATHL); 5987 if (fname == NULL) 5988 goto theend; 5989 5990 /* 5991 * Init the aff and dic pointers. 5992 * Get the region names if there are more than 2 arguments. 5993 */ 5994 for (i = 0; i < incount; ++i) 5995 { 5996 afile[i] = NULL; 5997 5998 if (incount > 1) 5999 { 6000 len = (int)STRLEN(innames[i]); 6001 if (STRLEN(gettail(innames[i])) < 5 6002 || innames[i][len - 3] != '_') 6003 { 6004 semsg(_("E755: Invalid region in %s"), innames[i]); 6005 goto theend; 6006 } 6007 spin.si_region_name[i * 2] = TOLOWER_ASC(innames[i][len - 2]); 6008 spin.si_region_name[i * 2 + 1] = 6009 TOLOWER_ASC(innames[i][len - 1]); 6010 } 6011 } 6012 spin.si_region_count = incount; 6013 6014 spin.si_foldroot = wordtree_alloc(&spin); 6015 spin.si_keeproot = wordtree_alloc(&spin); 6016 spin.si_prefroot = wordtree_alloc(&spin); 6017 if (spin.si_foldroot == NULL 6018 || spin.si_keeproot == NULL 6019 || spin.si_prefroot == NULL) 6020 { 6021 free_blocks(spin.si_blocks); 6022 goto theend; 6023 } 6024 6025 // When not producing a .add.spl file clear the character table when 6026 // we encounter one in the .aff file. This means we dump the current 6027 // one in the .spl file if the .aff file doesn't define one. That's 6028 // better than guessing the contents, the table will match a 6029 // previously loaded spell file. 6030 if (!spin.si_add) 6031 spin.si_clear_chartab = TRUE; 6032 6033 /* 6034 * Read all the .aff and .dic files. 6035 * Text is converted to 'encoding'. 6036 * Words are stored in the case-folded and keep-case trees. 6037 */ 6038 for (i = 0; i < incount && !error; ++i) 6039 { 6040 spin.si_conv.vc_type = CONV_NONE; 6041 spin.si_region = 1 << i; 6042 6043 vim_snprintf((char *)fname, MAXPATHL, "%s.aff", innames[i]); 6044 if (mch_stat((char *)fname, &st) >= 0) 6045 { 6046 // Read the .aff file. Will init "spin->si_conv" based on the 6047 // "SET" line. 6048 afile[i] = spell_read_aff(&spin, fname); 6049 if (afile[i] == NULL) 6050 error = TRUE; 6051 else 6052 { 6053 // Read the .dic file and store the words in the trees. 6054 vim_snprintf((char *)fname, MAXPATHL, "%s.dic", 6055 innames[i]); 6056 if (spell_read_dic(&spin, fname, afile[i]) == FAIL) 6057 error = TRUE; 6058 } 6059 } 6060 else 6061 { 6062 // No .aff file, try reading the file as a word list. Store 6063 // the words in the trees. 6064 if (spell_read_wordfile(&spin, innames[i]) == FAIL) 6065 error = TRUE; 6066 } 6067 6068 // Free any conversion stuff. 6069 convert_setup(&spin.si_conv, NULL, NULL); 6070 } 6071 6072 if (spin.si_compflags != NULL && spin.si_nobreak) 6073 msg(_("Warning: both compounding and NOBREAK specified")); 6074 6075 if (!error && !got_int) 6076 { 6077 /* 6078 * Combine tails in the tree. 6079 */ 6080 spell_message(&spin, (char_u *)_(msg_compressing)); 6081 wordtree_compress(&spin, spin.si_foldroot, "case-folded"); 6082 wordtree_compress(&spin, spin.si_keeproot, "keep-case"); 6083 wordtree_compress(&spin, spin.si_prefroot, "prefixes"); 6084 } 6085 6086 if (!error && !got_int) 6087 { 6088 /* 6089 * Write the info in the spell file. 6090 */ 6091 vim_snprintf((char *)IObuff, IOSIZE, 6092 _("Writing spell file %s..."), wfname); 6093 spell_message(&spin, IObuff); 6094 6095 error = write_vim_spell(&spin, wfname) == FAIL; 6096 6097 spell_message(&spin, (char_u *)_("Done!")); 6098 vim_snprintf((char *)IObuff, IOSIZE, 6099 _("Estimated runtime memory use: %d bytes"), spin.si_memtot); 6100 spell_message(&spin, IObuff); 6101 6102 /* 6103 * If the file is loaded need to reload it. 6104 */ 6105 if (!error) 6106 spell_reload_one(wfname, added_word); 6107 } 6108 6109 // Free the allocated memory. 6110 ga_clear(&spin.si_rep); 6111 ga_clear(&spin.si_repsal); 6112 ga_clear(&spin.si_sal); 6113 ga_clear(&spin.si_map); 6114 ga_clear(&spin.si_comppat); 6115 ga_clear(&spin.si_prefcond); 6116 hash_clear_all(&spin.si_commonwords, 0); 6117 6118 // Free the .aff file structures. 6119 for (i = 0; i < incount; ++i) 6120 if (afile[i] != NULL) 6121 spell_free_aff(afile[i]); 6122 6123 // Free all the bits and pieces at once. 6124 free_blocks(spin.si_blocks); 6125 6126 /* 6127 * If there is soundfolding info and no NOSUGFILE item create the 6128 * .sug file with the soundfolded word trie. 6129 */ 6130 if (spin.si_sugtime != 0 && !error && !got_int) 6131 spell_make_sugfile(&spin, wfname); 6132 6133 } 6134 6135 theend: 6136 vim_free(fname); 6137 vim_free(wfname); 6138 } 6139 6140 /* 6141 * Display a message for spell file processing when 'verbose' is set or using 6142 * ":mkspell". "str" can be IObuff. 6143 */ 6144 static void 6145 spell_message(spellinfo_T *spin, char_u *str) 6146 { 6147 if (spin->si_verbose || p_verbose > 2) 6148 { 6149 if (!spin->si_verbose) 6150 verbose_enter(); 6151 msg((char *)str); 6152 out_flush(); 6153 if (!spin->si_verbose) 6154 verbose_leave(); 6155 } 6156 } 6157 6158 /* 6159 * ":[count]spellgood {word}" 6160 * ":[count]spellwrong {word}" 6161 * ":[count]spellundo {word}" 6162 * ":[count]spellrare {word}" 6163 */ 6164 void 6165 ex_spell(exarg_T *eap) 6166 { 6167 spell_add_word(eap->arg, (int)STRLEN(eap->arg), 6168 eap->cmdidx == CMD_spellwrong ? SPELL_ADD_BAD : 6169 eap->cmdidx == CMD_spellrare ? SPELL_ADD_RARE : SPELL_ADD_GOOD, 6170 eap->forceit ? 0 : (int)eap->line2, 6171 eap->cmdidx == CMD_spellundo); 6172 } 6173 6174 /* 6175 * Add "word[len]" to 'spellfile' as a good, rare or bad word. 6176 */ 6177 void 6178 spell_add_word( 6179 char_u *word, 6180 int len, 6181 int what, // SPELL_ADD_ values 6182 int idx, // "zG" and "zW": zero, otherwise index in 6183 // 'spellfile' 6184 int undo) // TRUE for "zug", "zuG", "zuw" and "zuW" 6185 { 6186 FILE *fd = NULL; 6187 buf_T *buf = NULL; 6188 int new_spf = FALSE; 6189 char_u *fname; 6190 char_u *fnamebuf = NULL; 6191 char_u line[MAXWLEN * 2]; 6192 long fpos, fpos_next = 0; 6193 int i; 6194 char_u *spf; 6195 6196 if (idx == 0) // use internal wordlist 6197 { 6198 if (int_wordlist == NULL) 6199 { 6200 int_wordlist = vim_tempname('s', FALSE); 6201 if (int_wordlist == NULL) 6202 return; 6203 } 6204 fname = int_wordlist; 6205 } 6206 else 6207 { 6208 // If 'spellfile' isn't set figure out a good default value. 6209 if (*curwin->w_s->b_p_spf == NUL) 6210 { 6211 init_spellfile(); 6212 new_spf = TRUE; 6213 } 6214 6215 if (*curwin->w_s->b_p_spf == NUL) 6216 { 6217 semsg(_(e_notset), "spellfile"); 6218 return; 6219 } 6220 fnamebuf = alloc(MAXPATHL); 6221 if (fnamebuf == NULL) 6222 return; 6223 6224 for (spf = curwin->w_s->b_p_spf, i = 1; *spf != NUL; ++i) 6225 { 6226 copy_option_part(&spf, fnamebuf, MAXPATHL, ","); 6227 if (i == idx) 6228 break; 6229 if (*spf == NUL) 6230 { 6231 semsg(_("E765: 'spellfile' does not have %d entries"), idx); 6232 vim_free(fnamebuf); 6233 return; 6234 } 6235 } 6236 6237 // Check that the user isn't editing the .add file somewhere. 6238 buf = buflist_findname_exp(fnamebuf); 6239 if (buf != NULL && buf->b_ml.ml_mfp == NULL) 6240 buf = NULL; 6241 if (buf != NULL && bufIsChanged(buf)) 6242 { 6243 emsg(_(e_bufloaded)); 6244 vim_free(fnamebuf); 6245 return; 6246 } 6247 6248 fname = fnamebuf; 6249 } 6250 6251 if (what == SPELL_ADD_BAD || undo) 6252 { 6253 // When the word appears as good word we need to remove that one, 6254 // since its flags sort before the one with WF_BANNED. 6255 fd = mch_fopen((char *)fname, "r"); 6256 if (fd != NULL) 6257 { 6258 while (!vim_fgets(line, MAXWLEN * 2, fd)) 6259 { 6260 fpos = fpos_next; 6261 fpos_next = ftell(fd); 6262 if (STRNCMP(word, line, len) == 0 6263 && (line[len] == '/' || line[len] < ' ')) 6264 { 6265 // Found duplicate word. Remove it by writing a '#' at 6266 // the start of the line. Mixing reading and writing 6267 // doesn't work for all systems, close the file first. 6268 fclose(fd); 6269 fd = mch_fopen((char *)fname, "r+"); 6270 if (fd == NULL) 6271 break; 6272 if (fseek(fd, fpos, SEEK_SET) == 0) 6273 { 6274 fputc('#', fd); 6275 if (undo) 6276 { 6277 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 6278 smsg(_("Word '%.*s' removed from %s"), 6279 len, word, NameBuff); 6280 } 6281 } 6282 if (fseek(fd, fpos_next, SEEK_SET) != 0) 6283 { 6284 PERROR(_("Seek error in spellfile")); 6285 break; 6286 } 6287 } 6288 } 6289 if (fd != NULL) 6290 fclose(fd); 6291 } 6292 } 6293 6294 if (!undo) 6295 { 6296 fd = mch_fopen((char *)fname, "a"); 6297 if (fd == NULL && new_spf) 6298 { 6299 char_u *p; 6300 6301 // We just initialized the 'spellfile' option and can't open the 6302 // file. We may need to create the "spell" directory first. We 6303 // already checked the runtime directory is writable in 6304 // init_spellfile(). 6305 if (!dir_of_file_exists(fname) && (p = gettail_sep(fname)) != fname) 6306 { 6307 int c = *p; 6308 6309 // The directory doesn't exist. Try creating it and opening 6310 // the file again. 6311 *p = NUL; 6312 vim_mkdir(fname, 0755); 6313 *p = c; 6314 fd = mch_fopen((char *)fname, "a"); 6315 } 6316 } 6317 6318 if (fd == NULL) 6319 semsg(_(e_notopen), fname); 6320 else 6321 { 6322 if (what == SPELL_ADD_BAD) 6323 fprintf(fd, "%.*s/!\n", len, word); 6324 else if (what == SPELL_ADD_RARE) 6325 fprintf(fd, "%.*s/?\n", len, word); 6326 else 6327 fprintf(fd, "%.*s\n", len, word); 6328 fclose(fd); 6329 6330 home_replace(NULL, fname, NameBuff, MAXPATHL, TRUE); 6331 smsg(_("Word '%.*s' added to %s"), len, word, NameBuff); 6332 } 6333 } 6334 6335 if (fd != NULL) 6336 { 6337 // Update the .add.spl file. 6338 mkspell(1, &fname, FALSE, TRUE, TRUE); 6339 6340 // If the .add file is edited somewhere, reload it. 6341 if (buf != NULL) 6342 buf_reload(buf, buf->b_orig_mode); 6343 6344 redraw_all_later(SOME_VALID); 6345 } 6346 vim_free(fnamebuf); 6347 } 6348 6349 /* 6350 * Initialize 'spellfile' for the current buffer. 6351 */ 6352 static void 6353 init_spellfile(void) 6354 { 6355 char_u *buf; 6356 int l; 6357 char_u *fname; 6358 char_u *rtp; 6359 char_u *lend; 6360 int aspath = FALSE; 6361 char_u *lstart = curbuf->b_s.b_p_spl; 6362 6363 if (*curwin->w_s->b_p_spl != NUL && curwin->w_s->b_langp.ga_len > 0) 6364 { 6365 buf = alloc(MAXPATHL); 6366 if (buf == NULL) 6367 return; 6368 6369 // Find the end of the language name. Exclude the region. If there 6370 // is a path separator remember the start of the tail. 6371 for (lend = curwin->w_s->b_p_spl; *lend != NUL 6372 && vim_strchr((char_u *)",._", *lend) == NULL; ++lend) 6373 if (vim_ispathsep(*lend)) 6374 { 6375 aspath = TRUE; 6376 lstart = lend + 1; 6377 } 6378 6379 // Loop over all entries in 'runtimepath'. Use the first one where we 6380 // are allowed to write. 6381 rtp = p_rtp; 6382 while (*rtp != NUL) 6383 { 6384 if (aspath) 6385 // Use directory of an entry with path, e.g., for 6386 // "/dir/lg.utf-8.spl" use "/dir". 6387 vim_strncpy(buf, curbuf->b_s.b_p_spl, 6388 lstart - curbuf->b_s.b_p_spl - 1); 6389 else 6390 // Copy the path from 'runtimepath' to buf[]. 6391 copy_option_part(&rtp, buf, MAXPATHL, ","); 6392 if (filewritable(buf) == 2) 6393 { 6394 // Use the first language name from 'spelllang' and the 6395 // encoding used in the first loaded .spl file. 6396 if (aspath) 6397 vim_strncpy(buf, curbuf->b_s.b_p_spl, 6398 lend - curbuf->b_s.b_p_spl); 6399 else 6400 { 6401 // Create the "spell" directory if it doesn't exist yet. 6402 l = (int)STRLEN(buf); 6403 vim_snprintf((char *)buf + l, MAXPATHL - l, "/spell"); 6404 if (filewritable(buf) != 2) 6405 vim_mkdir(buf, 0755); 6406 6407 l = (int)STRLEN(buf); 6408 vim_snprintf((char *)buf + l, MAXPATHL - l, 6409 "/%.*s", (int)(lend - lstart), lstart); 6410 } 6411 l = (int)STRLEN(buf); 6412 fname = LANGP_ENTRY(curwin->w_s->b_langp, 0) 6413 ->lp_slang->sl_fname; 6414 vim_snprintf((char *)buf + l, MAXPATHL - l, ".%s.add", 6415 fname != NULL 6416 && strstr((char *)gettail(fname), ".ascii.") != NULL 6417 ? (char_u *)"ascii" : spell_enc()); 6418 set_option_value((char_u *)"spellfile", 0L, buf, OPT_LOCAL); 6419 break; 6420 } 6421 aspath = FALSE; 6422 } 6423 6424 vim_free(buf); 6425 } 6426 } 6427 6428 6429 6430 /* 6431 * Set the spell character tables from strings in the affix file. 6432 */ 6433 static int 6434 set_spell_chartab(char_u *fol, char_u *low, char_u *upp) 6435 { 6436 // We build the new tables here first, so that we can compare with the 6437 // previous one. 6438 spelltab_T new_st; 6439 char_u *pf = fol, *pl = low, *pu = upp; 6440 int f, l, u; 6441 6442 clear_spell_chartab(&new_st); 6443 6444 while (*pf != NUL) 6445 { 6446 if (*pl == NUL || *pu == NUL) 6447 { 6448 emsg(_(e_affform)); 6449 return FAIL; 6450 } 6451 f = mb_ptr2char_adv(&pf); 6452 l = mb_ptr2char_adv(&pl); 6453 u = mb_ptr2char_adv(&pu); 6454 6455 // Every character that appears is a word character. 6456 if (f < 256) 6457 new_st.st_isw[f] = TRUE; 6458 if (l < 256) 6459 new_st.st_isw[l] = TRUE; 6460 if (u < 256) 6461 new_st.st_isw[u] = TRUE; 6462 6463 // if "LOW" and "FOL" are not the same the "LOW" char needs 6464 // case-folding 6465 if (l < 256 && l != f) 6466 { 6467 if (f >= 256) 6468 { 6469 emsg(_(e_affrange)); 6470 return FAIL; 6471 } 6472 new_st.st_fold[l] = f; 6473 } 6474 6475 // if "UPP" and "FOL" are not the same the "UPP" char needs 6476 // case-folding, it's upper case and the "UPP" is the upper case of 6477 // "FOL" . 6478 if (u < 256 && u != f) 6479 { 6480 if (f >= 256) 6481 { 6482 emsg(_(e_affrange)); 6483 return FAIL; 6484 } 6485 new_st.st_fold[u] = f; 6486 new_st.st_isu[u] = TRUE; 6487 new_st.st_upper[f] = u; 6488 } 6489 } 6490 6491 if (*pl != NUL || *pu != NUL) 6492 { 6493 emsg(_(e_affform)); 6494 return FAIL; 6495 } 6496 6497 return set_spell_finish(&new_st); 6498 } 6499 6500 /* 6501 * Set the spell character tables from strings in the .spl file. 6502 */ 6503 static void 6504 set_spell_charflags( 6505 char_u *flags, 6506 int cnt, // length of "flags" 6507 char_u *fol) 6508 { 6509 // We build the new tables here first, so that we can compare with the 6510 // previous one. 6511 spelltab_T new_st; 6512 int i; 6513 char_u *p = fol; 6514 int c; 6515 6516 clear_spell_chartab(&new_st); 6517 6518 for (i = 0; i < 128; ++i) 6519 { 6520 if (i < cnt) 6521 { 6522 new_st.st_isw[i + 128] = (flags[i] & CF_WORD) != 0; 6523 new_st.st_isu[i + 128] = (flags[i] & CF_UPPER) != 0; 6524 } 6525 6526 if (*p != NUL) 6527 { 6528 c = mb_ptr2char_adv(&p); 6529 new_st.st_fold[i + 128] = c; 6530 if (i + 128 != c && new_st.st_isu[i + 128] && c < 256) 6531 new_st.st_upper[c] = i + 128; 6532 } 6533 } 6534 6535 (void)set_spell_finish(&new_st); 6536 } 6537 6538 static int 6539 set_spell_finish(spelltab_T *new_st) 6540 { 6541 int i; 6542 6543 if (did_set_spelltab) 6544 { 6545 // check that it's the same table 6546 for (i = 0; i < 256; ++i) 6547 { 6548 if (spelltab.st_isw[i] != new_st->st_isw[i] 6549 || spelltab.st_isu[i] != new_st->st_isu[i] 6550 || spelltab.st_fold[i] != new_st->st_fold[i] 6551 || spelltab.st_upper[i] != new_st->st_upper[i]) 6552 { 6553 emsg(_("E763: Word characters differ between spell files")); 6554 return FAIL; 6555 } 6556 } 6557 } 6558 else 6559 { 6560 // copy the new spelltab into the one being used 6561 spelltab = *new_st; 6562 did_set_spelltab = TRUE; 6563 } 6564 6565 return OK; 6566 } 6567 6568 /* 6569 * Write the table with prefix conditions to the .spl file. 6570 * When "fd" is NULL only count the length of what is written. 6571 */ 6572 static int 6573 write_spell_prefcond(FILE *fd, garray_T *gap) 6574 { 6575 int i; 6576 char_u *p; 6577 int len; 6578 int totlen; 6579 size_t x = 1; // collect return value of fwrite() 6580 6581 if (fd != NULL) 6582 put_bytes(fd, (long_u)gap->ga_len, 2); // <prefcondcnt> 6583 6584 totlen = 2 + gap->ga_len; // length of <prefcondcnt> and <condlen> bytes 6585 6586 for (i = 0; i < gap->ga_len; ++i) 6587 { 6588 // <prefcond> : <condlen> <condstr> 6589 p = ((char_u **)gap->ga_data)[i]; 6590 if (p != NULL) 6591 { 6592 len = (int)STRLEN(p); 6593 if (fd != NULL) 6594 { 6595 fputc(len, fd); 6596 x &= fwrite(p, (size_t)len, (size_t)1, fd); 6597 } 6598 totlen += len; 6599 } 6600 else if (fd != NULL) 6601 fputc(0, fd); 6602 } 6603 6604 return totlen; 6605 } 6606 6607 6608 /* 6609 * Use map string "map" for languages "lp". 6610 */ 6611 static void 6612 set_map_str(slang_T *lp, char_u *map) 6613 { 6614 char_u *p; 6615 int headc = 0; 6616 int c; 6617 int i; 6618 6619 if (*map == NUL) 6620 { 6621 lp->sl_has_map = FALSE; 6622 return; 6623 } 6624 lp->sl_has_map = TRUE; 6625 6626 // Init the array and hash tables empty. 6627 for (i = 0; i < 256; ++i) 6628 lp->sl_map_array[i] = 0; 6629 hash_init(&lp->sl_map_hash); 6630 6631 /* 6632 * The similar characters are stored separated with slashes: 6633 * "aaa/bbb/ccc/". Fill sl_map_array[c] with the character before c and 6634 * before the same slash. For characters above 255 sl_map_hash is used. 6635 */ 6636 for (p = map; *p != NUL; ) 6637 { 6638 c = mb_cptr2char_adv(&p); 6639 if (c == '/') 6640 headc = 0; 6641 else 6642 { 6643 if (headc == 0) 6644 headc = c; 6645 6646 // Characters above 255 don't fit in sl_map_array[], put them in 6647 // the hash table. Each entry is the char, a NUL the headchar and 6648 // a NUL. 6649 if (c >= 256) 6650 { 6651 int cl = mb_char2len(c); 6652 int headcl = mb_char2len(headc); 6653 char_u *b; 6654 hash_T hash; 6655 hashitem_T *hi; 6656 6657 b = alloc(cl + headcl + 2); 6658 if (b == NULL) 6659 return; 6660 mb_char2bytes(c, b); 6661 b[cl] = NUL; 6662 mb_char2bytes(headc, b + cl + 1); 6663 b[cl + 1 + headcl] = NUL; 6664 hash = hash_hash(b); 6665 hi = hash_lookup(&lp->sl_map_hash, b, hash); 6666 if (HASHITEM_EMPTY(hi)) 6667 hash_add_item(&lp->sl_map_hash, hi, b, hash); 6668 else 6669 { 6670 // This should have been checked when generating the .spl 6671 // file. 6672 emsg(_("E783: duplicate char in MAP entry")); 6673 vim_free(b); 6674 } 6675 } 6676 else 6677 lp->sl_map_array[c] = headc; 6678 } 6679 } 6680 } 6681 6682 #endif // FEAT_SPELL 6683