266         u32 iCode;  in fts5UnicodeAddExceptions()  local
268         READ_UTF8(zCsr, zTerm, iCode);  in fts5UnicodeAddExceptions()
269         if( iCode<128 ){  in fts5UnicodeAddExceptions()
270           p->aTokenChar[iCode] = (unsigned char)bTokenChars;  in fts5UnicodeAddExceptions()
272           bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];  in fts5UnicodeAddExceptions()
275           if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){  in fts5UnicodeAddExceptions()
278               if( (u32)aNew[i]>iCode ) break;  in fts5UnicodeAddExceptions()
281             aNew[i] = iCode;  in fts5UnicodeAddExceptions()
299 static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){  in fts5UnicodeIsException()  argument
307       if( iCode==a[iTest] ){  in fts5UnicodeIsException()
309       }else if( iCode>a[iTest] ){  in fts5UnicodeIsException()
431 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){  in fts5UnicodeIsAlnum()  argument
433     p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]  in fts5UnicodeIsAlnum()
434     ^ fts5UnicodeIsException(p, iCode)  in fts5UnicodeIsAlnum()
462     u32 iCode;                    /* non-ASCII codepoint read from input */  in fts5UnicodeTokenize()  local
474         READ_UTF8(zCsr, zTerm, iCode);  in fts5UnicodeTokenize()
475         if( fts5UnicodeIsAlnum(p, iCode) ){  in fts5UnicodeTokenize()
510         READ_UTF8(zCsr, zTerm, iCode);  in fts5UnicodeTokenize()
511         if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){  in fts5UnicodeTokenize()
513           iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);  in fts5UnicodeTokenize()
514           if( iCode ) WRITE_UTF8(zOut, iCode);  in fts5UnicodeTokenize()
1329   u32 iCode;  in fts5TriTokenize()  local
1337     READ_UTF8(zIn, zEof, iCode);  in fts5TriTokenize()
1338     if( iCode==0 ) break;  in fts5TriTokenize()
1341       if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);  in fts5TriTokenize()
1342       WRITE_UTF8(zOut, iCode);  in fts5TriTokenize()
1343       READ_UTF8(zIn, zEof, iCode);  in fts5TriTokenize()
1344       if( iCode==0 ) break;  in fts5TriTokenize()
1349       if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);  in fts5TriTokenize()
1350       WRITE_UTF8(zOut, iCode);  in fts5TriTokenize()
1351       READ_UTF8(zIn, zEof, iCode);  in fts5TriTokenize()
1352       if( iCode==0 ) break;  in fts5TriTokenize()
1353       if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);  in fts5TriTokenize()
1354       WRITE_UTF8(zOut, iCode);  in fts5TriTokenize()