Searched refs:tokenizer (Results 1 – 8 of 8) sorted by relevance
15 SRCS= eval.c expr.c look.c main.c misc.c gnum4.c trace.c parser.y tokenizer.l19 tokenizer.o: parser.h21 CLEANFILES+= parser.c parser.h tokenizer.o
34 SRCS= editline.c readline.c tokenizer.c history.c 74 sh ${.CURDIR}/makelist -n tokenizer.c > ${.TARGET}
238 typedef struct tokenizer Tokenizer;
74 struct TYPE(tokenizer) { in TYPE() argument
547 a tokenizer for the front ends.
448 straightforward. The lexical analyzer's tokenizer hands each
3775 Get rid of unnecessary eol variable in tokenizer.
226659 fts5_tokenizer tokenizer; /* Parent tokenizer module */226671 p->tokenizer.xDelete(p->pTokenizer);226698 rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);226705 rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);227363 return p->tokenizer.xTokenize(