Lines Matching refs:Tokenizer
86 class Tokenizer { class
88 Tokenizer(StringRef Input) : Data(Input), DataLength(Input.size()), Pos(0) {} in Tokenizer() function in __anon727922840111::Tokenizer
148 void Tokenizer::skipCurrentLine() { in skipCurrentLine()
156 Expected<std::vector<RCToken>> Tokenizer::run() { in run()
199 bool Tokenizer::advance(size_t Amount) { in advance()
204 bool Tokenizer::skipWhitespaces() { in skipWhitespaces()
210 Error Tokenizer::consumeToken(const Kind TokenKind) { in consumeToken()
276 bool Tokenizer::willNowRead(StringRef FollowingChars) const { in willNowRead()
280 bool Tokenizer::canStartIdentifier() const { in canStartIdentifier()
287 bool Tokenizer::canContinueIdentifier() const { in canContinueIdentifier()
294 bool Tokenizer::canStartInt() const { in canStartInt()
299 bool Tokenizer::canStartBlockComment() const { in canStartBlockComment()
304 bool Tokenizer::canStartLineComment() const { in canStartLineComment()
309 bool Tokenizer::canContinueInt() const { in canContinueInt()
314 bool Tokenizer::canStartString() const { in canStartString()
318 bool Tokenizer::streamEof() const { return Pos == DataLength; } in streamEof()
320 Kind Tokenizer::classifyCurrentToken() const { in classifyCurrentToken()
349 void Tokenizer::processIdentifier(RCToken &Token) const { in processIdentifier()
364 return Tokenizer(Input).run(); in tokenizeRC()