15188c4b9SDmitri Gribenko #include "clang/AST/CommentLexer.h" 25188c4b9SDmitri Gribenko #include "llvm/ADT/StringSwitch.h" 35188c4b9SDmitri Gribenko #include "llvm/Support/ErrorHandling.h" 45188c4b9SDmitri Gribenko 55188c4b9SDmitri Gribenko namespace clang { 65188c4b9SDmitri Gribenko namespace comments { 75188c4b9SDmitri Gribenko 85188c4b9SDmitri Gribenko void Token::dump(const Lexer &L, const SourceManager &SM) const { 95188c4b9SDmitri Gribenko llvm::errs() << "comments::Token Kind=" << Kind << " "; 105188c4b9SDmitri Gribenko Loc.dump(SM); 115188c4b9SDmitri Gribenko llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n"; 125188c4b9SDmitri Gribenko } 135188c4b9SDmitri Gribenko 145188c4b9SDmitri Gribenko bool Lexer::isVerbatimBlockCommand(StringRef BeginName, 155188c4b9SDmitri Gribenko StringRef &EndName) const { 165188c4b9SDmitri Gribenko const char *Result = llvm::StringSwitch<const char *>(BeginName) 175188c4b9SDmitri Gribenko .Case("code", "endcode") 185188c4b9SDmitri Gribenko .Case("verbatim", "endverbatim") 195188c4b9SDmitri Gribenko .Case("htmlonly", "endhtmlonly") 205188c4b9SDmitri Gribenko .Case("latexonly", "endlatexonly") 215188c4b9SDmitri Gribenko .Case("xmlonly", "endxmlonly") 225188c4b9SDmitri Gribenko .Case("manonly", "endmanonly") 235188c4b9SDmitri Gribenko .Case("rtfonly", "endrtfonly") 245188c4b9SDmitri Gribenko 255188c4b9SDmitri Gribenko .Case("dot", "enddot") 265188c4b9SDmitri Gribenko .Case("msc", "endmsc") 275188c4b9SDmitri Gribenko 285188c4b9SDmitri Gribenko .Case("f$", "f$") // Inline LaTeX formula 295188c4b9SDmitri Gribenko .Case("f[", "f]") // Displayed LaTeX formula 305188c4b9SDmitri Gribenko .Case("f{", "f}") // LaTeX environment 315188c4b9SDmitri Gribenko 325188c4b9SDmitri Gribenko .Default(NULL); 335188c4b9SDmitri Gribenko 345188c4b9SDmitri Gribenko if (Result) { 355188c4b9SDmitri Gribenko EndName = Result; 365188c4b9SDmitri Gribenko return true; 375188c4b9SDmitri Gribenko } 385188c4b9SDmitri Gribenko 395188c4b9SDmitri Gribenko for (VerbatimBlockCommandVector::const_iterator 405188c4b9SDmitri Gribenko I = VerbatimBlockCommands.begin(), 415188c4b9SDmitri Gribenko E = VerbatimBlockCommands.end(); 425188c4b9SDmitri Gribenko I != E; ++I) 435188c4b9SDmitri Gribenko if (I->BeginName == BeginName) { 445188c4b9SDmitri Gribenko EndName = I->EndName; 455188c4b9SDmitri Gribenko return true; 465188c4b9SDmitri Gribenko } 475188c4b9SDmitri Gribenko 485188c4b9SDmitri Gribenko return false; 495188c4b9SDmitri Gribenko } 505188c4b9SDmitri Gribenko 515188c4b9SDmitri Gribenko bool Lexer::isVerbatimLineCommand(StringRef Name) const { 525188c4b9SDmitri Gribenko bool Result = llvm::StringSwitch<bool>(Name) 535188c4b9SDmitri Gribenko .Case("fn", true) 545188c4b9SDmitri Gribenko .Case("var", true) 555188c4b9SDmitri Gribenko .Case("property", true) 565188c4b9SDmitri Gribenko .Case("typedef", true) 575188c4b9SDmitri Gribenko 585188c4b9SDmitri Gribenko .Case("overload", true) 595188c4b9SDmitri Gribenko 605188c4b9SDmitri Gribenko .Case("defgroup", true) 615188c4b9SDmitri Gribenko .Case("ingroup", true) 625188c4b9SDmitri Gribenko .Case("addtogroup", true) 635188c4b9SDmitri Gribenko .Case("weakgroup", true) 645188c4b9SDmitri Gribenko .Case("name", true) 655188c4b9SDmitri Gribenko 665188c4b9SDmitri Gribenko .Case("section", true) 675188c4b9SDmitri Gribenko .Case("subsection", true) 685188c4b9SDmitri Gribenko .Case("subsubsection", true) 695188c4b9SDmitri Gribenko .Case("paragraph", true) 705188c4b9SDmitri Gribenko 715188c4b9SDmitri Gribenko .Case("mainpage", true) 725188c4b9SDmitri Gribenko .Case("subpage", true) 735188c4b9SDmitri Gribenko .Case("ref", true) 745188c4b9SDmitri Gribenko 755188c4b9SDmitri Gribenko .Default(false); 765188c4b9SDmitri Gribenko 775188c4b9SDmitri Gribenko if (Result) 785188c4b9SDmitri Gribenko return true; 795188c4b9SDmitri Gribenko 805188c4b9SDmitri Gribenko for (VerbatimLineCommandVector::const_iterator 815188c4b9SDmitri Gribenko I = VerbatimLineCommands.begin(), 825188c4b9SDmitri Gribenko E = VerbatimLineCommands.end(); 835188c4b9SDmitri Gribenko I != E; ++I) 845188c4b9SDmitri Gribenko if (I->Name == Name) 855188c4b9SDmitri Gribenko return true; 865188c4b9SDmitri Gribenko 875188c4b9SDmitri Gribenko return false; 885188c4b9SDmitri Gribenko } 895188c4b9SDmitri Gribenko 905188c4b9SDmitri Gribenko void Lexer::skipLineStartingDecorations() { 915188c4b9SDmitri Gribenko // This function should be called only for C comments 925188c4b9SDmitri Gribenko assert(CommentState == LCS_InsideCComment); 935188c4b9SDmitri Gribenko 945188c4b9SDmitri Gribenko if (BufferPtr == CommentEnd) 955188c4b9SDmitri Gribenko return; 965188c4b9SDmitri Gribenko 975188c4b9SDmitri Gribenko switch (*BufferPtr) { 985188c4b9SDmitri Gribenko case ' ': 995188c4b9SDmitri Gribenko case '\t': 1005188c4b9SDmitri Gribenko case '\f': 1015188c4b9SDmitri Gribenko case '\v': { 1025188c4b9SDmitri Gribenko const char *NewBufferPtr = BufferPtr; 1035188c4b9SDmitri Gribenko NewBufferPtr++; 1045188c4b9SDmitri Gribenko if (NewBufferPtr == CommentEnd) 1055188c4b9SDmitri Gribenko return; 1065188c4b9SDmitri Gribenko 1075188c4b9SDmitri Gribenko char C = *NewBufferPtr; 1085188c4b9SDmitri Gribenko while (C == ' ' || C == '\t' || C == '\f' || C == '\v') { 1095188c4b9SDmitri Gribenko NewBufferPtr++; 1105188c4b9SDmitri Gribenko if (NewBufferPtr == CommentEnd) 1115188c4b9SDmitri Gribenko return; 1125188c4b9SDmitri Gribenko C = *NewBufferPtr; 1135188c4b9SDmitri Gribenko } 1145188c4b9SDmitri Gribenko if (C == '*') 1155188c4b9SDmitri Gribenko BufferPtr = NewBufferPtr + 1; 1165188c4b9SDmitri Gribenko break; 1175188c4b9SDmitri Gribenko } 1185188c4b9SDmitri Gribenko case '*': 1195188c4b9SDmitri Gribenko BufferPtr++; 1205188c4b9SDmitri Gribenko break; 1215188c4b9SDmitri Gribenko } 1225188c4b9SDmitri Gribenko } 1235188c4b9SDmitri Gribenko 1245188c4b9SDmitri Gribenko namespace { 125ec92531cSDmitri Gribenko /// Returns pointer to the first newline character in the string. 1265188c4b9SDmitri Gribenko const char *findNewline(const char *BufferPtr, const char *BufferEnd) { 1275188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 1285188c4b9SDmitri Gribenko const char C = *BufferPtr; 1295188c4b9SDmitri Gribenko if (C == '\n' || C == '\r') 1305188c4b9SDmitri Gribenko return BufferPtr; 1315188c4b9SDmitri Gribenko } 1325188c4b9SDmitri Gribenko return BufferEnd; 1335188c4b9SDmitri Gribenko } 1345188c4b9SDmitri Gribenko 1355188c4b9SDmitri Gribenko const char *skipNewline(const char *BufferPtr, const char *BufferEnd) { 1365188c4b9SDmitri Gribenko if (BufferPtr == BufferEnd) 1375188c4b9SDmitri Gribenko return BufferPtr; 1385188c4b9SDmitri Gribenko 1395188c4b9SDmitri Gribenko if (*BufferPtr == '\n') 1405188c4b9SDmitri Gribenko BufferPtr++; 1415188c4b9SDmitri Gribenko else { 1425188c4b9SDmitri Gribenko assert(*BufferPtr == '\r'); 1435188c4b9SDmitri Gribenko BufferPtr++; 1445188c4b9SDmitri Gribenko if (BufferPtr != BufferEnd && *BufferPtr == '\n') 1455188c4b9SDmitri Gribenko BufferPtr++; 1465188c4b9SDmitri Gribenko } 1475188c4b9SDmitri Gribenko return BufferPtr; 1485188c4b9SDmitri Gribenko } 1495188c4b9SDmitri Gribenko 15017709ae8SDmitri Gribenko bool isHTMLIdentifierStartingCharacter(char C) { 15117709ae8SDmitri Gribenko return (C >= 'a' && C <= 'z') || 15217709ae8SDmitri Gribenko (C >= 'A' && C <= 'Z'); 15317709ae8SDmitri Gribenko } 15417709ae8SDmitri Gribenko 1555188c4b9SDmitri Gribenko bool isHTMLIdentifierCharacter(char C) { 1565188c4b9SDmitri Gribenko return (C >= 'a' && C <= 'z') || 1575188c4b9SDmitri Gribenko (C >= 'A' && C <= 'Z') || 1585188c4b9SDmitri Gribenko (C >= '0' && C <= '9'); 1595188c4b9SDmitri Gribenko } 1605188c4b9SDmitri Gribenko 1615188c4b9SDmitri Gribenko const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) { 1625188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 1635188c4b9SDmitri Gribenko if (!isHTMLIdentifierCharacter(*BufferPtr)) 1645188c4b9SDmitri Gribenko return BufferPtr; 1655188c4b9SDmitri Gribenko } 1665188c4b9SDmitri Gribenko return BufferEnd; 1675188c4b9SDmitri Gribenko } 1685188c4b9SDmitri Gribenko 1695188c4b9SDmitri Gribenko /// Skip HTML string quoted in single or double quotes. Escaping quotes inside 1705188c4b9SDmitri Gribenko /// string allowed. 1715188c4b9SDmitri Gribenko /// 1725188c4b9SDmitri Gribenko /// Returns pointer to closing quote. 1735188c4b9SDmitri Gribenko const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd) 1745188c4b9SDmitri Gribenko { 1755188c4b9SDmitri Gribenko const char Quote = *BufferPtr; 1765188c4b9SDmitri Gribenko assert(Quote == '\"' || Quote == '\''); 1775188c4b9SDmitri Gribenko 1785188c4b9SDmitri Gribenko BufferPtr++; 1795188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 1805188c4b9SDmitri Gribenko const char C = *BufferPtr; 1815188c4b9SDmitri Gribenko if (C == Quote && BufferPtr[-1] != '\\') 1825188c4b9SDmitri Gribenko return BufferPtr; 1835188c4b9SDmitri Gribenko } 1845188c4b9SDmitri Gribenko return BufferEnd; 1855188c4b9SDmitri Gribenko } 1865188c4b9SDmitri Gribenko 1875188c4b9SDmitri Gribenko bool isHorizontalWhitespace(char C) { 1885188c4b9SDmitri Gribenko return C == ' ' || C == '\t' || C == '\f' || C == '\v'; 1895188c4b9SDmitri Gribenko } 1905188c4b9SDmitri Gribenko 1915188c4b9SDmitri Gribenko bool isWhitespace(char C) { 1925188c4b9SDmitri Gribenko return C == ' ' || C == '\n' || C == '\r' || 1935188c4b9SDmitri Gribenko C == '\t' || C == '\f' || C == '\v'; 1945188c4b9SDmitri Gribenko } 1955188c4b9SDmitri Gribenko 1965188c4b9SDmitri Gribenko const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) { 1975188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 1985188c4b9SDmitri Gribenko if (!isWhitespace(*BufferPtr)) 1995188c4b9SDmitri Gribenko return BufferPtr; 2005188c4b9SDmitri Gribenko } 2015188c4b9SDmitri Gribenko return BufferEnd; 2025188c4b9SDmitri Gribenko } 2035188c4b9SDmitri Gribenko 204*e4a3997dSDmitri Gribenko bool isWhitespace(const char *BufferPtr, const char *BufferEnd) { 205*e4a3997dSDmitri Gribenko return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd; 206*e4a3997dSDmitri Gribenko } 207*e4a3997dSDmitri Gribenko 2085188c4b9SDmitri Gribenko bool isCommandNameCharacter(char C) { 2095188c4b9SDmitri Gribenko return (C >= 'a' && C <= 'z') || 2105188c4b9SDmitri Gribenko (C >= 'A' && C <= 'Z') || 2115188c4b9SDmitri Gribenko (C >= '0' && C <= '9'); 2125188c4b9SDmitri Gribenko } 2135188c4b9SDmitri Gribenko 2145188c4b9SDmitri Gribenko const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) { 2155188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 2165188c4b9SDmitri Gribenko if (!isCommandNameCharacter(*BufferPtr)) 2175188c4b9SDmitri Gribenko return BufferPtr; 2185188c4b9SDmitri Gribenko } 2195188c4b9SDmitri Gribenko return BufferEnd; 2205188c4b9SDmitri Gribenko } 2215188c4b9SDmitri Gribenko 2225188c4b9SDmitri Gribenko /// Return the one past end pointer for BCPL comments. 2235188c4b9SDmitri Gribenko /// Handles newlines escaped with backslash or trigraph for backslahs. 2245188c4b9SDmitri Gribenko const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) { 2255188c4b9SDmitri Gribenko const char *CurPtr = BufferPtr; 2265188c4b9SDmitri Gribenko while (CurPtr != BufferEnd) { 2275188c4b9SDmitri Gribenko char C = *CurPtr; 2285188c4b9SDmitri Gribenko while (C != '\n' && C != '\r') { 2295188c4b9SDmitri Gribenko CurPtr++; 2305188c4b9SDmitri Gribenko if (CurPtr == BufferEnd) 2315188c4b9SDmitri Gribenko return BufferEnd; 2325188c4b9SDmitri Gribenko C = *CurPtr; 2335188c4b9SDmitri Gribenko } 2345188c4b9SDmitri Gribenko // We found a newline, check if it is escaped. 2355188c4b9SDmitri Gribenko const char *EscapePtr = CurPtr - 1; 2365188c4b9SDmitri Gribenko while(isHorizontalWhitespace(*EscapePtr)) 2375188c4b9SDmitri Gribenko EscapePtr--; 2385188c4b9SDmitri Gribenko 2395188c4b9SDmitri Gribenko if (*EscapePtr == '\\' || 2405188c4b9SDmitri Gribenko (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' && 2415188c4b9SDmitri Gribenko EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) { 2425188c4b9SDmitri Gribenko // We found an escaped newline. 2435188c4b9SDmitri Gribenko CurPtr = skipNewline(CurPtr, BufferEnd); 2445188c4b9SDmitri Gribenko } else 2455188c4b9SDmitri Gribenko return CurPtr; // Not an escaped newline. 2465188c4b9SDmitri Gribenko } 2475188c4b9SDmitri Gribenko return BufferEnd; 2485188c4b9SDmitri Gribenko } 2495188c4b9SDmitri Gribenko 2505188c4b9SDmitri Gribenko /// Return the one past end pointer for C comments. 2515188c4b9SDmitri Gribenko /// Very dumb, does not handle escaped newlines or trigraphs. 2525188c4b9SDmitri Gribenko const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) { 2535188c4b9SDmitri Gribenko for ( ; BufferPtr != BufferEnd; ++BufferPtr) { 2545188c4b9SDmitri Gribenko if (*BufferPtr == '*') { 2555188c4b9SDmitri Gribenko assert(BufferPtr + 1 != BufferEnd); 2565188c4b9SDmitri Gribenko if (*(BufferPtr + 1) == '/') 2575188c4b9SDmitri Gribenko return BufferPtr; 2585188c4b9SDmitri Gribenko } 2595188c4b9SDmitri Gribenko } 2605188c4b9SDmitri Gribenko llvm_unreachable("buffer end hit before '*/' was seen"); 2615188c4b9SDmitri Gribenko } 2625188c4b9SDmitri Gribenko } // unnamed namespace 2635188c4b9SDmitri Gribenko 2645188c4b9SDmitri Gribenko void Lexer::lexCommentText(Token &T) { 2655188c4b9SDmitri Gribenko assert(CommentState == LCS_InsideBCPLComment || 2665188c4b9SDmitri Gribenko CommentState == LCS_InsideCComment); 2675188c4b9SDmitri Gribenko 2685188c4b9SDmitri Gribenko switch (State) { 2695188c4b9SDmitri Gribenko case LS_Normal: 2705188c4b9SDmitri Gribenko break; 2715188c4b9SDmitri Gribenko case LS_VerbatimBlockFirstLine: 2725188c4b9SDmitri Gribenko lexVerbatimBlockFirstLine(T); 2735188c4b9SDmitri Gribenko return; 2745188c4b9SDmitri Gribenko case LS_VerbatimBlockBody: 2755188c4b9SDmitri Gribenko lexVerbatimBlockBody(T); 2765188c4b9SDmitri Gribenko return; 2771669f702SDmitri Gribenko case LS_VerbatimLineText: 2781669f702SDmitri Gribenko lexVerbatimLineText(T); 2791669f702SDmitri Gribenko return; 280e00ffc7bSDmitri Gribenko case LS_HTMLStartTag: 281e00ffc7bSDmitri Gribenko lexHTMLStartTag(T); 2825188c4b9SDmitri Gribenko return; 283e00ffc7bSDmitri Gribenko case LS_HTMLEndTag: 284e00ffc7bSDmitri Gribenko lexHTMLEndTag(T); 285ec92531cSDmitri Gribenko return; 2865188c4b9SDmitri Gribenko } 2875188c4b9SDmitri Gribenko 2885188c4b9SDmitri Gribenko assert(State == LS_Normal); 2895188c4b9SDmitri Gribenko 2905188c4b9SDmitri Gribenko const char *TokenPtr = BufferPtr; 2915188c4b9SDmitri Gribenko assert(TokenPtr < CommentEnd); 2925188c4b9SDmitri Gribenko while (TokenPtr != CommentEnd) { 2935188c4b9SDmitri Gribenko switch(*TokenPtr) { 2945188c4b9SDmitri Gribenko case '\\': 2955188c4b9SDmitri Gribenko case '@': { 2965188c4b9SDmitri Gribenko TokenPtr++; 2975188c4b9SDmitri Gribenko if (TokenPtr == CommentEnd) { 29860ddd8a1SDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 2995188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 30060ddd8a1SDmitri Gribenko T.setText(Text); 3015188c4b9SDmitri Gribenko return; 3025188c4b9SDmitri Gribenko } 3035188c4b9SDmitri Gribenko char C = *TokenPtr; 3045188c4b9SDmitri Gribenko switch (C) { 3055188c4b9SDmitri Gribenko default: 3065188c4b9SDmitri Gribenko break; 3075188c4b9SDmitri Gribenko 3085188c4b9SDmitri Gribenko case '\\': case '@': case '&': case '$': 3095188c4b9SDmitri Gribenko case '#': case '<': case '>': case '%': 3105188c4b9SDmitri Gribenko case '\"': case '.': case ':': 3115188c4b9SDmitri Gribenko // This is one of \\ \@ \& \$ etc escape sequences. 3125188c4b9SDmitri Gribenko TokenPtr++; 3135188c4b9SDmitri Gribenko if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') { 3145188c4b9SDmitri Gribenko // This is the \:: escape sequence. 3155188c4b9SDmitri Gribenko TokenPtr++; 3165188c4b9SDmitri Gribenko } 31760ddd8a1SDmitri Gribenko StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1)); 3185188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 31960ddd8a1SDmitri Gribenko T.setText(UnescapedText); 3205188c4b9SDmitri Gribenko return; 3215188c4b9SDmitri Gribenko } 3225188c4b9SDmitri Gribenko 3235188c4b9SDmitri Gribenko // Don't make zero-length commands. 3245188c4b9SDmitri Gribenko if (!isCommandNameCharacter(*TokenPtr)) { 32560ddd8a1SDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 3265188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 32760ddd8a1SDmitri Gribenko T.setText(Text); 3285188c4b9SDmitri Gribenko return; 3295188c4b9SDmitri Gribenko } 3305188c4b9SDmitri Gribenko 3315188c4b9SDmitri Gribenko TokenPtr = skipCommandName(TokenPtr, CommentEnd); 3325188c4b9SDmitri Gribenko unsigned Length = TokenPtr - (BufferPtr + 1); 3335188c4b9SDmitri Gribenko 3345188c4b9SDmitri Gribenko // Hardcoded support for lexing LaTeX formula commands 3355188c4b9SDmitri Gribenko // \f$ \f[ \f] \f{ \f} as a single command. 3365188c4b9SDmitri Gribenko if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) { 3375188c4b9SDmitri Gribenko C = *TokenPtr; 3385188c4b9SDmitri Gribenko if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') { 3395188c4b9SDmitri Gribenko TokenPtr++; 3405188c4b9SDmitri Gribenko Length++; 3415188c4b9SDmitri Gribenko } 3425188c4b9SDmitri Gribenko } 3435188c4b9SDmitri Gribenko 3445188c4b9SDmitri Gribenko const StringRef CommandName(BufferPtr + 1, Length); 3455188c4b9SDmitri Gribenko StringRef EndName; 3465188c4b9SDmitri Gribenko 3475188c4b9SDmitri Gribenko if (isVerbatimBlockCommand(CommandName, EndName)) { 3485188c4b9SDmitri Gribenko setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, EndName); 3495188c4b9SDmitri Gribenko return; 3505188c4b9SDmitri Gribenko } 3515188c4b9SDmitri Gribenko if (isVerbatimLineCommand(CommandName)) { 3521669f702SDmitri Gribenko setupAndLexVerbatimLine(T, TokenPtr); 3535188c4b9SDmitri Gribenko return; 3545188c4b9SDmitri Gribenko } 3555188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::command); 3565188c4b9SDmitri Gribenko T.setCommandName(CommandName); 3575188c4b9SDmitri Gribenko return; 3585188c4b9SDmitri Gribenko } 3595188c4b9SDmitri Gribenko 3605188c4b9SDmitri Gribenko case '<': { 3615188c4b9SDmitri Gribenko TokenPtr++; 3625188c4b9SDmitri Gribenko if (TokenPtr == CommentEnd) { 36360ddd8a1SDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 3645188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 36560ddd8a1SDmitri Gribenko T.setText(Text); 3665188c4b9SDmitri Gribenko return; 3675188c4b9SDmitri Gribenko } 3685188c4b9SDmitri Gribenko const char C = *TokenPtr; 36917709ae8SDmitri Gribenko if (isHTMLIdentifierStartingCharacter(C)) 370e00ffc7bSDmitri Gribenko setupAndLexHTMLStartTag(T); 3715188c4b9SDmitri Gribenko else if (C == '/') 372e00ffc7bSDmitri Gribenko setupAndLexHTMLEndTag(T); 373632d58afSDmitri Gribenko else { 374632d58afSDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 375632d58afSDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 376632d58afSDmitri Gribenko T.setText(Text); 377632d58afSDmitri Gribenko } 3785188c4b9SDmitri Gribenko return; 3795188c4b9SDmitri Gribenko } 3805188c4b9SDmitri Gribenko 3815188c4b9SDmitri Gribenko case '\n': 3825188c4b9SDmitri Gribenko case '\r': 3835188c4b9SDmitri Gribenko TokenPtr = skipNewline(TokenPtr, CommentEnd); 3845188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::newline); 3855188c4b9SDmitri Gribenko 3865188c4b9SDmitri Gribenko if (CommentState == LCS_InsideCComment) 3875188c4b9SDmitri Gribenko skipLineStartingDecorations(); 3885188c4b9SDmitri Gribenko return; 3895188c4b9SDmitri Gribenko 3905188c4b9SDmitri Gribenko default: { 3915188c4b9SDmitri Gribenko while (true) { 3925188c4b9SDmitri Gribenko TokenPtr++; 3935188c4b9SDmitri Gribenko if (TokenPtr == CommentEnd) 3945188c4b9SDmitri Gribenko break; 39517709ae8SDmitri Gribenko const char C = *TokenPtr; 3965188c4b9SDmitri Gribenko if(C == '\n' || C == '\r' || 3975188c4b9SDmitri Gribenko C == '\\' || C == '@' || C == '<') 3985188c4b9SDmitri Gribenko break; 3995188c4b9SDmitri Gribenko } 40060ddd8a1SDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 4015188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 40260ddd8a1SDmitri Gribenko T.setText(Text); 4035188c4b9SDmitri Gribenko return; 4045188c4b9SDmitri Gribenko } 4055188c4b9SDmitri Gribenko } 4065188c4b9SDmitri Gribenko } 4075188c4b9SDmitri Gribenko } 4085188c4b9SDmitri Gribenko 4095188c4b9SDmitri Gribenko void Lexer::setupAndLexVerbatimBlock(Token &T, 4105188c4b9SDmitri Gribenko const char *TextBegin, 4115188c4b9SDmitri Gribenko char Marker, StringRef EndName) { 4125188c4b9SDmitri Gribenko VerbatimBlockEndCommandName.clear(); 4135188c4b9SDmitri Gribenko VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@"); 4145188c4b9SDmitri Gribenko VerbatimBlockEndCommandName.append(EndName); 4155188c4b9SDmitri Gribenko 41660ddd8a1SDmitri Gribenko StringRef Name(BufferPtr + 1, TextBegin - (BufferPtr + 1)); 4175188c4b9SDmitri Gribenko formTokenWithChars(T, TextBegin, tok::verbatim_block_begin); 41860ddd8a1SDmitri Gribenko T.setVerbatimBlockName(Name); 4195188c4b9SDmitri Gribenko 420ec92531cSDmitri Gribenko // If there is a newline following the verbatim opening command, skip the 421ec92531cSDmitri Gribenko // newline so that we don't create an tok::verbatim_block_line with empty 422ec92531cSDmitri Gribenko // text content. 423ec92531cSDmitri Gribenko if (BufferPtr != CommentEnd) { 424ec92531cSDmitri Gribenko const char C = *BufferPtr; 425ec92531cSDmitri Gribenko if (C == '\n' || C == '\r') { 426ec92531cSDmitri Gribenko BufferPtr = skipNewline(BufferPtr, CommentEnd); 427ec92531cSDmitri Gribenko State = LS_VerbatimBlockBody; 428ec92531cSDmitri Gribenko return; 429ec92531cSDmitri Gribenko } 430ec92531cSDmitri Gribenko } 431ec92531cSDmitri Gribenko 4325188c4b9SDmitri Gribenko State = LS_VerbatimBlockFirstLine; 4335188c4b9SDmitri Gribenko } 4345188c4b9SDmitri Gribenko 4355188c4b9SDmitri Gribenko void Lexer::lexVerbatimBlockFirstLine(Token &T) { 436*e4a3997dSDmitri Gribenko again: 4375188c4b9SDmitri Gribenko assert(BufferPtr < CommentEnd); 4385188c4b9SDmitri Gribenko 4395188c4b9SDmitri Gribenko // FIXME: It would be better to scan the text once, finding either the block 4405188c4b9SDmitri Gribenko // end command or newline. 4415188c4b9SDmitri Gribenko // 4425188c4b9SDmitri Gribenko // Extract current line. 4435188c4b9SDmitri Gribenko const char *Newline = findNewline(BufferPtr, CommentEnd); 4445188c4b9SDmitri Gribenko StringRef Line(BufferPtr, Newline - BufferPtr); 4455188c4b9SDmitri Gribenko 4465188c4b9SDmitri Gribenko // Look for end command in current line. 4475188c4b9SDmitri Gribenko size_t Pos = Line.find(VerbatimBlockEndCommandName); 448ec92531cSDmitri Gribenko const char *TextEnd; 4495188c4b9SDmitri Gribenko const char *NextLine; 4505188c4b9SDmitri Gribenko if (Pos == StringRef::npos) { 4515188c4b9SDmitri Gribenko // Current line is completely verbatim. 452ec92531cSDmitri Gribenko TextEnd = Newline; 4535188c4b9SDmitri Gribenko NextLine = skipNewline(Newline, CommentEnd); 4545188c4b9SDmitri Gribenko } else if (Pos == 0) { 4555188c4b9SDmitri Gribenko // Current line contains just an end command. 4565188c4b9SDmitri Gribenko const char *End = BufferPtr + VerbatimBlockEndCommandName.size(); 45760ddd8a1SDmitri Gribenko StringRef Name(BufferPtr + 1, End - (BufferPtr + 1)); 4585188c4b9SDmitri Gribenko formTokenWithChars(T, End, tok::verbatim_block_end); 45960ddd8a1SDmitri Gribenko T.setVerbatimBlockName(Name); 4605188c4b9SDmitri Gribenko State = LS_Normal; 4615188c4b9SDmitri Gribenko return; 4625188c4b9SDmitri Gribenko } else { 4635188c4b9SDmitri Gribenko // There is some text, followed by end command. Extract text first. 464ec92531cSDmitri Gribenko TextEnd = BufferPtr + Pos; 465ec92531cSDmitri Gribenko NextLine = TextEnd; 466*e4a3997dSDmitri Gribenko // If there is only whitespace before end command, skip whitespace. 467*e4a3997dSDmitri Gribenko if (isWhitespace(BufferPtr, TextEnd)) { 468*e4a3997dSDmitri Gribenko BufferPtr = TextEnd; 469*e4a3997dSDmitri Gribenko goto again; 470*e4a3997dSDmitri Gribenko } 4715188c4b9SDmitri Gribenko } 4725188c4b9SDmitri Gribenko 473ec92531cSDmitri Gribenko StringRef Text(BufferPtr, TextEnd - BufferPtr); 4745188c4b9SDmitri Gribenko formTokenWithChars(T, NextLine, tok::verbatim_block_line); 47560ddd8a1SDmitri Gribenko T.setVerbatimBlockText(Text); 4765188c4b9SDmitri Gribenko 4775188c4b9SDmitri Gribenko State = LS_VerbatimBlockBody; 4785188c4b9SDmitri Gribenko } 4795188c4b9SDmitri Gribenko 4805188c4b9SDmitri Gribenko void Lexer::lexVerbatimBlockBody(Token &T) { 4815188c4b9SDmitri Gribenko assert(State == LS_VerbatimBlockBody); 4825188c4b9SDmitri Gribenko 4835188c4b9SDmitri Gribenko if (CommentState == LCS_InsideCComment) 4845188c4b9SDmitri Gribenko skipLineStartingDecorations(); 4855188c4b9SDmitri Gribenko 4865188c4b9SDmitri Gribenko lexVerbatimBlockFirstLine(T); 4875188c4b9SDmitri Gribenko } 4885188c4b9SDmitri Gribenko 4891669f702SDmitri Gribenko void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) { 4901669f702SDmitri Gribenko const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1); 4911669f702SDmitri Gribenko formTokenWithChars(T, TextBegin, tok::verbatim_line_name); 4921669f702SDmitri Gribenko T.setVerbatimLineName(Name); 4931669f702SDmitri Gribenko 4941669f702SDmitri Gribenko State = LS_VerbatimLineText; 4951669f702SDmitri Gribenko } 4961669f702SDmitri Gribenko 4971669f702SDmitri Gribenko void Lexer::lexVerbatimLineText(Token &T) { 4981669f702SDmitri Gribenko assert(State == LS_VerbatimLineText); 4991669f702SDmitri Gribenko 5005188c4b9SDmitri Gribenko // Extract current line. 5015188c4b9SDmitri Gribenko const char *Newline = findNewline(BufferPtr, CommentEnd); 5021669f702SDmitri Gribenko const StringRef Text(BufferPtr, Newline - BufferPtr); 5031669f702SDmitri Gribenko formTokenWithChars(T, Newline, tok::verbatim_line_text); 5045188c4b9SDmitri Gribenko T.setVerbatimLineText(Text); 5051669f702SDmitri Gribenko 5061669f702SDmitri Gribenko State = LS_Normal; 5075188c4b9SDmitri Gribenko } 5085188c4b9SDmitri Gribenko 509e00ffc7bSDmitri Gribenko void Lexer::setupAndLexHTMLStartTag(Token &T) { 51017709ae8SDmitri Gribenko assert(BufferPtr[0] == '<' && 51117709ae8SDmitri Gribenko isHTMLIdentifierStartingCharacter(BufferPtr[1])); 5125188c4b9SDmitri Gribenko const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd); 5135188c4b9SDmitri Gribenko 51460ddd8a1SDmitri Gribenko StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1)); 515e00ffc7bSDmitri Gribenko formTokenWithChars(T, TagNameEnd, tok::html_start_tag); 516e00ffc7bSDmitri Gribenko T.setHTMLTagStartName(Name); 5175188c4b9SDmitri Gribenko 5185188c4b9SDmitri Gribenko BufferPtr = skipWhitespace(BufferPtr, CommentEnd); 5195188c4b9SDmitri Gribenko 52017709ae8SDmitri Gribenko const char C = *BufferPtr; 52117709ae8SDmitri Gribenko if (BufferPtr != CommentEnd && 522f26054f0SDmitri Gribenko (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C))) 523e00ffc7bSDmitri Gribenko State = LS_HTMLStartTag; 5245188c4b9SDmitri Gribenko } 5255188c4b9SDmitri Gribenko 526e00ffc7bSDmitri Gribenko void Lexer::lexHTMLStartTag(Token &T) { 527e00ffc7bSDmitri Gribenko assert(State == LS_HTMLStartTag); 5285188c4b9SDmitri Gribenko 5295188c4b9SDmitri Gribenko const char *TokenPtr = BufferPtr; 5305188c4b9SDmitri Gribenko char C = *TokenPtr; 5315188c4b9SDmitri Gribenko if (isHTMLIdentifierCharacter(C)) { 5325188c4b9SDmitri Gribenko TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd); 53360ddd8a1SDmitri Gribenko StringRef Ident(BufferPtr, TokenPtr - BufferPtr); 5345188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::html_ident); 53560ddd8a1SDmitri Gribenko T.setHTMLIdent(Ident); 5365188c4b9SDmitri Gribenko } else { 5375188c4b9SDmitri Gribenko switch (C) { 5385188c4b9SDmitri Gribenko case '=': 5395188c4b9SDmitri Gribenko TokenPtr++; 5405188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::html_equals); 5415188c4b9SDmitri Gribenko break; 5425188c4b9SDmitri Gribenko case '\"': 5435188c4b9SDmitri Gribenko case '\'': { 5445188c4b9SDmitri Gribenko const char *OpenQuote = TokenPtr; 5455188c4b9SDmitri Gribenko TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd); 5465188c4b9SDmitri Gribenko const char *ClosingQuote = TokenPtr; 5475188c4b9SDmitri Gribenko if (TokenPtr != CommentEnd) // Skip closing quote. 5485188c4b9SDmitri Gribenko TokenPtr++; 5495188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::html_quoted_string); 5505188c4b9SDmitri Gribenko T.setHTMLQuotedString(StringRef(OpenQuote + 1, 5515188c4b9SDmitri Gribenko ClosingQuote - (OpenQuote + 1))); 5525188c4b9SDmitri Gribenko break; 5535188c4b9SDmitri Gribenko } 5545188c4b9SDmitri Gribenko case '>': 5555188c4b9SDmitri Gribenko TokenPtr++; 5565188c4b9SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::html_greater); 55717709ae8SDmitri Gribenko State = LS_Normal; 55817709ae8SDmitri Gribenko return; 559f26054f0SDmitri Gribenko case '/': 560f26054f0SDmitri Gribenko TokenPtr++; 561f26054f0SDmitri Gribenko if (TokenPtr != CommentEnd && *TokenPtr == '>') { 562f26054f0SDmitri Gribenko TokenPtr++; 563f26054f0SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::html_slash_greater); 564f26054f0SDmitri Gribenko } else { 565f26054f0SDmitri Gribenko StringRef Text(BufferPtr, TokenPtr - BufferPtr); 566f26054f0SDmitri Gribenko formTokenWithChars(T, TokenPtr, tok::text); 567f26054f0SDmitri Gribenko T.setText(Text); 568f26054f0SDmitri Gribenko } 569f26054f0SDmitri Gribenko State = LS_Normal; 570f26054f0SDmitri Gribenko return; 5715188c4b9SDmitri Gribenko } 5725188c4b9SDmitri Gribenko } 5735188c4b9SDmitri Gribenko 5745188c4b9SDmitri Gribenko // Now look ahead and return to normal state if we don't see any HTML tokens 5755188c4b9SDmitri Gribenko // ahead. 5765188c4b9SDmitri Gribenko BufferPtr = skipWhitespace(BufferPtr, CommentEnd); 5775188c4b9SDmitri Gribenko if (BufferPtr == CommentEnd) { 5785188c4b9SDmitri Gribenko State = LS_Normal; 5795188c4b9SDmitri Gribenko return; 5805188c4b9SDmitri Gribenko } 5815188c4b9SDmitri Gribenko 5825188c4b9SDmitri Gribenko C = *BufferPtr; 58317709ae8SDmitri Gribenko if (!isHTMLIdentifierStartingCharacter(C) && 5845188c4b9SDmitri Gribenko C != '=' && C != '\"' && C != '\'' && C != '>') { 5855188c4b9SDmitri Gribenko State = LS_Normal; 5865188c4b9SDmitri Gribenko return; 5875188c4b9SDmitri Gribenko } 5885188c4b9SDmitri Gribenko } 5895188c4b9SDmitri Gribenko 590e00ffc7bSDmitri Gribenko void Lexer::setupAndLexHTMLEndTag(Token &T) { 5915188c4b9SDmitri Gribenko assert(BufferPtr[0] == '<' && BufferPtr[1] == '/'); 5925188c4b9SDmitri Gribenko 5935188c4b9SDmitri Gribenko const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd); 5945188c4b9SDmitri Gribenko const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd); 5955188c4b9SDmitri Gribenko 5965188c4b9SDmitri Gribenko const char *End = skipWhitespace(TagNameEnd, CommentEnd); 5975188c4b9SDmitri Gribenko 598e00ffc7bSDmitri Gribenko formTokenWithChars(T, End, tok::html_end_tag); 599e00ffc7bSDmitri Gribenko T.setHTMLTagEndName(StringRef(TagNameBegin, TagNameEnd - TagNameBegin)); 600ec92531cSDmitri Gribenko 601ec92531cSDmitri Gribenko if (BufferPtr != CommentEnd && *BufferPtr == '>') 602e00ffc7bSDmitri Gribenko State = LS_HTMLEndTag; 603ec92531cSDmitri Gribenko } 604ec92531cSDmitri Gribenko 605e00ffc7bSDmitri Gribenko void Lexer::lexHTMLEndTag(Token &T) { 606ec92531cSDmitri Gribenko assert(BufferPtr != CommentEnd && *BufferPtr == '>'); 607ec92531cSDmitri Gribenko 608ec92531cSDmitri Gribenko formTokenWithChars(T, BufferPtr + 1, tok::html_greater); 609ec92531cSDmitri Gribenko State = LS_Normal; 6105188c4b9SDmitri Gribenko } 6115188c4b9SDmitri Gribenko 6125188c4b9SDmitri Gribenko Lexer::Lexer(SourceLocation FileLoc, const CommentOptions &CommOpts, 6135188c4b9SDmitri Gribenko const char *BufferStart, const char *BufferEnd): 6145188c4b9SDmitri Gribenko BufferStart(BufferStart), BufferEnd(BufferEnd), 6155188c4b9SDmitri Gribenko FileLoc(FileLoc), CommOpts(CommOpts), BufferPtr(BufferStart), 6165188c4b9SDmitri Gribenko CommentState(LCS_BeforeComment), State(LS_Normal) { 6175188c4b9SDmitri Gribenko } 6185188c4b9SDmitri Gribenko 6195188c4b9SDmitri Gribenko void Lexer::lex(Token &T) { 6205188c4b9SDmitri Gribenko again: 6215188c4b9SDmitri Gribenko switch (CommentState) { 6225188c4b9SDmitri Gribenko case LCS_BeforeComment: 6235188c4b9SDmitri Gribenko if (BufferPtr == BufferEnd) { 6245188c4b9SDmitri Gribenko formTokenWithChars(T, BufferPtr, tok::eof); 6255188c4b9SDmitri Gribenko return; 6265188c4b9SDmitri Gribenko } 6275188c4b9SDmitri Gribenko 6285188c4b9SDmitri Gribenko assert(*BufferPtr == '/'); 6295188c4b9SDmitri Gribenko BufferPtr++; // Skip first slash. 6305188c4b9SDmitri Gribenko switch(*BufferPtr) { 6315188c4b9SDmitri Gribenko case '/': { // BCPL comment. 6325188c4b9SDmitri Gribenko BufferPtr++; // Skip second slash. 6335188c4b9SDmitri Gribenko 6345188c4b9SDmitri Gribenko if (BufferPtr != BufferEnd) { 6355188c4b9SDmitri Gribenko // Skip Doxygen magic marker, if it is present. 6365188c4b9SDmitri Gribenko // It might be missing because of a typo //< or /*<, or because we 6375188c4b9SDmitri Gribenko // merged this non-Doxygen comment into a bunch of Doxygen comments 6385188c4b9SDmitri Gribenko // around it: /** ... */ /* ... */ /** ... */ 6395188c4b9SDmitri Gribenko const char C = *BufferPtr; 6405188c4b9SDmitri Gribenko if (C == '/' || C == '!') 6415188c4b9SDmitri Gribenko BufferPtr++; 6425188c4b9SDmitri Gribenko } 6435188c4b9SDmitri Gribenko 6445188c4b9SDmitri Gribenko // Skip less-than symbol that marks trailing comments. 6455188c4b9SDmitri Gribenko // Skip it even if the comment is not a Doxygen one, because //< and /*< 6465188c4b9SDmitri Gribenko // are frequent typos. 6475188c4b9SDmitri Gribenko if (BufferPtr != BufferEnd && *BufferPtr == '<') 6485188c4b9SDmitri Gribenko BufferPtr++; 6495188c4b9SDmitri Gribenko 6505188c4b9SDmitri Gribenko CommentState = LCS_InsideBCPLComment; 651ec92531cSDmitri Gribenko if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine) 6525188c4b9SDmitri Gribenko State = LS_Normal; 6535188c4b9SDmitri Gribenko CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd); 6545188c4b9SDmitri Gribenko goto again; 6555188c4b9SDmitri Gribenko } 6565188c4b9SDmitri Gribenko case '*': { // C comment. 6575188c4b9SDmitri Gribenko BufferPtr++; // Skip star. 6585188c4b9SDmitri Gribenko 6595188c4b9SDmitri Gribenko // Skip Doxygen magic marker. 6605188c4b9SDmitri Gribenko const char C = *BufferPtr; 6615188c4b9SDmitri Gribenko if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!') 6625188c4b9SDmitri Gribenko BufferPtr++; 6635188c4b9SDmitri Gribenko 6645188c4b9SDmitri Gribenko // Skip less-than symbol that marks trailing comments. 6655188c4b9SDmitri Gribenko if (BufferPtr != BufferEnd && *BufferPtr == '<') 6665188c4b9SDmitri Gribenko BufferPtr++; 6675188c4b9SDmitri Gribenko 6685188c4b9SDmitri Gribenko CommentState = LCS_InsideCComment; 6695188c4b9SDmitri Gribenko State = LS_Normal; 6705188c4b9SDmitri Gribenko CommentEnd = findCCommentEnd(BufferPtr, BufferEnd); 6715188c4b9SDmitri Gribenko goto again; 6725188c4b9SDmitri Gribenko } 6735188c4b9SDmitri Gribenko default: 6745188c4b9SDmitri Gribenko llvm_unreachable("second character of comment should be '/' or '*'"); 6755188c4b9SDmitri Gribenko } 6765188c4b9SDmitri Gribenko 6775188c4b9SDmitri Gribenko case LCS_BetweenComments: { 6785188c4b9SDmitri Gribenko // Consecutive comments are extracted only if there is only whitespace 6795188c4b9SDmitri Gribenko // between them. So we can search for the start of the next comment. 6805188c4b9SDmitri Gribenko const char *EndWhitespace = BufferPtr; 6815188c4b9SDmitri Gribenko while(EndWhitespace != BufferEnd && *EndWhitespace != '/') 6825188c4b9SDmitri Gribenko EndWhitespace++; 6835188c4b9SDmitri Gribenko 6845188c4b9SDmitri Gribenko // Turn any whitespace between comments (and there is only whitespace 68517709ae8SDmitri Gribenko // between them -- guaranteed by comment extraction) into a newline. We 68617709ae8SDmitri Gribenko // have two newlines between C comments in total (first one was synthesized 68717709ae8SDmitri Gribenko // after a comment). 6885188c4b9SDmitri Gribenko formTokenWithChars(T, EndWhitespace, tok::newline); 6895188c4b9SDmitri Gribenko 6905188c4b9SDmitri Gribenko CommentState = LCS_BeforeComment; 6915188c4b9SDmitri Gribenko break; 6925188c4b9SDmitri Gribenko } 6935188c4b9SDmitri Gribenko 6945188c4b9SDmitri Gribenko case LCS_InsideBCPLComment: 6955188c4b9SDmitri Gribenko case LCS_InsideCComment: 6965188c4b9SDmitri Gribenko if (BufferPtr != CommentEnd) { 6975188c4b9SDmitri Gribenko lexCommentText(T); 6985188c4b9SDmitri Gribenko break; 6995188c4b9SDmitri Gribenko } else { 7005188c4b9SDmitri Gribenko // Skip C comment closing sequence. 7015188c4b9SDmitri Gribenko if (CommentState == LCS_InsideCComment) { 7025188c4b9SDmitri Gribenko assert(BufferPtr[0] == '*' && BufferPtr[1] == '/'); 7035188c4b9SDmitri Gribenko BufferPtr += 2; 7045188c4b9SDmitri Gribenko assert(BufferPtr <= BufferEnd); 7055188c4b9SDmitri Gribenko 7065188c4b9SDmitri Gribenko // Synthenize newline just after the C comment, regardless if there is 7075188c4b9SDmitri Gribenko // actually a newline. 7085188c4b9SDmitri Gribenko formTokenWithChars(T, BufferPtr, tok::newline); 7095188c4b9SDmitri Gribenko 7105188c4b9SDmitri Gribenko CommentState = LCS_BetweenComments; 7115188c4b9SDmitri Gribenko break; 7125188c4b9SDmitri Gribenko } else { 7135188c4b9SDmitri Gribenko // Don't synthesized a newline after BCPL comment. 7145188c4b9SDmitri Gribenko CommentState = LCS_BetweenComments; 7155188c4b9SDmitri Gribenko goto again; 7165188c4b9SDmitri Gribenko } 7175188c4b9SDmitri Gribenko } 7185188c4b9SDmitri Gribenko } 7195188c4b9SDmitri Gribenko } 7205188c4b9SDmitri Gribenko 7215188c4b9SDmitri Gribenko StringRef Lexer::getSpelling(const Token &Tok, 7225188c4b9SDmitri Gribenko const SourceManager &SourceMgr, 7235188c4b9SDmitri Gribenko bool *Invalid) const { 7245188c4b9SDmitri Gribenko SourceLocation Loc = Tok.getLocation(); 7255188c4b9SDmitri Gribenko std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc); 7265188c4b9SDmitri Gribenko 7275188c4b9SDmitri Gribenko bool InvalidTemp = false; 7285188c4b9SDmitri Gribenko StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp); 7295188c4b9SDmitri Gribenko if (InvalidTemp) { 7305188c4b9SDmitri Gribenko *Invalid = true; 7315188c4b9SDmitri Gribenko return StringRef(); 7325188c4b9SDmitri Gribenko } 7335188c4b9SDmitri Gribenko 7345188c4b9SDmitri Gribenko const char *Begin = File.data() + LocInfo.second; 7355188c4b9SDmitri Gribenko return StringRef(Begin, Tok.getLength()); 7365188c4b9SDmitri Gribenko } 7375188c4b9SDmitri Gribenko 7385188c4b9SDmitri Gribenko void Lexer::addVerbatimBlockCommand(StringRef BeginName, StringRef EndName) { 7395188c4b9SDmitri Gribenko VerbatimBlockCommand VBC; 7405188c4b9SDmitri Gribenko VBC.BeginName = BeginName; 7415188c4b9SDmitri Gribenko VBC.EndName = EndName; 7425188c4b9SDmitri Gribenko VerbatimBlockCommands.push_back(VBC); 7435188c4b9SDmitri Gribenko } 7445188c4b9SDmitri Gribenko 7455188c4b9SDmitri Gribenko void Lexer::addVerbatimLineCommand(StringRef Name) { 7465188c4b9SDmitri Gribenko VerbatimLineCommand VLC; 7475188c4b9SDmitri Gribenko VLC.Name = Name; 7485188c4b9SDmitri Gribenko VerbatimLineCommands.push_back(VLC); 7495188c4b9SDmitri Gribenko } 7505188c4b9SDmitri Gribenko 7515188c4b9SDmitri Gribenko } // end namespace comments 7525188c4b9SDmitri Gribenko } // end namespace clang 7535188c4b9SDmitri Gribenko 754