15188c4b9SDmitri Gribenko #include "clang/AST/CommentLexer.h"
25188c4b9SDmitri Gribenko #include "llvm/ADT/StringSwitch.h"
35188c4b9SDmitri Gribenko #include "llvm/Support/ErrorHandling.h"
45188c4b9SDmitri Gribenko 
55188c4b9SDmitri Gribenko namespace clang {
65188c4b9SDmitri Gribenko namespace comments {
75188c4b9SDmitri Gribenko 
85188c4b9SDmitri Gribenko void Token::dump(const Lexer &L, const SourceManager &SM) const {
95188c4b9SDmitri Gribenko   llvm::errs() << "comments::Token Kind=" << Kind << " ";
105188c4b9SDmitri Gribenko   Loc.dump(SM);
115188c4b9SDmitri Gribenko   llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
125188c4b9SDmitri Gribenko }
135188c4b9SDmitri Gribenko 
145188c4b9SDmitri Gribenko bool Lexer::isVerbatimBlockCommand(StringRef BeginName,
155188c4b9SDmitri Gribenko                                   StringRef &EndName) const {
165188c4b9SDmitri Gribenko   const char *Result = llvm::StringSwitch<const char *>(BeginName)
175188c4b9SDmitri Gribenko     .Case("code", "endcode")
185188c4b9SDmitri Gribenko     .Case("verbatim", "endverbatim")
195188c4b9SDmitri Gribenko     .Case("htmlonly", "endhtmlonly")
205188c4b9SDmitri Gribenko     .Case("latexonly", "endlatexonly")
215188c4b9SDmitri Gribenko     .Case("xmlonly", "endxmlonly")
225188c4b9SDmitri Gribenko     .Case("manonly", "endmanonly")
235188c4b9SDmitri Gribenko     .Case("rtfonly", "endrtfonly")
245188c4b9SDmitri Gribenko 
255188c4b9SDmitri Gribenko     .Case("dot", "enddot")
265188c4b9SDmitri Gribenko     .Case("msc", "endmsc")
275188c4b9SDmitri Gribenko 
285188c4b9SDmitri Gribenko     .Case("f$", "f$") // Inline LaTeX formula
295188c4b9SDmitri Gribenko     .Case("f[", "f]") // Displayed LaTeX formula
305188c4b9SDmitri Gribenko     .Case("f{", "f}") // LaTeX environment
315188c4b9SDmitri Gribenko 
325188c4b9SDmitri Gribenko     .Default(NULL);
335188c4b9SDmitri Gribenko 
345188c4b9SDmitri Gribenko   if (Result) {
355188c4b9SDmitri Gribenko     EndName = Result;
365188c4b9SDmitri Gribenko     return true;
375188c4b9SDmitri Gribenko   }
385188c4b9SDmitri Gribenko 
395188c4b9SDmitri Gribenko   for (VerbatimBlockCommandVector::const_iterator
405188c4b9SDmitri Gribenko            I = VerbatimBlockCommands.begin(),
415188c4b9SDmitri Gribenko            E = VerbatimBlockCommands.end();
425188c4b9SDmitri Gribenko        I != E; ++I)
435188c4b9SDmitri Gribenko     if (I->BeginName == BeginName) {
445188c4b9SDmitri Gribenko       EndName = I->EndName;
455188c4b9SDmitri Gribenko       return true;
465188c4b9SDmitri Gribenko     }
475188c4b9SDmitri Gribenko 
485188c4b9SDmitri Gribenko   return false;
495188c4b9SDmitri Gribenko }
505188c4b9SDmitri Gribenko 
515188c4b9SDmitri Gribenko bool Lexer::isVerbatimLineCommand(StringRef Name) const {
525188c4b9SDmitri Gribenko   bool Result = llvm::StringSwitch<bool>(Name)
535188c4b9SDmitri Gribenko   .Case("fn", true)
545188c4b9SDmitri Gribenko   .Case("var", true)
555188c4b9SDmitri Gribenko   .Case("property", true)
565188c4b9SDmitri Gribenko   .Case("typedef", true)
575188c4b9SDmitri Gribenko 
585188c4b9SDmitri Gribenko   .Case("overload", true)
595188c4b9SDmitri Gribenko 
605188c4b9SDmitri Gribenko   .Case("defgroup", true)
615188c4b9SDmitri Gribenko   .Case("ingroup", true)
625188c4b9SDmitri Gribenko   .Case("addtogroup", true)
635188c4b9SDmitri Gribenko   .Case("weakgroup", true)
645188c4b9SDmitri Gribenko   .Case("name", true)
655188c4b9SDmitri Gribenko 
665188c4b9SDmitri Gribenko   .Case("section", true)
675188c4b9SDmitri Gribenko   .Case("subsection", true)
685188c4b9SDmitri Gribenko   .Case("subsubsection", true)
695188c4b9SDmitri Gribenko   .Case("paragraph", true)
705188c4b9SDmitri Gribenko 
715188c4b9SDmitri Gribenko   .Case("mainpage", true)
725188c4b9SDmitri Gribenko   .Case("subpage", true)
735188c4b9SDmitri Gribenko   .Case("ref", true)
745188c4b9SDmitri Gribenko 
755188c4b9SDmitri Gribenko   .Default(false);
765188c4b9SDmitri Gribenko 
775188c4b9SDmitri Gribenko   if (Result)
785188c4b9SDmitri Gribenko     return true;
795188c4b9SDmitri Gribenko 
805188c4b9SDmitri Gribenko   for (VerbatimLineCommandVector::const_iterator
815188c4b9SDmitri Gribenko            I = VerbatimLineCommands.begin(),
825188c4b9SDmitri Gribenko            E = VerbatimLineCommands.end();
835188c4b9SDmitri Gribenko        I != E; ++I)
845188c4b9SDmitri Gribenko     if (I->Name == Name)
855188c4b9SDmitri Gribenko       return true;
865188c4b9SDmitri Gribenko 
875188c4b9SDmitri Gribenko   return false;
885188c4b9SDmitri Gribenko }
895188c4b9SDmitri Gribenko 
905188c4b9SDmitri Gribenko void Lexer::skipLineStartingDecorations() {
915188c4b9SDmitri Gribenko   // This function should be called only for C comments
925188c4b9SDmitri Gribenko   assert(CommentState == LCS_InsideCComment);
935188c4b9SDmitri Gribenko 
945188c4b9SDmitri Gribenko   if (BufferPtr == CommentEnd)
955188c4b9SDmitri Gribenko     return;
965188c4b9SDmitri Gribenko 
975188c4b9SDmitri Gribenko   switch (*BufferPtr) {
985188c4b9SDmitri Gribenko   case ' ':
995188c4b9SDmitri Gribenko   case '\t':
1005188c4b9SDmitri Gribenko   case '\f':
1015188c4b9SDmitri Gribenko   case '\v': {
1025188c4b9SDmitri Gribenko     const char *NewBufferPtr = BufferPtr;
1035188c4b9SDmitri Gribenko     NewBufferPtr++;
1045188c4b9SDmitri Gribenko     if (NewBufferPtr == CommentEnd)
1055188c4b9SDmitri Gribenko       return;
1065188c4b9SDmitri Gribenko 
1075188c4b9SDmitri Gribenko     char C = *NewBufferPtr;
1085188c4b9SDmitri Gribenko     while (C == ' ' || C == '\t' || C == '\f' || C == '\v') {
1095188c4b9SDmitri Gribenko       NewBufferPtr++;
1105188c4b9SDmitri Gribenko       if (NewBufferPtr == CommentEnd)
1115188c4b9SDmitri Gribenko         return;
1125188c4b9SDmitri Gribenko       C = *NewBufferPtr;
1135188c4b9SDmitri Gribenko     }
1145188c4b9SDmitri Gribenko     if (C == '*')
1155188c4b9SDmitri Gribenko       BufferPtr = NewBufferPtr + 1;
1165188c4b9SDmitri Gribenko     break;
1175188c4b9SDmitri Gribenko   }
1185188c4b9SDmitri Gribenko   case '*':
1195188c4b9SDmitri Gribenko     BufferPtr++;
1205188c4b9SDmitri Gribenko     break;
1215188c4b9SDmitri Gribenko   }
1225188c4b9SDmitri Gribenko }
1235188c4b9SDmitri Gribenko 
1245188c4b9SDmitri Gribenko namespace {
125ec92531cSDmitri Gribenko /// Returns pointer to the first newline character in the string.
1265188c4b9SDmitri Gribenko const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
1275188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
1285188c4b9SDmitri Gribenko     const char C = *BufferPtr;
1295188c4b9SDmitri Gribenko     if (C == '\n' || C == '\r')
1305188c4b9SDmitri Gribenko       return BufferPtr;
1315188c4b9SDmitri Gribenko   }
1325188c4b9SDmitri Gribenko   return BufferEnd;
1335188c4b9SDmitri Gribenko }
1345188c4b9SDmitri Gribenko 
1355188c4b9SDmitri Gribenko const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
1365188c4b9SDmitri Gribenko   if (BufferPtr == BufferEnd)
1375188c4b9SDmitri Gribenko     return BufferPtr;
1385188c4b9SDmitri Gribenko 
1395188c4b9SDmitri Gribenko   if (*BufferPtr == '\n')
1405188c4b9SDmitri Gribenko     BufferPtr++;
1415188c4b9SDmitri Gribenko   else {
1425188c4b9SDmitri Gribenko     assert(*BufferPtr == '\r');
1435188c4b9SDmitri Gribenko     BufferPtr++;
1445188c4b9SDmitri Gribenko     if (BufferPtr != BufferEnd && *BufferPtr == '\n')
1455188c4b9SDmitri Gribenko       BufferPtr++;
1465188c4b9SDmitri Gribenko   }
1475188c4b9SDmitri Gribenko   return BufferPtr;
1485188c4b9SDmitri Gribenko }
1495188c4b9SDmitri Gribenko 
15017709ae8SDmitri Gribenko bool isHTMLIdentifierStartingCharacter(char C) {
15117709ae8SDmitri Gribenko   return (C >= 'a' && C <= 'z') ||
15217709ae8SDmitri Gribenko          (C >= 'A' && C <= 'Z');
15317709ae8SDmitri Gribenko }
15417709ae8SDmitri Gribenko 
1555188c4b9SDmitri Gribenko bool isHTMLIdentifierCharacter(char C) {
1565188c4b9SDmitri Gribenko   return (C >= 'a' && C <= 'z') ||
1575188c4b9SDmitri Gribenko          (C >= 'A' && C <= 'Z') ||
1585188c4b9SDmitri Gribenko          (C >= '0' && C <= '9');
1595188c4b9SDmitri Gribenko }
1605188c4b9SDmitri Gribenko 
1615188c4b9SDmitri Gribenko const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
1625188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
1635188c4b9SDmitri Gribenko     if (!isHTMLIdentifierCharacter(*BufferPtr))
1645188c4b9SDmitri Gribenko       return BufferPtr;
1655188c4b9SDmitri Gribenko   }
1665188c4b9SDmitri Gribenko   return BufferEnd;
1675188c4b9SDmitri Gribenko }
1685188c4b9SDmitri Gribenko 
1695188c4b9SDmitri Gribenko /// Skip HTML string quoted in single or double quotes.  Escaping quotes inside
1705188c4b9SDmitri Gribenko /// string allowed.
1715188c4b9SDmitri Gribenko ///
1725188c4b9SDmitri Gribenko /// Returns pointer to closing quote.
1735188c4b9SDmitri Gribenko const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
1745188c4b9SDmitri Gribenko {
1755188c4b9SDmitri Gribenko   const char Quote = *BufferPtr;
1765188c4b9SDmitri Gribenko   assert(Quote == '\"' || Quote == '\'');
1775188c4b9SDmitri Gribenko 
1785188c4b9SDmitri Gribenko   BufferPtr++;
1795188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
1805188c4b9SDmitri Gribenko     const char C = *BufferPtr;
1815188c4b9SDmitri Gribenko     if (C == Quote && BufferPtr[-1] != '\\')
1825188c4b9SDmitri Gribenko       return BufferPtr;
1835188c4b9SDmitri Gribenko   }
1845188c4b9SDmitri Gribenko   return BufferEnd;
1855188c4b9SDmitri Gribenko }
1865188c4b9SDmitri Gribenko 
1875188c4b9SDmitri Gribenko bool isHorizontalWhitespace(char C) {
1885188c4b9SDmitri Gribenko   return C == ' ' || C == '\t' || C == '\f' || C == '\v';
1895188c4b9SDmitri Gribenko }
1905188c4b9SDmitri Gribenko 
1915188c4b9SDmitri Gribenko bool isWhitespace(char C) {
1925188c4b9SDmitri Gribenko   return C == ' ' || C == '\n' || C == '\r' ||
1935188c4b9SDmitri Gribenko          C == '\t' || C == '\f' || C == '\v';
1945188c4b9SDmitri Gribenko }
1955188c4b9SDmitri Gribenko 
1965188c4b9SDmitri Gribenko const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
1975188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
1985188c4b9SDmitri Gribenko     if (!isWhitespace(*BufferPtr))
1995188c4b9SDmitri Gribenko       return BufferPtr;
2005188c4b9SDmitri Gribenko   }
2015188c4b9SDmitri Gribenko   return BufferEnd;
2025188c4b9SDmitri Gribenko }
2035188c4b9SDmitri Gribenko 
204*e4a3997dSDmitri Gribenko bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
205*e4a3997dSDmitri Gribenko   return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
206*e4a3997dSDmitri Gribenko }
207*e4a3997dSDmitri Gribenko 
2085188c4b9SDmitri Gribenko bool isCommandNameCharacter(char C) {
2095188c4b9SDmitri Gribenko   return (C >= 'a' && C <= 'z') ||
2105188c4b9SDmitri Gribenko          (C >= 'A' && C <= 'Z') ||
2115188c4b9SDmitri Gribenko          (C >= '0' && C <= '9');
2125188c4b9SDmitri Gribenko }
2135188c4b9SDmitri Gribenko 
2145188c4b9SDmitri Gribenko const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
2155188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
2165188c4b9SDmitri Gribenko     if (!isCommandNameCharacter(*BufferPtr))
2175188c4b9SDmitri Gribenko       return BufferPtr;
2185188c4b9SDmitri Gribenko   }
2195188c4b9SDmitri Gribenko   return BufferEnd;
2205188c4b9SDmitri Gribenko }
2215188c4b9SDmitri Gribenko 
2225188c4b9SDmitri Gribenko /// Return the one past end pointer for BCPL comments.
2235188c4b9SDmitri Gribenko /// Handles newlines escaped with backslash or trigraph for backslahs.
2245188c4b9SDmitri Gribenko const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
2255188c4b9SDmitri Gribenko   const char *CurPtr = BufferPtr;
2265188c4b9SDmitri Gribenko   while (CurPtr != BufferEnd) {
2275188c4b9SDmitri Gribenko     char C = *CurPtr;
2285188c4b9SDmitri Gribenko     while (C != '\n' && C != '\r') {
2295188c4b9SDmitri Gribenko       CurPtr++;
2305188c4b9SDmitri Gribenko       if (CurPtr == BufferEnd)
2315188c4b9SDmitri Gribenko         return BufferEnd;
2325188c4b9SDmitri Gribenko       C = *CurPtr;
2335188c4b9SDmitri Gribenko     }
2345188c4b9SDmitri Gribenko     // We found a newline, check if it is escaped.
2355188c4b9SDmitri Gribenko     const char *EscapePtr = CurPtr - 1;
2365188c4b9SDmitri Gribenko     while(isHorizontalWhitespace(*EscapePtr))
2375188c4b9SDmitri Gribenko       EscapePtr--;
2385188c4b9SDmitri Gribenko 
2395188c4b9SDmitri Gribenko     if (*EscapePtr == '\\' ||
2405188c4b9SDmitri Gribenko         (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
2415188c4b9SDmitri Gribenko          EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
2425188c4b9SDmitri Gribenko       // We found an escaped newline.
2435188c4b9SDmitri Gribenko       CurPtr = skipNewline(CurPtr, BufferEnd);
2445188c4b9SDmitri Gribenko     } else
2455188c4b9SDmitri Gribenko       return CurPtr; // Not an escaped newline.
2465188c4b9SDmitri Gribenko   }
2475188c4b9SDmitri Gribenko   return BufferEnd;
2485188c4b9SDmitri Gribenko }
2495188c4b9SDmitri Gribenko 
2505188c4b9SDmitri Gribenko /// Return the one past end pointer for C comments.
2515188c4b9SDmitri Gribenko /// Very dumb, does not handle escaped newlines or trigraphs.
2525188c4b9SDmitri Gribenko const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
2535188c4b9SDmitri Gribenko   for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
2545188c4b9SDmitri Gribenko     if (*BufferPtr == '*') {
2555188c4b9SDmitri Gribenko       assert(BufferPtr + 1 != BufferEnd);
2565188c4b9SDmitri Gribenko       if (*(BufferPtr + 1) == '/')
2575188c4b9SDmitri Gribenko         return BufferPtr;
2585188c4b9SDmitri Gribenko     }
2595188c4b9SDmitri Gribenko   }
2605188c4b9SDmitri Gribenko   llvm_unreachable("buffer end hit before '*/' was seen");
2615188c4b9SDmitri Gribenko }
2625188c4b9SDmitri Gribenko } // unnamed namespace
2635188c4b9SDmitri Gribenko 
2645188c4b9SDmitri Gribenko void Lexer::lexCommentText(Token &T) {
2655188c4b9SDmitri Gribenko   assert(CommentState == LCS_InsideBCPLComment ||
2665188c4b9SDmitri Gribenko          CommentState == LCS_InsideCComment);
2675188c4b9SDmitri Gribenko 
2685188c4b9SDmitri Gribenko   switch (State) {
2695188c4b9SDmitri Gribenko   case LS_Normal:
2705188c4b9SDmitri Gribenko     break;
2715188c4b9SDmitri Gribenko   case LS_VerbatimBlockFirstLine:
2725188c4b9SDmitri Gribenko     lexVerbatimBlockFirstLine(T);
2735188c4b9SDmitri Gribenko     return;
2745188c4b9SDmitri Gribenko   case LS_VerbatimBlockBody:
2755188c4b9SDmitri Gribenko     lexVerbatimBlockBody(T);
2765188c4b9SDmitri Gribenko     return;
2771669f702SDmitri Gribenko   case LS_VerbatimLineText:
2781669f702SDmitri Gribenko     lexVerbatimLineText(T);
2791669f702SDmitri Gribenko     return;
280e00ffc7bSDmitri Gribenko   case LS_HTMLStartTag:
281e00ffc7bSDmitri Gribenko     lexHTMLStartTag(T);
2825188c4b9SDmitri Gribenko     return;
283e00ffc7bSDmitri Gribenko   case LS_HTMLEndTag:
284e00ffc7bSDmitri Gribenko     lexHTMLEndTag(T);
285ec92531cSDmitri Gribenko     return;
2865188c4b9SDmitri Gribenko   }
2875188c4b9SDmitri Gribenko 
2885188c4b9SDmitri Gribenko   assert(State == LS_Normal);
2895188c4b9SDmitri Gribenko 
2905188c4b9SDmitri Gribenko   const char *TokenPtr = BufferPtr;
2915188c4b9SDmitri Gribenko   assert(TokenPtr < CommentEnd);
2925188c4b9SDmitri Gribenko   while (TokenPtr != CommentEnd) {
2935188c4b9SDmitri Gribenko     switch(*TokenPtr) {
2945188c4b9SDmitri Gribenko       case '\\':
2955188c4b9SDmitri Gribenko       case '@': {
2965188c4b9SDmitri Gribenko         TokenPtr++;
2975188c4b9SDmitri Gribenko         if (TokenPtr == CommentEnd) {
29860ddd8a1SDmitri Gribenko           StringRef Text(BufferPtr, TokenPtr - BufferPtr);
2995188c4b9SDmitri Gribenko           formTokenWithChars(T, TokenPtr, tok::text);
30060ddd8a1SDmitri Gribenko           T.setText(Text);
3015188c4b9SDmitri Gribenko           return;
3025188c4b9SDmitri Gribenko         }
3035188c4b9SDmitri Gribenko         char C = *TokenPtr;
3045188c4b9SDmitri Gribenko         switch (C) {
3055188c4b9SDmitri Gribenko         default:
3065188c4b9SDmitri Gribenko           break;
3075188c4b9SDmitri Gribenko 
3085188c4b9SDmitri Gribenko         case '\\': case '@': case '&': case '$':
3095188c4b9SDmitri Gribenko         case '#':  case '<': case '>': case '%':
3105188c4b9SDmitri Gribenko         case '\"': case '.': case ':':
3115188c4b9SDmitri Gribenko           // This is one of \\ \@ \& \$ etc escape sequences.
3125188c4b9SDmitri Gribenko           TokenPtr++;
3135188c4b9SDmitri Gribenko           if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
3145188c4b9SDmitri Gribenko             // This is the \:: escape sequence.
3155188c4b9SDmitri Gribenko             TokenPtr++;
3165188c4b9SDmitri Gribenko           }
31760ddd8a1SDmitri Gribenko           StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
3185188c4b9SDmitri Gribenko           formTokenWithChars(T, TokenPtr, tok::text);
31960ddd8a1SDmitri Gribenko           T.setText(UnescapedText);
3205188c4b9SDmitri Gribenko           return;
3215188c4b9SDmitri Gribenko         }
3225188c4b9SDmitri Gribenko 
3235188c4b9SDmitri Gribenko         // Don't make zero-length commands.
3245188c4b9SDmitri Gribenko         if (!isCommandNameCharacter(*TokenPtr)) {
32560ddd8a1SDmitri Gribenko           StringRef Text(BufferPtr, TokenPtr - BufferPtr);
3265188c4b9SDmitri Gribenko           formTokenWithChars(T, TokenPtr, tok::text);
32760ddd8a1SDmitri Gribenko           T.setText(Text);
3285188c4b9SDmitri Gribenko           return;
3295188c4b9SDmitri Gribenko         }
3305188c4b9SDmitri Gribenko 
3315188c4b9SDmitri Gribenko         TokenPtr = skipCommandName(TokenPtr, CommentEnd);
3325188c4b9SDmitri Gribenko         unsigned Length = TokenPtr - (BufferPtr + 1);
3335188c4b9SDmitri Gribenko 
3345188c4b9SDmitri Gribenko         // Hardcoded support for lexing LaTeX formula commands
3355188c4b9SDmitri Gribenko         // \f$ \f[ \f] \f{ \f} as a single command.
3365188c4b9SDmitri Gribenko         if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
3375188c4b9SDmitri Gribenko           C = *TokenPtr;
3385188c4b9SDmitri Gribenko           if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
3395188c4b9SDmitri Gribenko             TokenPtr++;
3405188c4b9SDmitri Gribenko             Length++;
3415188c4b9SDmitri Gribenko           }
3425188c4b9SDmitri Gribenko         }
3435188c4b9SDmitri Gribenko 
3445188c4b9SDmitri Gribenko         const StringRef CommandName(BufferPtr + 1, Length);
3455188c4b9SDmitri Gribenko         StringRef EndName;
3465188c4b9SDmitri Gribenko 
3475188c4b9SDmitri Gribenko         if (isVerbatimBlockCommand(CommandName, EndName)) {
3485188c4b9SDmitri Gribenko           setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, EndName);
3495188c4b9SDmitri Gribenko           return;
3505188c4b9SDmitri Gribenko         }
3515188c4b9SDmitri Gribenko         if (isVerbatimLineCommand(CommandName)) {
3521669f702SDmitri Gribenko           setupAndLexVerbatimLine(T, TokenPtr);
3535188c4b9SDmitri Gribenko           return;
3545188c4b9SDmitri Gribenko         }
3555188c4b9SDmitri Gribenko         formTokenWithChars(T, TokenPtr, tok::command);
3565188c4b9SDmitri Gribenko         T.setCommandName(CommandName);
3575188c4b9SDmitri Gribenko         return;
3585188c4b9SDmitri Gribenko       }
3595188c4b9SDmitri Gribenko 
3605188c4b9SDmitri Gribenko       case '<': {
3615188c4b9SDmitri Gribenko         TokenPtr++;
3625188c4b9SDmitri Gribenko         if (TokenPtr == CommentEnd) {
36360ddd8a1SDmitri Gribenko           StringRef Text(BufferPtr, TokenPtr - BufferPtr);
3645188c4b9SDmitri Gribenko           formTokenWithChars(T, TokenPtr, tok::text);
36560ddd8a1SDmitri Gribenko           T.setText(Text);
3665188c4b9SDmitri Gribenko           return;
3675188c4b9SDmitri Gribenko         }
3685188c4b9SDmitri Gribenko         const char C = *TokenPtr;
36917709ae8SDmitri Gribenko         if (isHTMLIdentifierStartingCharacter(C))
370e00ffc7bSDmitri Gribenko           setupAndLexHTMLStartTag(T);
3715188c4b9SDmitri Gribenko         else if (C == '/')
372e00ffc7bSDmitri Gribenko           setupAndLexHTMLEndTag(T);
373632d58afSDmitri Gribenko         else {
374632d58afSDmitri Gribenko           StringRef Text(BufferPtr, TokenPtr - BufferPtr);
375632d58afSDmitri Gribenko           formTokenWithChars(T, TokenPtr, tok::text);
376632d58afSDmitri Gribenko           T.setText(Text);
377632d58afSDmitri Gribenko         }
3785188c4b9SDmitri Gribenko         return;
3795188c4b9SDmitri Gribenko       }
3805188c4b9SDmitri Gribenko 
3815188c4b9SDmitri Gribenko       case '\n':
3825188c4b9SDmitri Gribenko       case '\r':
3835188c4b9SDmitri Gribenko         TokenPtr = skipNewline(TokenPtr, CommentEnd);
3845188c4b9SDmitri Gribenko         formTokenWithChars(T, TokenPtr, tok::newline);
3855188c4b9SDmitri Gribenko 
3865188c4b9SDmitri Gribenko         if (CommentState == LCS_InsideCComment)
3875188c4b9SDmitri Gribenko           skipLineStartingDecorations();
3885188c4b9SDmitri Gribenko         return;
3895188c4b9SDmitri Gribenko 
3905188c4b9SDmitri Gribenko       default: {
3915188c4b9SDmitri Gribenko         while (true) {
3925188c4b9SDmitri Gribenko           TokenPtr++;
3935188c4b9SDmitri Gribenko           if (TokenPtr == CommentEnd)
3945188c4b9SDmitri Gribenko             break;
39517709ae8SDmitri Gribenko           const char C = *TokenPtr;
3965188c4b9SDmitri Gribenko           if(C == '\n' || C == '\r' ||
3975188c4b9SDmitri Gribenko              C == '\\' || C == '@' || C == '<')
3985188c4b9SDmitri Gribenko             break;
3995188c4b9SDmitri Gribenko         }
40060ddd8a1SDmitri Gribenko         StringRef Text(BufferPtr, TokenPtr - BufferPtr);
4015188c4b9SDmitri Gribenko         formTokenWithChars(T, TokenPtr, tok::text);
40260ddd8a1SDmitri Gribenko         T.setText(Text);
4035188c4b9SDmitri Gribenko         return;
4045188c4b9SDmitri Gribenko       }
4055188c4b9SDmitri Gribenko     }
4065188c4b9SDmitri Gribenko   }
4075188c4b9SDmitri Gribenko }
4085188c4b9SDmitri Gribenko 
4095188c4b9SDmitri Gribenko void Lexer::setupAndLexVerbatimBlock(Token &T,
4105188c4b9SDmitri Gribenko                                      const char *TextBegin,
4115188c4b9SDmitri Gribenko                                      char Marker, StringRef EndName) {
4125188c4b9SDmitri Gribenko   VerbatimBlockEndCommandName.clear();
4135188c4b9SDmitri Gribenko   VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
4145188c4b9SDmitri Gribenko   VerbatimBlockEndCommandName.append(EndName);
4155188c4b9SDmitri Gribenko 
41660ddd8a1SDmitri Gribenko   StringRef Name(BufferPtr + 1, TextBegin - (BufferPtr + 1));
4175188c4b9SDmitri Gribenko   formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
41860ddd8a1SDmitri Gribenko   T.setVerbatimBlockName(Name);
4195188c4b9SDmitri Gribenko 
420ec92531cSDmitri Gribenko   // If there is a newline following the verbatim opening command, skip the
421ec92531cSDmitri Gribenko   // newline so that we don't create an tok::verbatim_block_line with empty
422ec92531cSDmitri Gribenko   // text content.
423ec92531cSDmitri Gribenko   if (BufferPtr != CommentEnd) {
424ec92531cSDmitri Gribenko     const char C = *BufferPtr;
425ec92531cSDmitri Gribenko     if (C == '\n' || C == '\r') {
426ec92531cSDmitri Gribenko       BufferPtr = skipNewline(BufferPtr, CommentEnd);
427ec92531cSDmitri Gribenko       State = LS_VerbatimBlockBody;
428ec92531cSDmitri Gribenko       return;
429ec92531cSDmitri Gribenko     }
430ec92531cSDmitri Gribenko   }
431ec92531cSDmitri Gribenko 
4325188c4b9SDmitri Gribenko   State = LS_VerbatimBlockFirstLine;
4335188c4b9SDmitri Gribenko }
4345188c4b9SDmitri Gribenko 
4355188c4b9SDmitri Gribenko void Lexer::lexVerbatimBlockFirstLine(Token &T) {
436*e4a3997dSDmitri Gribenko again:
4375188c4b9SDmitri Gribenko   assert(BufferPtr < CommentEnd);
4385188c4b9SDmitri Gribenko 
4395188c4b9SDmitri Gribenko   // FIXME: It would be better to scan the text once, finding either the block
4405188c4b9SDmitri Gribenko   // end command or newline.
4415188c4b9SDmitri Gribenko   //
4425188c4b9SDmitri Gribenko   // Extract current line.
4435188c4b9SDmitri Gribenko   const char *Newline = findNewline(BufferPtr, CommentEnd);
4445188c4b9SDmitri Gribenko   StringRef Line(BufferPtr, Newline - BufferPtr);
4455188c4b9SDmitri Gribenko 
4465188c4b9SDmitri Gribenko   // Look for end command in current line.
4475188c4b9SDmitri Gribenko   size_t Pos = Line.find(VerbatimBlockEndCommandName);
448ec92531cSDmitri Gribenko   const char *TextEnd;
4495188c4b9SDmitri Gribenko   const char *NextLine;
4505188c4b9SDmitri Gribenko   if (Pos == StringRef::npos) {
4515188c4b9SDmitri Gribenko     // Current line is completely verbatim.
452ec92531cSDmitri Gribenko     TextEnd = Newline;
4535188c4b9SDmitri Gribenko     NextLine = skipNewline(Newline, CommentEnd);
4545188c4b9SDmitri Gribenko   } else if (Pos == 0) {
4555188c4b9SDmitri Gribenko     // Current line contains just an end command.
4565188c4b9SDmitri Gribenko     const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
45760ddd8a1SDmitri Gribenko     StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
4585188c4b9SDmitri Gribenko     formTokenWithChars(T, End, tok::verbatim_block_end);
45960ddd8a1SDmitri Gribenko     T.setVerbatimBlockName(Name);
4605188c4b9SDmitri Gribenko     State = LS_Normal;
4615188c4b9SDmitri Gribenko     return;
4625188c4b9SDmitri Gribenko   } else {
4635188c4b9SDmitri Gribenko     // There is some text, followed by end command.  Extract text first.
464ec92531cSDmitri Gribenko     TextEnd = BufferPtr + Pos;
465ec92531cSDmitri Gribenko     NextLine = TextEnd;
466*e4a3997dSDmitri Gribenko     // If there is only whitespace before end command, skip whitespace.
467*e4a3997dSDmitri Gribenko     if (isWhitespace(BufferPtr, TextEnd)) {
468*e4a3997dSDmitri Gribenko       BufferPtr = TextEnd;
469*e4a3997dSDmitri Gribenko       goto again;
470*e4a3997dSDmitri Gribenko     }
4715188c4b9SDmitri Gribenko   }
4725188c4b9SDmitri Gribenko 
473ec92531cSDmitri Gribenko   StringRef Text(BufferPtr, TextEnd - BufferPtr);
4745188c4b9SDmitri Gribenko   formTokenWithChars(T, NextLine, tok::verbatim_block_line);
47560ddd8a1SDmitri Gribenko   T.setVerbatimBlockText(Text);
4765188c4b9SDmitri Gribenko 
4775188c4b9SDmitri Gribenko   State = LS_VerbatimBlockBody;
4785188c4b9SDmitri Gribenko }
4795188c4b9SDmitri Gribenko 
4805188c4b9SDmitri Gribenko void Lexer::lexVerbatimBlockBody(Token &T) {
4815188c4b9SDmitri Gribenko   assert(State == LS_VerbatimBlockBody);
4825188c4b9SDmitri Gribenko 
4835188c4b9SDmitri Gribenko   if (CommentState == LCS_InsideCComment)
4845188c4b9SDmitri Gribenko     skipLineStartingDecorations();
4855188c4b9SDmitri Gribenko 
4865188c4b9SDmitri Gribenko   lexVerbatimBlockFirstLine(T);
4875188c4b9SDmitri Gribenko }
4885188c4b9SDmitri Gribenko 
4891669f702SDmitri Gribenko void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) {
4901669f702SDmitri Gribenko   const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1);
4911669f702SDmitri Gribenko   formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
4921669f702SDmitri Gribenko   T.setVerbatimLineName(Name);
4931669f702SDmitri Gribenko 
4941669f702SDmitri Gribenko   State = LS_VerbatimLineText;
4951669f702SDmitri Gribenko }
4961669f702SDmitri Gribenko 
4971669f702SDmitri Gribenko void Lexer::lexVerbatimLineText(Token &T) {
4981669f702SDmitri Gribenko   assert(State == LS_VerbatimLineText);
4991669f702SDmitri Gribenko 
5005188c4b9SDmitri Gribenko   // Extract current line.
5015188c4b9SDmitri Gribenko   const char *Newline = findNewline(BufferPtr, CommentEnd);
5021669f702SDmitri Gribenko   const StringRef Text(BufferPtr, Newline - BufferPtr);
5031669f702SDmitri Gribenko   formTokenWithChars(T, Newline, tok::verbatim_line_text);
5045188c4b9SDmitri Gribenko   T.setVerbatimLineText(Text);
5051669f702SDmitri Gribenko 
5061669f702SDmitri Gribenko   State = LS_Normal;
5075188c4b9SDmitri Gribenko }
5085188c4b9SDmitri Gribenko 
509e00ffc7bSDmitri Gribenko void Lexer::setupAndLexHTMLStartTag(Token &T) {
51017709ae8SDmitri Gribenko   assert(BufferPtr[0] == '<' &&
51117709ae8SDmitri Gribenko          isHTMLIdentifierStartingCharacter(BufferPtr[1]));
5125188c4b9SDmitri Gribenko   const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
5135188c4b9SDmitri Gribenko 
51460ddd8a1SDmitri Gribenko   StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
515e00ffc7bSDmitri Gribenko   formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
516e00ffc7bSDmitri Gribenko   T.setHTMLTagStartName(Name);
5175188c4b9SDmitri Gribenko 
5185188c4b9SDmitri Gribenko   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
5195188c4b9SDmitri Gribenko 
52017709ae8SDmitri Gribenko   const char C = *BufferPtr;
52117709ae8SDmitri Gribenko   if (BufferPtr != CommentEnd &&
522f26054f0SDmitri Gribenko       (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
523e00ffc7bSDmitri Gribenko     State = LS_HTMLStartTag;
5245188c4b9SDmitri Gribenko }
5255188c4b9SDmitri Gribenko 
526e00ffc7bSDmitri Gribenko void Lexer::lexHTMLStartTag(Token &T) {
527e00ffc7bSDmitri Gribenko   assert(State == LS_HTMLStartTag);
5285188c4b9SDmitri Gribenko 
5295188c4b9SDmitri Gribenko   const char *TokenPtr = BufferPtr;
5305188c4b9SDmitri Gribenko   char C = *TokenPtr;
5315188c4b9SDmitri Gribenko   if (isHTMLIdentifierCharacter(C)) {
5325188c4b9SDmitri Gribenko     TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
53360ddd8a1SDmitri Gribenko     StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
5345188c4b9SDmitri Gribenko     formTokenWithChars(T, TokenPtr, tok::html_ident);
53560ddd8a1SDmitri Gribenko     T.setHTMLIdent(Ident);
5365188c4b9SDmitri Gribenko   } else {
5375188c4b9SDmitri Gribenko     switch (C) {
5385188c4b9SDmitri Gribenko     case '=':
5395188c4b9SDmitri Gribenko       TokenPtr++;
5405188c4b9SDmitri Gribenko       formTokenWithChars(T, TokenPtr, tok::html_equals);
5415188c4b9SDmitri Gribenko       break;
5425188c4b9SDmitri Gribenko     case '\"':
5435188c4b9SDmitri Gribenko     case '\'': {
5445188c4b9SDmitri Gribenko       const char *OpenQuote = TokenPtr;
5455188c4b9SDmitri Gribenko       TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
5465188c4b9SDmitri Gribenko       const char *ClosingQuote = TokenPtr;
5475188c4b9SDmitri Gribenko       if (TokenPtr != CommentEnd) // Skip closing quote.
5485188c4b9SDmitri Gribenko         TokenPtr++;
5495188c4b9SDmitri Gribenko       formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
5505188c4b9SDmitri Gribenko       T.setHTMLQuotedString(StringRef(OpenQuote + 1,
5515188c4b9SDmitri Gribenko                                       ClosingQuote - (OpenQuote + 1)));
5525188c4b9SDmitri Gribenko       break;
5535188c4b9SDmitri Gribenko     }
5545188c4b9SDmitri Gribenko     case '>':
5555188c4b9SDmitri Gribenko       TokenPtr++;
5565188c4b9SDmitri Gribenko       formTokenWithChars(T, TokenPtr, tok::html_greater);
55717709ae8SDmitri Gribenko       State = LS_Normal;
55817709ae8SDmitri Gribenko       return;
559f26054f0SDmitri Gribenko     case '/':
560f26054f0SDmitri Gribenko       TokenPtr++;
561f26054f0SDmitri Gribenko       if (TokenPtr != CommentEnd && *TokenPtr == '>') {
562f26054f0SDmitri Gribenko         TokenPtr++;
563f26054f0SDmitri Gribenko         formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
564f26054f0SDmitri Gribenko       } else {
565f26054f0SDmitri Gribenko         StringRef Text(BufferPtr, TokenPtr - BufferPtr);
566f26054f0SDmitri Gribenko         formTokenWithChars(T, TokenPtr, tok::text);
567f26054f0SDmitri Gribenko         T.setText(Text);
568f26054f0SDmitri Gribenko       }
569f26054f0SDmitri Gribenko       State = LS_Normal;
570f26054f0SDmitri Gribenko       return;
5715188c4b9SDmitri Gribenko     }
5725188c4b9SDmitri Gribenko   }
5735188c4b9SDmitri Gribenko 
5745188c4b9SDmitri Gribenko   // Now look ahead and return to normal state if we don't see any HTML tokens
5755188c4b9SDmitri Gribenko   // ahead.
5765188c4b9SDmitri Gribenko   BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
5775188c4b9SDmitri Gribenko   if (BufferPtr == CommentEnd) {
5785188c4b9SDmitri Gribenko     State = LS_Normal;
5795188c4b9SDmitri Gribenko     return;
5805188c4b9SDmitri Gribenko   }
5815188c4b9SDmitri Gribenko 
5825188c4b9SDmitri Gribenko   C = *BufferPtr;
58317709ae8SDmitri Gribenko   if (!isHTMLIdentifierStartingCharacter(C) &&
5845188c4b9SDmitri Gribenko       C != '=' && C != '\"' && C != '\'' && C != '>') {
5855188c4b9SDmitri Gribenko     State = LS_Normal;
5865188c4b9SDmitri Gribenko     return;
5875188c4b9SDmitri Gribenko   }
5885188c4b9SDmitri Gribenko }
5895188c4b9SDmitri Gribenko 
590e00ffc7bSDmitri Gribenko void Lexer::setupAndLexHTMLEndTag(Token &T) {
5915188c4b9SDmitri Gribenko   assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
5925188c4b9SDmitri Gribenko 
5935188c4b9SDmitri Gribenko   const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
5945188c4b9SDmitri Gribenko   const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
5955188c4b9SDmitri Gribenko 
5965188c4b9SDmitri Gribenko   const char *End = skipWhitespace(TagNameEnd, CommentEnd);
5975188c4b9SDmitri Gribenko 
598e00ffc7bSDmitri Gribenko   formTokenWithChars(T, End, tok::html_end_tag);
599e00ffc7bSDmitri Gribenko   T.setHTMLTagEndName(StringRef(TagNameBegin, TagNameEnd - TagNameBegin));
600ec92531cSDmitri Gribenko 
601ec92531cSDmitri Gribenko   if (BufferPtr != CommentEnd && *BufferPtr == '>')
602e00ffc7bSDmitri Gribenko     State = LS_HTMLEndTag;
603ec92531cSDmitri Gribenko }
604ec92531cSDmitri Gribenko 
605e00ffc7bSDmitri Gribenko void Lexer::lexHTMLEndTag(Token &T) {
606ec92531cSDmitri Gribenko   assert(BufferPtr != CommentEnd && *BufferPtr == '>');
607ec92531cSDmitri Gribenko 
608ec92531cSDmitri Gribenko   formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
609ec92531cSDmitri Gribenko   State = LS_Normal;
6105188c4b9SDmitri Gribenko }
6115188c4b9SDmitri Gribenko 
6125188c4b9SDmitri Gribenko Lexer::Lexer(SourceLocation FileLoc, const CommentOptions &CommOpts,
6135188c4b9SDmitri Gribenko              const char *BufferStart, const char *BufferEnd):
6145188c4b9SDmitri Gribenko     BufferStart(BufferStart), BufferEnd(BufferEnd),
6155188c4b9SDmitri Gribenko     FileLoc(FileLoc), CommOpts(CommOpts), BufferPtr(BufferStart),
6165188c4b9SDmitri Gribenko     CommentState(LCS_BeforeComment), State(LS_Normal) {
6175188c4b9SDmitri Gribenko }
6185188c4b9SDmitri Gribenko 
6195188c4b9SDmitri Gribenko void Lexer::lex(Token &T) {
6205188c4b9SDmitri Gribenko again:
6215188c4b9SDmitri Gribenko   switch (CommentState) {
6225188c4b9SDmitri Gribenko   case LCS_BeforeComment:
6235188c4b9SDmitri Gribenko     if (BufferPtr == BufferEnd) {
6245188c4b9SDmitri Gribenko       formTokenWithChars(T, BufferPtr, tok::eof);
6255188c4b9SDmitri Gribenko       return;
6265188c4b9SDmitri Gribenko     }
6275188c4b9SDmitri Gribenko 
6285188c4b9SDmitri Gribenko     assert(*BufferPtr == '/');
6295188c4b9SDmitri Gribenko     BufferPtr++; // Skip first slash.
6305188c4b9SDmitri Gribenko     switch(*BufferPtr) {
6315188c4b9SDmitri Gribenko     case '/': { // BCPL comment.
6325188c4b9SDmitri Gribenko       BufferPtr++; // Skip second slash.
6335188c4b9SDmitri Gribenko 
6345188c4b9SDmitri Gribenko       if (BufferPtr != BufferEnd) {
6355188c4b9SDmitri Gribenko         // Skip Doxygen magic marker, if it is present.
6365188c4b9SDmitri Gribenko         // It might be missing because of a typo //< or /*<, or because we
6375188c4b9SDmitri Gribenko         // merged this non-Doxygen comment into a bunch of Doxygen comments
6385188c4b9SDmitri Gribenko         // around it: /** ... */ /* ... */ /** ... */
6395188c4b9SDmitri Gribenko         const char C = *BufferPtr;
6405188c4b9SDmitri Gribenko         if (C == '/' || C == '!')
6415188c4b9SDmitri Gribenko           BufferPtr++;
6425188c4b9SDmitri Gribenko       }
6435188c4b9SDmitri Gribenko 
6445188c4b9SDmitri Gribenko       // Skip less-than symbol that marks trailing comments.
6455188c4b9SDmitri Gribenko       // Skip it even if the comment is not a Doxygen one, because //< and /*<
6465188c4b9SDmitri Gribenko       // are frequent typos.
6475188c4b9SDmitri Gribenko       if (BufferPtr != BufferEnd && *BufferPtr == '<')
6485188c4b9SDmitri Gribenko         BufferPtr++;
6495188c4b9SDmitri Gribenko 
6505188c4b9SDmitri Gribenko       CommentState = LCS_InsideBCPLComment;
651ec92531cSDmitri Gribenko       if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
6525188c4b9SDmitri Gribenko         State = LS_Normal;
6535188c4b9SDmitri Gribenko       CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
6545188c4b9SDmitri Gribenko       goto again;
6555188c4b9SDmitri Gribenko     }
6565188c4b9SDmitri Gribenko     case '*': { // C comment.
6575188c4b9SDmitri Gribenko       BufferPtr++; // Skip star.
6585188c4b9SDmitri Gribenko 
6595188c4b9SDmitri Gribenko       // Skip Doxygen magic marker.
6605188c4b9SDmitri Gribenko       const char C = *BufferPtr;
6615188c4b9SDmitri Gribenko       if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
6625188c4b9SDmitri Gribenko         BufferPtr++;
6635188c4b9SDmitri Gribenko 
6645188c4b9SDmitri Gribenko       // Skip less-than symbol that marks trailing comments.
6655188c4b9SDmitri Gribenko       if (BufferPtr != BufferEnd && *BufferPtr == '<')
6665188c4b9SDmitri Gribenko         BufferPtr++;
6675188c4b9SDmitri Gribenko 
6685188c4b9SDmitri Gribenko       CommentState = LCS_InsideCComment;
6695188c4b9SDmitri Gribenko       State = LS_Normal;
6705188c4b9SDmitri Gribenko       CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
6715188c4b9SDmitri Gribenko       goto again;
6725188c4b9SDmitri Gribenko     }
6735188c4b9SDmitri Gribenko     default:
6745188c4b9SDmitri Gribenko       llvm_unreachable("second character of comment should be '/' or '*'");
6755188c4b9SDmitri Gribenko     }
6765188c4b9SDmitri Gribenko 
6775188c4b9SDmitri Gribenko   case LCS_BetweenComments: {
6785188c4b9SDmitri Gribenko     // Consecutive comments are extracted only if there is only whitespace
6795188c4b9SDmitri Gribenko     // between them.  So we can search for the start of the next comment.
6805188c4b9SDmitri Gribenko     const char *EndWhitespace = BufferPtr;
6815188c4b9SDmitri Gribenko     while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
6825188c4b9SDmitri Gribenko       EndWhitespace++;
6835188c4b9SDmitri Gribenko 
6845188c4b9SDmitri Gribenko     // Turn any whitespace between comments (and there is only whitespace
68517709ae8SDmitri Gribenko     // between them -- guaranteed by comment extraction) into a newline.  We
68617709ae8SDmitri Gribenko     // have two newlines between C comments in total (first one was synthesized
68717709ae8SDmitri Gribenko     // after a comment).
6885188c4b9SDmitri Gribenko     formTokenWithChars(T, EndWhitespace, tok::newline);
6895188c4b9SDmitri Gribenko 
6905188c4b9SDmitri Gribenko     CommentState = LCS_BeforeComment;
6915188c4b9SDmitri Gribenko     break;
6925188c4b9SDmitri Gribenko   }
6935188c4b9SDmitri Gribenko 
6945188c4b9SDmitri Gribenko   case LCS_InsideBCPLComment:
6955188c4b9SDmitri Gribenko   case LCS_InsideCComment:
6965188c4b9SDmitri Gribenko     if (BufferPtr != CommentEnd) {
6975188c4b9SDmitri Gribenko       lexCommentText(T);
6985188c4b9SDmitri Gribenko       break;
6995188c4b9SDmitri Gribenko     } else {
7005188c4b9SDmitri Gribenko       // Skip C comment closing sequence.
7015188c4b9SDmitri Gribenko       if (CommentState == LCS_InsideCComment) {
7025188c4b9SDmitri Gribenko         assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
7035188c4b9SDmitri Gribenko         BufferPtr += 2;
7045188c4b9SDmitri Gribenko         assert(BufferPtr <= BufferEnd);
7055188c4b9SDmitri Gribenko 
7065188c4b9SDmitri Gribenko         // Synthenize newline just after the C comment, regardless if there is
7075188c4b9SDmitri Gribenko         // actually a newline.
7085188c4b9SDmitri Gribenko         formTokenWithChars(T, BufferPtr, tok::newline);
7095188c4b9SDmitri Gribenko 
7105188c4b9SDmitri Gribenko         CommentState = LCS_BetweenComments;
7115188c4b9SDmitri Gribenko         break;
7125188c4b9SDmitri Gribenko       } else {
7135188c4b9SDmitri Gribenko         // Don't synthesized a newline after BCPL comment.
7145188c4b9SDmitri Gribenko         CommentState = LCS_BetweenComments;
7155188c4b9SDmitri Gribenko         goto again;
7165188c4b9SDmitri Gribenko       }
7175188c4b9SDmitri Gribenko     }
7185188c4b9SDmitri Gribenko   }
7195188c4b9SDmitri Gribenko }
7205188c4b9SDmitri Gribenko 
7215188c4b9SDmitri Gribenko StringRef Lexer::getSpelling(const Token &Tok,
7225188c4b9SDmitri Gribenko                              const SourceManager &SourceMgr,
7235188c4b9SDmitri Gribenko                              bool *Invalid) const {
7245188c4b9SDmitri Gribenko   SourceLocation Loc = Tok.getLocation();
7255188c4b9SDmitri Gribenko   std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
7265188c4b9SDmitri Gribenko 
7275188c4b9SDmitri Gribenko   bool InvalidTemp = false;
7285188c4b9SDmitri Gribenko   StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
7295188c4b9SDmitri Gribenko   if (InvalidTemp) {
7305188c4b9SDmitri Gribenko     *Invalid = true;
7315188c4b9SDmitri Gribenko     return StringRef();
7325188c4b9SDmitri Gribenko   }
7335188c4b9SDmitri Gribenko 
7345188c4b9SDmitri Gribenko   const char *Begin = File.data() + LocInfo.second;
7355188c4b9SDmitri Gribenko   return StringRef(Begin, Tok.getLength());
7365188c4b9SDmitri Gribenko }
7375188c4b9SDmitri Gribenko 
7385188c4b9SDmitri Gribenko void Lexer::addVerbatimBlockCommand(StringRef BeginName, StringRef EndName) {
7395188c4b9SDmitri Gribenko   VerbatimBlockCommand VBC;
7405188c4b9SDmitri Gribenko   VBC.BeginName = BeginName;
7415188c4b9SDmitri Gribenko   VBC.EndName = EndName;
7425188c4b9SDmitri Gribenko   VerbatimBlockCommands.push_back(VBC);
7435188c4b9SDmitri Gribenko }
7445188c4b9SDmitri Gribenko 
7455188c4b9SDmitri Gribenko void Lexer::addVerbatimLineCommand(StringRef Name) {
7465188c4b9SDmitri Gribenko   VerbatimLineCommand VLC;
7475188c4b9SDmitri Gribenko   VLC.Name = Name;
7485188c4b9SDmitri Gribenko   VerbatimLineCommands.push_back(VLC);
7495188c4b9SDmitri Gribenko }
7505188c4b9SDmitri Gribenko 
7515188c4b9SDmitri Gribenko } // end namespace comments
7525188c4b9SDmitri Gribenko } // end namespace clang
7535188c4b9SDmitri Gribenko 
754