1 //===--- FormatTokenLexer.cpp - Lex FormatTokens -------------*- C++ ----*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file implements FormatTokenLexer, which tokenizes a source file
12 /// into a FormatToken stream suitable for ClangFormat.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "FormatTokenLexer.h"
17 #include "FormatToken.h"
18 #include "clang/Basic/SourceLocation.h"
19 #include "clang/Basic/SourceManager.h"
20 #include "clang/Format/Format.h"
21 #include "llvm/Support/Regex.h"
22 
23 namespace clang {
24 namespace format {
25 
26 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
27                                    const FormatStyle &Style,
28                                    encoding::Encoding Encoding)
29     : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
30       Column(0), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
31       Style(Style), IdentTable(getFormattingLangOpts(Style)),
32       Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
33       FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
34       MacroBlockEndRegex(Style.MacroBlockEnd) {
35   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
36                       getFormattingLangOpts(Style)));
37   Lex->SetKeepWhitespaceMode(true);
38 
39   for (const std::string &ForEachMacro : Style.ForEachMacros)
40     ForEachMacros.push_back(&IdentTable.get(ForEachMacro));
41   std::sort(ForEachMacros.begin(), ForEachMacros.end());
42 }
43 
44 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
45   assert(Tokens.empty());
46   assert(FirstInLineIndex == 0);
47   do {
48     Tokens.push_back(getNextToken());
49     if (Style.Language == FormatStyle::LK_JavaScript) {
50       tryParseJSRegexLiteral();
51       handleTemplateStrings();
52     }
53     tryMergePreviousTokens();
54     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
55       FirstInLineIndex = Tokens.size() - 1;
56   } while (Tokens.back()->Tok.isNot(tok::eof));
57   return Tokens;
58 }
59 
60 void FormatTokenLexer::tryMergePreviousTokens() {
61   if (tryMerge_TMacro())
62     return;
63   if (tryMergeConflictMarkers())
64     return;
65   if (tryMergeLessLess())
66     return;
67 
68   if (Style.Language == FormatStyle::LK_JavaScript) {
69     static const tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal};
70     static const tok::TokenKind JSNotIdentity[] = {tok::exclaimequal,
71                                                    tok::equal};
72     static const tok::TokenKind JSShiftEqual[] = {tok::greater, tok::greater,
73                                                   tok::greaterequal};
74     static const tok::TokenKind JSRightArrow[] = {tok::equal, tok::greater};
75     // FIXME: Investigate what token type gives the correct operator priority.
76     if (tryMergeTokens(JSIdentity, TT_BinaryOperator))
77       return;
78     if (tryMergeTokens(JSNotIdentity, TT_BinaryOperator))
79       return;
80     if (tryMergeTokens(JSShiftEqual, TT_BinaryOperator))
81       return;
82     if (tryMergeTokens(JSRightArrow, TT_JsFatArrow))
83       return;
84   }
85 }
86 
87 bool FormatTokenLexer::tryMergeLessLess() {
88   // Merge X,less,less,Y into X,lessless,Y unless X or Y is less.
89   if (Tokens.size() < 3)
90     return false;
91 
92   bool FourthTokenIsLess = false;
93   if (Tokens.size() > 3)
94     FourthTokenIsLess = (Tokens.end() - 4)[0]->is(tok::less);
95 
96   auto First = Tokens.end() - 3;
97   if (First[2]->is(tok::less) || First[1]->isNot(tok::less) ||
98       First[0]->isNot(tok::less) || FourthTokenIsLess)
99     return false;
100 
101   // Only merge if there currently is no whitespace between the two "<".
102   if (First[1]->WhitespaceRange.getBegin() !=
103       First[1]->WhitespaceRange.getEnd())
104     return false;
105 
106   First[0]->Tok.setKind(tok::lessless);
107   First[0]->TokenText = "<<";
108   First[0]->ColumnWidth += 1;
109   Tokens.erase(Tokens.end() - 2);
110   return true;
111 }
112 
113 bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
114                                       TokenType NewType) {
115   if (Tokens.size() < Kinds.size())
116     return false;
117 
118   SmallVectorImpl<FormatToken *>::const_iterator First =
119       Tokens.end() - Kinds.size();
120   if (!First[0]->is(Kinds[0]))
121     return false;
122   unsigned AddLength = 0;
123   for (unsigned i = 1; i < Kinds.size(); ++i) {
124     if (!First[i]->is(Kinds[i]) ||
125         First[i]->WhitespaceRange.getBegin() !=
126             First[i]->WhitespaceRange.getEnd())
127       return false;
128     AddLength += First[i]->TokenText.size();
129   }
130   Tokens.resize(Tokens.size() - Kinds.size() + 1);
131   First[0]->TokenText = StringRef(First[0]->TokenText.data(),
132                                   First[0]->TokenText.size() + AddLength);
133   First[0]->ColumnWidth += AddLength;
134   First[0]->Type = NewType;
135   return true;
136 }
137 
138 // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
139 bool FormatTokenLexer::precedesOperand(FormatToken *Tok) {
140   // NB: This is not entirely correct, as an r_paren can introduce an operand
141   // location in e.g. `if (foo) /bar/.exec(...);`. That is a rare enough
142   // corner case to not matter in practice, though.
143   return Tok->isOneOf(tok::period, tok::l_paren, tok::comma, tok::l_brace,
144                       tok::r_brace, tok::l_square, tok::semi, tok::exclaim,
145                       tok::colon, tok::question, tok::tilde) ||
146          Tok->isOneOf(tok::kw_return, tok::kw_do, tok::kw_case, tok::kw_throw,
147                       tok::kw_else, tok::kw_new, tok::kw_delete, tok::kw_void,
148                       tok::kw_typeof, Keywords.kw_instanceof, Keywords.kw_in) ||
149          Tok->isBinaryOperator();
150 }
151 
152 bool FormatTokenLexer::canPrecedeRegexLiteral(FormatToken *Prev) {
153   if (!Prev)
154     return true;
155 
156   // Regex literals can only follow after prefix unary operators, not after
157   // postfix unary operators. If the '++' is followed by a non-operand
158   // introducing token, the slash here is the operand and not the start of a
159   // regex.
160   if (Prev->isOneOf(tok::plusplus, tok::minusminus))
161     return (Tokens.size() < 3 || precedesOperand(Tokens[Tokens.size() - 3]));
162 
163   // The previous token must introduce an operand location where regex
164   // literals can occur.
165   if (!precedesOperand(Prev))
166     return false;
167 
168   return true;
169 }
170 
171 // Tries to parse a JavaScript Regex literal starting at the current token,
172 // if that begins with a slash and is in a location where JavaScript allows
173 // regex literals. Changes the current token to a regex literal and updates
174 // its text if successful.
175 void FormatTokenLexer::tryParseJSRegexLiteral() {
176   FormatToken *RegexToken = Tokens.back();
177   if (!RegexToken->isOneOf(tok::slash, tok::slashequal))
178     return;
179 
180   FormatToken *Prev = nullptr;
181   for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; ++I) {
182     // NB: Because previous pointers are not initialized yet, this cannot use
183     // Token.getPreviousNonComment.
184     if ((*I)->isNot(tok::comment)) {
185       Prev = *I;
186       break;
187     }
188   }
189 
190   if (!canPrecedeRegexLiteral(Prev))
191     return;
192 
193   // 'Manually' lex ahead in the current file buffer.
194   const char *Offset = Lex->getBufferLocation();
195   const char *RegexBegin = Offset - RegexToken->TokenText.size();
196   StringRef Buffer = Lex->getBuffer();
197   bool InCharacterClass = false;
198   bool HaveClosingSlash = false;
199   for (; !HaveClosingSlash && Offset != Buffer.end(); ++Offset) {
200     // Regular expressions are terminated with a '/', which can only be
201     // escaped using '\' or a character class between '[' and ']'.
202     // See http://www.ecma-international.org/ecma-262/5.1/#sec-7.8.5.
203     switch (*Offset) {
204     case '\\':
205       // Skip the escaped character.
206       ++Offset;
207       break;
208     case '[':
209       InCharacterClass = true;
210       break;
211     case ']':
212       InCharacterClass = false;
213       break;
214     case '/':
215       if (!InCharacterClass)
216         HaveClosingSlash = true;
217       break;
218     }
219   }
220 
221   RegexToken->Type = TT_RegexLiteral;
222   // Treat regex literals like other string_literals.
223   RegexToken->Tok.setKind(tok::string_literal);
224   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
225   RegexToken->ColumnWidth = RegexToken->TokenText.size();
226 
227   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
228 }
229 
230 void FormatTokenLexer::handleTemplateStrings() {
231   FormatToken *BacktickToken = Tokens.back();
232 
233   if (BacktickToken->is(tok::l_brace)) {
234     StateStack.push(LexerState::NORMAL);
235     return;
236   }
237   if (BacktickToken->is(tok::r_brace)) {
238     StateStack.pop();
239     if (StateStack.top() != LexerState::TEMPLATE_STRING)
240       return;
241     // If back in TEMPLATE_STRING, fallthrough and continue parsing the
242   } else if (BacktickToken->is(tok::unknown) &&
243              BacktickToken->TokenText == "`") {
244     StateStack.push(LexerState::TEMPLATE_STRING);
245   } else {
246     return; // Not actually a template
247   }
248 
249   // 'Manually' lex ahead in the current file buffer.
250   const char *Offset = Lex->getBufferLocation();
251   const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
252   for (; Offset != Lex->getBuffer().end(); ++Offset) {
253     if (Offset[0] == '`') {
254       StateStack.pop();
255       break;
256     }
257     if (Offset[0] == '\\') {
258       ++Offset; // Skip the escaped character.
259     } else if (Offset + 1 < Lex->getBuffer().end() && Offset[0] == '$' &&
260                Offset[1] == '{') {
261       // '${' introduces an expression interpolation in the template string.
262       StateStack.push(LexerState::NORMAL);
263       ++Offset;
264       break;
265     }
266   }
267 
268   StringRef LiteralText(TmplBegin, Offset - TmplBegin + 1);
269   BacktickToken->Type = TT_TemplateString;
270   BacktickToken->Tok.setKind(tok::string_literal);
271   BacktickToken->TokenText = LiteralText;
272 
273   // Adjust width for potentially multiline string literals.
274   size_t FirstBreak = LiteralText.find('\n');
275   StringRef FirstLineText = FirstBreak == StringRef::npos
276                                 ? LiteralText
277                                 : LiteralText.substr(0, FirstBreak);
278   BacktickToken->ColumnWidth = encoding::columnWidthWithTabs(
279       FirstLineText, BacktickToken->OriginalColumn, Style.TabWidth, Encoding);
280   size_t LastBreak = LiteralText.rfind('\n');
281   if (LastBreak != StringRef::npos) {
282     BacktickToken->IsMultiline = true;
283     unsigned StartColumn = 0; // The template tail spans the entire line.
284     BacktickToken->LastLineColumnWidth = encoding::columnWidthWithTabs(
285         LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
286         Style.TabWidth, Encoding);
287   }
288 
289   SourceLocation loc = Offset < Lex->getBuffer().end()
290                            ? Lex->getSourceLocation(Offset + 1)
291                            : SourceMgr.getLocForEndOfFile(ID);
292   resetLexer(SourceMgr.getFileOffset(loc));
293 }
294 
295 bool FormatTokenLexer::tryMerge_TMacro() {
296   if (Tokens.size() < 4)
297     return false;
298   FormatToken *Last = Tokens.back();
299   if (!Last->is(tok::r_paren))
300     return false;
301 
302   FormatToken *String = Tokens[Tokens.size() - 2];
303   if (!String->is(tok::string_literal) || String->IsMultiline)
304     return false;
305 
306   if (!Tokens[Tokens.size() - 3]->is(tok::l_paren))
307     return false;
308 
309   FormatToken *Macro = Tokens[Tokens.size() - 4];
310   if (Macro->TokenText != "_T")
311     return false;
312 
313   const char *Start = Macro->TokenText.data();
314   const char *End = Last->TokenText.data() + Last->TokenText.size();
315   String->TokenText = StringRef(Start, End - Start);
316   String->IsFirst = Macro->IsFirst;
317   String->LastNewlineOffset = Macro->LastNewlineOffset;
318   String->WhitespaceRange = Macro->WhitespaceRange;
319   String->OriginalColumn = Macro->OriginalColumn;
320   String->ColumnWidth = encoding::columnWidthWithTabs(
321       String->TokenText, String->OriginalColumn, Style.TabWidth, Encoding);
322   String->NewlinesBefore = Macro->NewlinesBefore;
323   String->HasUnescapedNewline = Macro->HasUnescapedNewline;
324 
325   Tokens.pop_back();
326   Tokens.pop_back();
327   Tokens.pop_back();
328   Tokens.back() = String;
329   return true;
330 }
331 
332 bool FormatTokenLexer::tryMergeConflictMarkers() {
333   if (Tokens.back()->NewlinesBefore == 0 && Tokens.back()->isNot(tok::eof))
334     return false;
335 
336   // Conflict lines look like:
337   // <marker> <text from the vcs>
338   // For example:
339   // >>>>>>> /file/in/file/system at revision 1234
340   //
341   // We merge all tokens in a line that starts with a conflict marker
342   // into a single token with a special token type that the unwrapped line
343   // parser will use to correctly rebuild the underlying code.
344 
345   FileID ID;
346   // Get the position of the first token in the line.
347   unsigned FirstInLineOffset;
348   std::tie(ID, FirstInLineOffset) = SourceMgr.getDecomposedLoc(
349       Tokens[FirstInLineIndex]->getStartOfNonWhitespace());
350   StringRef Buffer = SourceMgr.getBuffer(ID)->getBuffer();
351   // Calculate the offset of the start of the current line.
352   auto LineOffset = Buffer.rfind('\n', FirstInLineOffset);
353   if (LineOffset == StringRef::npos) {
354     LineOffset = 0;
355   } else {
356     ++LineOffset;
357   }
358 
359   auto FirstSpace = Buffer.find_first_of(" \n", LineOffset);
360   StringRef LineStart;
361   if (FirstSpace == StringRef::npos) {
362     LineStart = Buffer.substr(LineOffset);
363   } else {
364     LineStart = Buffer.substr(LineOffset, FirstSpace - LineOffset);
365   }
366 
367   TokenType Type = TT_Unknown;
368   if (LineStart == "<<<<<<<" || LineStart == ">>>>") {
369     Type = TT_ConflictStart;
370   } else if (LineStart == "|||||||" || LineStart == "=======" ||
371              LineStart == "====") {
372     Type = TT_ConflictAlternative;
373   } else if (LineStart == ">>>>>>>" || LineStart == "<<<<") {
374     Type = TT_ConflictEnd;
375   }
376 
377   if (Type != TT_Unknown) {
378     FormatToken *Next = Tokens.back();
379 
380     Tokens.resize(FirstInLineIndex + 1);
381     // We do not need to build a complete token here, as we will skip it
382     // during parsing anyway (as we must not touch whitespace around conflict
383     // markers).
384     Tokens.back()->Type = Type;
385     Tokens.back()->Tok.setKind(tok::kw___unknown_anytype);
386 
387     Tokens.push_back(Next);
388     return true;
389   }
390 
391   return false;
392 }
393 
394 FormatToken *FormatTokenLexer::getStashedToken() {
395   // Create a synthesized second '>' or '<' token.
396   Token Tok = FormatTok->Tok;
397   StringRef TokenText = FormatTok->TokenText;
398 
399   unsigned OriginalColumn = FormatTok->OriginalColumn;
400   FormatTok = new (Allocator.Allocate()) FormatToken;
401   FormatTok->Tok = Tok;
402   SourceLocation TokLocation =
403       FormatTok->Tok.getLocation().getLocWithOffset(Tok.getLength() - 1);
404   FormatTok->Tok.setLocation(TokLocation);
405   FormatTok->WhitespaceRange = SourceRange(TokLocation, TokLocation);
406   FormatTok->TokenText = TokenText;
407   FormatTok->ColumnWidth = 1;
408   FormatTok->OriginalColumn = OriginalColumn + 1;
409 
410   return FormatTok;
411 }
412 
413 FormatToken *FormatTokenLexer::getNextToken() {
414   if (StateStack.top() == LexerState::TOKEN_STASHED) {
415     StateStack.pop();
416     return getStashedToken();
417   }
418 
419   FormatTok = new (Allocator.Allocate()) FormatToken;
420   readRawToken(*FormatTok);
421   SourceLocation WhitespaceStart =
422       FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
423   FormatTok->IsFirst = IsFirstToken;
424   IsFirstToken = false;
425 
426   // Consume and record whitespace until we find a significant token.
427   unsigned WhitespaceLength = TrailingWhitespace;
428   while (FormatTok->Tok.is(tok::unknown)) {
429     StringRef Text = FormatTok->TokenText;
430     auto EscapesNewline = [&](int pos) {
431       // A '\r' here is just part of '\r\n'. Skip it.
432       if (pos >= 0 && Text[pos] == '\r')
433         --pos;
434       // See whether there is an odd number of '\' before this.
435       unsigned count = 0;
436       for (; pos >= 0; --pos, ++count)
437         if (Text[pos] != '\\')
438           break;
439       return count & 1;
440     };
441     // FIXME: This miscounts tok:unknown tokens that are not just
442     // whitespace, e.g. a '`' character.
443     for (int i = 0, e = Text.size(); i != e; ++i) {
444       switch (Text[i]) {
445       case '\n':
446         ++FormatTok->NewlinesBefore;
447         FormatTok->HasUnescapedNewline = !EscapesNewline(i - 1);
448         FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
449         Column = 0;
450         break;
451       case '\r':
452         FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
453         Column = 0;
454         break;
455       case '\f':
456       case '\v':
457         Column = 0;
458         break;
459       case ' ':
460         ++Column;
461         break;
462       case '\t':
463         Column += Style.TabWidth - Column % Style.TabWidth;
464         break;
465       case '\\':
466         if (i + 1 == e || (Text[i + 1] != '\r' && Text[i + 1] != '\n'))
467           FormatTok->Type = TT_ImplicitStringLiteral;
468         break;
469       default:
470         FormatTok->Type = TT_ImplicitStringLiteral;
471         break;
472       }
473       if (FormatTok->Type == TT_ImplicitStringLiteral)
474         break;
475     }
476 
477     if (FormatTok->is(TT_ImplicitStringLiteral))
478       break;
479     WhitespaceLength += FormatTok->Tok.getLength();
480 
481     readRawToken(*FormatTok);
482   }
483 
484   // In case the token starts with escaped newlines, we want to
485   // take them into account as whitespace - this pattern is quite frequent
486   // in macro definitions.
487   // FIXME: Add a more explicit test.
488   while (FormatTok->TokenText.size() > 1 && FormatTok->TokenText[0] == '\\' &&
489          FormatTok->TokenText[1] == '\n') {
490     ++FormatTok->NewlinesBefore;
491     WhitespaceLength += 2;
492     FormatTok->LastNewlineOffset = 2;
493     Column = 0;
494     FormatTok->TokenText = FormatTok->TokenText.substr(2);
495   }
496 
497   FormatTok->WhitespaceRange = SourceRange(
498       WhitespaceStart, WhitespaceStart.getLocWithOffset(WhitespaceLength));
499 
500   FormatTok->OriginalColumn = Column;
501 
502   TrailingWhitespace = 0;
503   if (FormatTok->Tok.is(tok::comment)) {
504     // FIXME: Add the trimmed whitespace to Column.
505     StringRef UntrimmedText = FormatTok->TokenText;
506     FormatTok->TokenText = FormatTok->TokenText.rtrim(" \t\v\f");
507     TrailingWhitespace = UntrimmedText.size() - FormatTok->TokenText.size();
508   } else if (FormatTok->Tok.is(tok::raw_identifier)) {
509     IdentifierInfo &Info = IdentTable.get(FormatTok->TokenText);
510     FormatTok->Tok.setIdentifierInfo(&Info);
511     FormatTok->Tok.setKind(Info.getTokenID());
512     if (Style.Language == FormatStyle::LK_Java &&
513         FormatTok->isOneOf(tok::kw_struct, tok::kw_union, tok::kw_delete,
514                            tok::kw_operator)) {
515       FormatTok->Tok.setKind(tok::identifier);
516       FormatTok->Tok.setIdentifierInfo(nullptr);
517     } else if (Style.Language == FormatStyle::LK_JavaScript &&
518                FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
519                                   tok::kw_operator)) {
520       FormatTok->Tok.setKind(tok::identifier);
521       FormatTok->Tok.setIdentifierInfo(nullptr);
522     }
523   } else if (FormatTok->Tok.is(tok::greatergreater)) {
524     FormatTok->Tok.setKind(tok::greater);
525     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
526     StateStack.push(LexerState::TOKEN_STASHED);
527   } else if (FormatTok->Tok.is(tok::lessless)) {
528     FormatTok->Tok.setKind(tok::less);
529     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
530     StateStack.push(LexerState::TOKEN_STASHED);
531   }
532 
533   // Now FormatTok is the next non-whitespace token.
534 
535   StringRef Text = FormatTok->TokenText;
536   size_t FirstNewlinePos = Text.find('\n');
537   if (FirstNewlinePos == StringRef::npos) {
538     // FIXME: ColumnWidth actually depends on the start column, we need to
539     // take this into account when the token is moved.
540     FormatTok->ColumnWidth =
541         encoding::columnWidthWithTabs(Text, Column, Style.TabWidth, Encoding);
542     Column += FormatTok->ColumnWidth;
543   } else {
544     FormatTok->IsMultiline = true;
545     // FIXME: ColumnWidth actually depends on the start column, we need to
546     // take this into account when the token is moved.
547     FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
548         Text.substr(0, FirstNewlinePos), Column, Style.TabWidth, Encoding);
549 
550     // The last line of the token always starts in column 0.
551     // Thus, the length can be precomputed even in the presence of tabs.
552     FormatTok->LastLineColumnWidth = encoding::columnWidthWithTabs(
553         Text.substr(Text.find_last_of('\n') + 1), 0, Style.TabWidth, Encoding);
554     Column = FormatTok->LastLineColumnWidth;
555   }
556 
557   if (Style.Language == FormatStyle::LK_Cpp) {
558     if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
559           Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
560               tok::pp_define) &&
561         std::find(ForEachMacros.begin(), ForEachMacros.end(),
562                   FormatTok->Tok.getIdentifierInfo()) != ForEachMacros.end()) {
563       FormatTok->Type = TT_ForEachMacro;
564     } else if (FormatTok->is(tok::identifier)) {
565       if (MacroBlockBeginRegex.match(Text)) {
566         FormatTok->Type = TT_MacroBlockBegin;
567       } else if (MacroBlockEndRegex.match(Text)) {
568         FormatTok->Type = TT_MacroBlockEnd;
569       }
570     }
571   }
572 
573   return FormatTok;
574 }
575 
576 void FormatTokenLexer::readRawToken(FormatToken &Tok) {
577   Lex->LexFromRawLexer(Tok.Tok);
578   Tok.TokenText = StringRef(SourceMgr.getCharacterData(Tok.Tok.getLocation()),
579                             Tok.Tok.getLength());
580   // For formatting, treat unterminated string literals like normal string
581   // literals.
582   if (Tok.is(tok::unknown)) {
583     if (!Tok.TokenText.empty() && Tok.TokenText[0] == '"') {
584       Tok.Tok.setKind(tok::string_literal);
585       Tok.IsUnterminatedLiteral = true;
586     } else if (Style.Language == FormatStyle::LK_JavaScript &&
587                Tok.TokenText == "''") {
588       Tok.Tok.setKind(tok::string_literal);
589     }
590   }
591 
592   if (Style.Language == FormatStyle::LK_JavaScript &&
593       Tok.is(tok::char_constant)) {
594     Tok.Tok.setKind(tok::string_literal);
595   }
596 
597   if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format on" ||
598                                Tok.TokenText == "/* clang-format on */")) {
599     FormattingDisabled = false;
600   }
601 
602   Tok.Finalized = FormattingDisabled;
603 
604   if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format off" ||
605                                Tok.TokenText == "/* clang-format off */")) {
606     FormattingDisabled = true;
607   }
608 }
609 
610 void FormatTokenLexer::resetLexer(unsigned Offset) {
611   StringRef Buffer = SourceMgr.getBufferData(ID);
612   Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID),
613                       getFormattingLangOpts(Style), Buffer.begin(),
614                       Buffer.begin() + Offset, Buffer.end()));
615   Lex->SetKeepWhitespaceMode(true);
616   TrailingWhitespace = 0;
617 }
618 
619 } // namespace format
620 } // namespace clang
621