1 //===--- FormatTokenLexer.cpp - Lex FormatTokens -------------*- C++ ----*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements FormatTokenLexer, which tokenizes a source file
11 /// into a FormatToken stream suitable for ClangFormat.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "FormatTokenLexer.h"
16 #include "FormatToken.h"
17 #include "clang/Basic/SourceLocation.h"
18 #include "clang/Basic/SourceManager.h"
19 #include "clang/Format/Format.h"
20 #include "llvm/Support/Regex.h"
21 
22 namespace clang {
23 namespace format {
24 
25 FormatTokenLexer::FormatTokenLexer(const SourceManager &SourceMgr, FileID ID,
26                                    unsigned Column, const FormatStyle &Style,
27                                    encoding::Encoding Encoding)
28     : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
29       Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
30       Style(Style), IdentTable(getFormattingLangOpts(Style)),
31       Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
32       FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
33       MacroBlockEndRegex(Style.MacroBlockEnd) {
34   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
35                       getFormattingLangOpts(Style)));
36   Lex->SetKeepWhitespaceMode(true);
37 
38   for (const std::string &ForEachMacro : Style.ForEachMacros)
39     Macros.insert({&IdentTable.get(ForEachMacro), TT_ForEachMacro});
40   for (const std::string &StatementMacro : Style.StatementMacros)
41     Macros.insert({&IdentTable.get(StatementMacro), TT_StatementMacro});
42 }
43 
44 ArrayRef<FormatToken *> FormatTokenLexer::lex() {
45   assert(Tokens.empty());
46   assert(FirstInLineIndex == 0);
47   do {
48     Tokens.push_back(getNextToken());
49     if (Style.Language == FormatStyle::LK_JavaScript) {
50       tryParseJSRegexLiteral();
51       handleTemplateStrings();
52     }
53     if (Style.Language == FormatStyle::LK_TextProto)
54       tryParsePythonComment();
55     tryMergePreviousTokens();
56     if (Tokens.back()->NewlinesBefore > 0 || Tokens.back()->IsMultiline)
57       FirstInLineIndex = Tokens.size() - 1;
58   } while (Tokens.back()->Tok.isNot(tok::eof));
59   return Tokens;
60 }
61 
62 void FormatTokenLexer::tryMergePreviousTokens() {
63   if (tryMerge_TMacro())
64     return;
65   if (tryMergeConflictMarkers())
66     return;
67   if (tryMergeLessLess())
68     return;
69   if (tryMergeNSStringLiteral())
70     return;
71 
72   if (Style.Language == FormatStyle::LK_JavaScript) {
73     static const tok::TokenKind JSIdentity[] = {tok::equalequal, tok::equal};
74     static const tok::TokenKind JSNotIdentity[] = {tok::exclaimequal,
75                                                    tok::equal};
76     static const tok::TokenKind JSShiftEqual[] = {tok::greater, tok::greater,
77                                                   tok::greaterequal};
78     static const tok::TokenKind JSRightArrow[] = {tok::equal, tok::greater};
79     static const tok::TokenKind JSExponentiation[] = {tok::star, tok::star};
80     static const tok::TokenKind JSExponentiationEqual[] = {tok::star,
81                                                            tok::starequal};
82 
83     // FIXME: Investigate what token type gives the correct operator priority.
84     if (tryMergeTokens(JSIdentity, TT_BinaryOperator))
85       return;
86     if (tryMergeTokens(JSNotIdentity, TT_BinaryOperator))
87       return;
88     if (tryMergeTokens(JSShiftEqual, TT_BinaryOperator))
89       return;
90     if (tryMergeTokens(JSRightArrow, TT_JsFatArrow))
91       return;
92     if (tryMergeTokens(JSExponentiation, TT_JsExponentiation))
93       return;
94     if (tryMergeTokens(JSExponentiationEqual, TT_JsExponentiationEqual)) {
95       Tokens.back()->Tok.setKind(tok::starequal);
96       return;
97     }
98     if (tryMergeJSPrivateIdentifier())
99       return;
100   }
101 
102   if (Style.Language == FormatStyle::LK_Java) {
103     static const tok::TokenKind JavaRightLogicalShiftAssign[] = {
104         tok::greater, tok::greater, tok::greaterequal};
105     if (tryMergeTokens(JavaRightLogicalShiftAssign, TT_BinaryOperator))
106       return;
107   }
108 }
109 
110 bool FormatTokenLexer::tryMergeNSStringLiteral() {
111   if (Tokens.size() < 2)
112     return false;
113   auto &At = *(Tokens.end() - 2);
114   auto &String = *(Tokens.end() - 1);
115   if (!At->is(tok::at) || !String->is(tok::string_literal))
116     return false;
117   At->Tok.setKind(tok::string_literal);
118   At->TokenText = StringRef(At->TokenText.begin(),
119                             String->TokenText.end() - At->TokenText.begin());
120   At->ColumnWidth += String->ColumnWidth;
121   At->Type = TT_ObjCStringLiteral;
122   Tokens.erase(Tokens.end() - 1);
123   return true;
124 }
125 
126 bool FormatTokenLexer::tryMergeJSPrivateIdentifier() {
127   // Merges #idenfier into a single identifier with the text #identifier
128   // but the token tok::identifier.
129   if (Tokens.size() < 2)
130     return false;
131   auto &Hash = *(Tokens.end() - 2);
132   auto &Identifier = *(Tokens.end() - 1);
133   if (!Hash->is(tok::hash) || !Identifier->is(tok::identifier))
134     return false;
135   Hash->Tok.setKind(tok::identifier);
136   Hash->TokenText =
137       StringRef(Hash->TokenText.begin(),
138                 Identifier->TokenText.end() - Hash->TokenText.begin());
139   Hash->ColumnWidth += Identifier->ColumnWidth;
140   Hash->Type = TT_JsPrivateIdentifier;
141   Tokens.erase(Tokens.end() - 1);
142   return true;
143 }
144 
145 bool FormatTokenLexer::tryMergeLessLess() {
146   // Merge X,less,less,Y into X,lessless,Y unless X or Y is less.
147   if (Tokens.size() < 3)
148     return false;
149 
150   bool FourthTokenIsLess = false;
151   if (Tokens.size() > 3)
152     FourthTokenIsLess = (Tokens.end() - 4)[0]->is(tok::less);
153 
154   auto First = Tokens.end() - 3;
155   if (First[2]->is(tok::less) || First[1]->isNot(tok::less) ||
156       First[0]->isNot(tok::less) || FourthTokenIsLess)
157     return false;
158 
159   // Only merge if there currently is no whitespace between the two "<".
160   if (First[1]->WhitespaceRange.getBegin() !=
161       First[1]->WhitespaceRange.getEnd())
162     return false;
163 
164   First[0]->Tok.setKind(tok::lessless);
165   First[0]->TokenText = "<<";
166   First[0]->ColumnWidth += 1;
167   Tokens.erase(Tokens.end() - 2);
168   return true;
169 }
170 
171 bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
172                                       TokenType NewType) {
173   if (Tokens.size() < Kinds.size())
174     return false;
175 
176   SmallVectorImpl<FormatToken *>::const_iterator First =
177       Tokens.end() - Kinds.size();
178   if (!First[0]->is(Kinds[0]))
179     return false;
180   unsigned AddLength = 0;
181   for (unsigned i = 1; i < Kinds.size(); ++i) {
182     if (!First[i]->is(Kinds[i]) || First[i]->WhitespaceRange.getBegin() !=
183                                        First[i]->WhitespaceRange.getEnd())
184       return false;
185     AddLength += First[i]->TokenText.size();
186   }
187   Tokens.resize(Tokens.size() - Kinds.size() + 1);
188   First[0]->TokenText = StringRef(First[0]->TokenText.data(),
189                                   First[0]->TokenText.size() + AddLength);
190   First[0]->ColumnWidth += AddLength;
191   First[0]->Type = NewType;
192   return true;
193 }
194 
195 // Returns \c true if \p Tok can only be followed by an operand in JavaScript.
196 bool FormatTokenLexer::precedesOperand(FormatToken *Tok) {
197   // NB: This is not entirely correct, as an r_paren can introduce an operand
198   // location in e.g. `if (foo) /bar/.exec(...);`. That is a rare enough
199   // corner case to not matter in practice, though.
200   return Tok->isOneOf(tok::period, tok::l_paren, tok::comma, tok::l_brace,
201                       tok::r_brace, tok::l_square, tok::semi, tok::exclaim,
202                       tok::colon, tok::question, tok::tilde) ||
203          Tok->isOneOf(tok::kw_return, tok::kw_do, tok::kw_case, tok::kw_throw,
204                       tok::kw_else, tok::kw_new, tok::kw_delete, tok::kw_void,
205                       tok::kw_typeof, Keywords.kw_instanceof, Keywords.kw_in) ||
206          Tok->isBinaryOperator();
207 }
208 
209 bool FormatTokenLexer::canPrecedeRegexLiteral(FormatToken *Prev) {
210   if (!Prev)
211     return true;
212 
213   // Regex literals can only follow after prefix unary operators, not after
214   // postfix unary operators. If the '++' is followed by a non-operand
215   // introducing token, the slash here is the operand and not the start of a
216   // regex.
217   // `!` is an unary prefix operator, but also a post-fix operator that casts
218   // away nullability, so the same check applies.
219   if (Prev->isOneOf(tok::plusplus, tok::minusminus, tok::exclaim))
220     return (Tokens.size() < 3 || precedesOperand(Tokens[Tokens.size() - 3]));
221 
222   // The previous token must introduce an operand location where regex
223   // literals can occur.
224   if (!precedesOperand(Prev))
225     return false;
226 
227   return true;
228 }
229 
230 // Tries to parse a JavaScript Regex literal starting at the current token,
231 // if that begins with a slash and is in a location where JavaScript allows
232 // regex literals. Changes the current token to a regex literal and updates
233 // its text if successful.
234 void FormatTokenLexer::tryParseJSRegexLiteral() {
235   FormatToken *RegexToken = Tokens.back();
236   if (!RegexToken->isOneOf(tok::slash, tok::slashequal))
237     return;
238 
239   FormatToken *Prev = nullptr;
240   for (auto I = Tokens.rbegin() + 1, E = Tokens.rend(); I != E; ++I) {
241     // NB: Because previous pointers are not initialized yet, this cannot use
242     // Token.getPreviousNonComment.
243     if ((*I)->isNot(tok::comment)) {
244       Prev = *I;
245       break;
246     }
247   }
248 
249   if (!canPrecedeRegexLiteral(Prev))
250     return;
251 
252   // 'Manually' lex ahead in the current file buffer.
253   const char *Offset = Lex->getBufferLocation();
254   const char *RegexBegin = Offset - RegexToken->TokenText.size();
255   StringRef Buffer = Lex->getBuffer();
256   bool InCharacterClass = false;
257   bool HaveClosingSlash = false;
258   for (; !HaveClosingSlash && Offset != Buffer.end(); ++Offset) {
259     // Regular expressions are terminated with a '/', which can only be
260     // escaped using '\' or a character class between '[' and ']'.
261     // See http://www.ecma-international.org/ecma-262/5.1/#sec-7.8.5.
262     switch (*Offset) {
263     case '\\':
264       // Skip the escaped character.
265       ++Offset;
266       break;
267     case '[':
268       InCharacterClass = true;
269       break;
270     case ']':
271       InCharacterClass = false;
272       break;
273     case '/':
274       if (!InCharacterClass)
275         HaveClosingSlash = true;
276       break;
277     }
278   }
279 
280   RegexToken->Type = TT_RegexLiteral;
281   // Treat regex literals like other string_literals.
282   RegexToken->Tok.setKind(tok::string_literal);
283   RegexToken->TokenText = StringRef(RegexBegin, Offset - RegexBegin);
284   RegexToken->ColumnWidth = RegexToken->TokenText.size();
285 
286   resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
287 }
288 
289 void FormatTokenLexer::handleTemplateStrings() {
290   FormatToken *BacktickToken = Tokens.back();
291 
292   if (BacktickToken->is(tok::l_brace)) {
293     StateStack.push(LexerState::NORMAL);
294     return;
295   }
296   if (BacktickToken->is(tok::r_brace)) {
297     if (StateStack.size() == 1)
298       return;
299     StateStack.pop();
300     if (StateStack.top() != LexerState::TEMPLATE_STRING)
301       return;
302     // If back in TEMPLATE_STRING, fallthrough and continue parsing the
303   } else if (BacktickToken->is(tok::unknown) &&
304              BacktickToken->TokenText == "`") {
305     StateStack.push(LexerState::TEMPLATE_STRING);
306   } else {
307     return; // Not actually a template
308   }
309 
310   // 'Manually' lex ahead in the current file buffer.
311   const char *Offset = Lex->getBufferLocation();
312   const char *TmplBegin = Offset - BacktickToken->TokenText.size(); // at "`"
313   for (; Offset != Lex->getBuffer().end(); ++Offset) {
314     if (Offset[0] == '`') {
315       StateStack.pop();
316       break;
317     }
318     if (Offset[0] == '\\') {
319       ++Offset; // Skip the escaped character.
320     } else if (Offset + 1 < Lex->getBuffer().end() && Offset[0] == '$' &&
321                Offset[1] == '{') {
322       // '${' introduces an expression interpolation in the template string.
323       StateStack.push(LexerState::NORMAL);
324       ++Offset;
325       break;
326     }
327   }
328 
329   StringRef LiteralText(TmplBegin, Offset - TmplBegin + 1);
330   BacktickToken->Type = TT_TemplateString;
331   BacktickToken->Tok.setKind(tok::string_literal);
332   BacktickToken->TokenText = LiteralText;
333 
334   // Adjust width for potentially multiline string literals.
335   size_t FirstBreak = LiteralText.find('\n');
336   StringRef FirstLineText = FirstBreak == StringRef::npos
337                                 ? LiteralText
338                                 : LiteralText.substr(0, FirstBreak);
339   BacktickToken->ColumnWidth = encoding::columnWidthWithTabs(
340       FirstLineText, BacktickToken->OriginalColumn, Style.TabWidth, Encoding);
341   size_t LastBreak = LiteralText.rfind('\n');
342   if (LastBreak != StringRef::npos) {
343     BacktickToken->IsMultiline = true;
344     unsigned StartColumn = 0; // The template tail spans the entire line.
345     BacktickToken->LastLineColumnWidth = encoding::columnWidthWithTabs(
346         LiteralText.substr(LastBreak + 1, LiteralText.size()), StartColumn,
347         Style.TabWidth, Encoding);
348   }
349 
350   SourceLocation loc = Offset < Lex->getBuffer().end()
351                            ? Lex->getSourceLocation(Offset + 1)
352                            : SourceMgr.getLocForEndOfFile(ID);
353   resetLexer(SourceMgr.getFileOffset(loc));
354 }
355 
356 void FormatTokenLexer::tryParsePythonComment() {
357   FormatToken *HashToken = Tokens.back();
358   if (!HashToken->isOneOf(tok::hash, tok::hashhash))
359     return;
360   // Turn the remainder of this line into a comment.
361   const char *CommentBegin =
362       Lex->getBufferLocation() - HashToken->TokenText.size(); // at "#"
363   size_t From = CommentBegin - Lex->getBuffer().begin();
364   size_t To = Lex->getBuffer().find_first_of('\n', From);
365   if (To == StringRef::npos)
366     To = Lex->getBuffer().size();
367   size_t Len = To - From;
368   HashToken->Type = TT_LineComment;
369   HashToken->Tok.setKind(tok::comment);
370   HashToken->TokenText = Lex->getBuffer().substr(From, Len);
371   SourceLocation Loc = To < Lex->getBuffer().size()
372                            ? Lex->getSourceLocation(CommentBegin + Len)
373                            : SourceMgr.getLocForEndOfFile(ID);
374   resetLexer(SourceMgr.getFileOffset(Loc));
375 }
376 
377 bool FormatTokenLexer::tryMerge_TMacro() {
378   if (Tokens.size() < 4)
379     return false;
380   FormatToken *Last = Tokens.back();
381   if (!Last->is(tok::r_paren))
382     return false;
383 
384   FormatToken *String = Tokens[Tokens.size() - 2];
385   if (!String->is(tok::string_literal) || String->IsMultiline)
386     return false;
387 
388   if (!Tokens[Tokens.size() - 3]->is(tok::l_paren))
389     return false;
390 
391   FormatToken *Macro = Tokens[Tokens.size() - 4];
392   if (Macro->TokenText != "_T")
393     return false;
394 
395   const char *Start = Macro->TokenText.data();
396   const char *End = Last->TokenText.data() + Last->TokenText.size();
397   String->TokenText = StringRef(Start, End - Start);
398   String->IsFirst = Macro->IsFirst;
399   String->LastNewlineOffset = Macro->LastNewlineOffset;
400   String->WhitespaceRange = Macro->WhitespaceRange;
401   String->OriginalColumn = Macro->OriginalColumn;
402   String->ColumnWidth = encoding::columnWidthWithTabs(
403       String->TokenText, String->OriginalColumn, Style.TabWidth, Encoding);
404   String->NewlinesBefore = Macro->NewlinesBefore;
405   String->HasUnescapedNewline = Macro->HasUnescapedNewline;
406 
407   Tokens.pop_back();
408   Tokens.pop_back();
409   Tokens.pop_back();
410   Tokens.back() = String;
411   return true;
412 }
413 
414 bool FormatTokenLexer::tryMergeConflictMarkers() {
415   if (Tokens.back()->NewlinesBefore == 0 && Tokens.back()->isNot(tok::eof))
416     return false;
417 
418   // Conflict lines look like:
419   // <marker> <text from the vcs>
420   // For example:
421   // >>>>>>> /file/in/file/system at revision 1234
422   //
423   // We merge all tokens in a line that starts with a conflict marker
424   // into a single token with a special token type that the unwrapped line
425   // parser will use to correctly rebuild the underlying code.
426 
427   FileID ID;
428   // Get the position of the first token in the line.
429   unsigned FirstInLineOffset;
430   std::tie(ID, FirstInLineOffset) = SourceMgr.getDecomposedLoc(
431       Tokens[FirstInLineIndex]->getStartOfNonWhitespace());
432   StringRef Buffer = SourceMgr.getBuffer(ID)->getBuffer();
433   // Calculate the offset of the start of the current line.
434   auto LineOffset = Buffer.rfind('\n', FirstInLineOffset);
435   if (LineOffset == StringRef::npos) {
436     LineOffset = 0;
437   } else {
438     ++LineOffset;
439   }
440 
441   auto FirstSpace = Buffer.find_first_of(" \n", LineOffset);
442   StringRef LineStart;
443   if (FirstSpace == StringRef::npos) {
444     LineStart = Buffer.substr(LineOffset);
445   } else {
446     LineStart = Buffer.substr(LineOffset, FirstSpace - LineOffset);
447   }
448 
449   TokenType Type = TT_Unknown;
450   if (LineStart == "<<<<<<<" || LineStart == ">>>>") {
451     Type = TT_ConflictStart;
452   } else if (LineStart == "|||||||" || LineStart == "=======" ||
453              LineStart == "====") {
454     Type = TT_ConflictAlternative;
455   } else if (LineStart == ">>>>>>>" || LineStart == "<<<<") {
456     Type = TT_ConflictEnd;
457   }
458 
459   if (Type != TT_Unknown) {
460     FormatToken *Next = Tokens.back();
461 
462     Tokens.resize(FirstInLineIndex + 1);
463     // We do not need to build a complete token here, as we will skip it
464     // during parsing anyway (as we must not touch whitespace around conflict
465     // markers).
466     Tokens.back()->Type = Type;
467     Tokens.back()->Tok.setKind(tok::kw___unknown_anytype);
468 
469     Tokens.push_back(Next);
470     return true;
471   }
472 
473   return false;
474 }
475 
476 FormatToken *FormatTokenLexer::getStashedToken() {
477   // Create a synthesized second '>' or '<' token.
478   Token Tok = FormatTok->Tok;
479   StringRef TokenText = FormatTok->TokenText;
480 
481   unsigned OriginalColumn = FormatTok->OriginalColumn;
482   FormatTok = new (Allocator.Allocate()) FormatToken;
483   FormatTok->Tok = Tok;
484   SourceLocation TokLocation =
485       FormatTok->Tok.getLocation().getLocWithOffset(Tok.getLength() - 1);
486   FormatTok->Tok.setLocation(TokLocation);
487   FormatTok->WhitespaceRange = SourceRange(TokLocation, TokLocation);
488   FormatTok->TokenText = TokenText;
489   FormatTok->ColumnWidth = 1;
490   FormatTok->OriginalColumn = OriginalColumn + 1;
491 
492   return FormatTok;
493 }
494 
495 FormatToken *FormatTokenLexer::getNextToken() {
496   if (StateStack.top() == LexerState::TOKEN_STASHED) {
497     StateStack.pop();
498     return getStashedToken();
499   }
500 
501   FormatTok = new (Allocator.Allocate()) FormatToken;
502   readRawToken(*FormatTok);
503   SourceLocation WhitespaceStart =
504       FormatTok->Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
505   FormatTok->IsFirst = IsFirstToken;
506   IsFirstToken = false;
507 
508   // Consume and record whitespace until we find a significant token.
509   unsigned WhitespaceLength = TrailingWhitespace;
510   while (FormatTok->Tok.is(tok::unknown)) {
511     StringRef Text = FormatTok->TokenText;
512     auto EscapesNewline = [&](int pos) {
513       // A '\r' here is just part of '\r\n'. Skip it.
514       if (pos >= 0 && Text[pos] == '\r')
515         --pos;
516       // See whether there is an odd number of '\' before this.
517       // FIXME: This is wrong. A '\' followed by a newline is always removed,
518       // regardless of whether there is another '\' before it.
519       // FIXME: Newlines can also be escaped by a '?' '?' '/' trigraph.
520       unsigned count = 0;
521       for (; pos >= 0; --pos, ++count)
522         if (Text[pos] != '\\')
523           break;
524       return count & 1;
525     };
526     // FIXME: This miscounts tok:unknown tokens that are not just
527     // whitespace, e.g. a '`' character.
528     for (int i = 0, e = Text.size(); i != e; ++i) {
529       switch (Text[i]) {
530       case '\n':
531         ++FormatTok->NewlinesBefore;
532         FormatTok->HasUnescapedNewline = !EscapesNewline(i - 1);
533         FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
534         Column = 0;
535         break;
536       case '\r':
537         FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
538         Column = 0;
539         break;
540       case '\f':
541       case '\v':
542         Column = 0;
543         break;
544       case ' ':
545         ++Column;
546         break;
547       case '\t':
548         Column += Style.TabWidth - Column % Style.TabWidth;
549         break;
550       case '\\':
551         if (i + 1 == e || (Text[i + 1] != '\r' && Text[i + 1] != '\n'))
552           FormatTok->Type = TT_ImplicitStringLiteral;
553         break;
554       default:
555         FormatTok->Type = TT_ImplicitStringLiteral;
556         break;
557       }
558       if (FormatTok->Type == TT_ImplicitStringLiteral)
559         break;
560     }
561 
562     if (FormatTok->is(TT_ImplicitStringLiteral))
563       break;
564     WhitespaceLength += FormatTok->Tok.getLength();
565 
566     readRawToken(*FormatTok);
567   }
568 
569   // JavaScript and Java do not allow to escape the end of the line with a
570   // backslash. Backslashes are syntax errors in plain source, but can occur in
571   // comments. When a single line comment ends with a \, it'll cause the next
572   // line of code to be lexed as a comment, breaking formatting. The code below
573   // finds comments that contain a backslash followed by a line break, truncates
574   // the comment token at the backslash, and resets the lexer to restart behind
575   // the backslash.
576   if ((Style.Language == FormatStyle::LK_JavaScript ||
577        Style.Language == FormatStyle::LK_Java) &&
578       FormatTok->is(tok::comment) && FormatTok->TokenText.startswith("//")) {
579     size_t BackslashPos = FormatTok->TokenText.find('\\');
580     while (BackslashPos != StringRef::npos) {
581       if (BackslashPos + 1 < FormatTok->TokenText.size() &&
582           FormatTok->TokenText[BackslashPos + 1] == '\n') {
583         const char *Offset = Lex->getBufferLocation();
584         Offset -= FormatTok->TokenText.size();
585         Offset += BackslashPos + 1;
586         resetLexer(SourceMgr.getFileOffset(Lex->getSourceLocation(Offset)));
587         FormatTok->TokenText = FormatTok->TokenText.substr(0, BackslashPos + 1);
588         FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
589             FormatTok->TokenText, FormatTok->OriginalColumn, Style.TabWidth,
590             Encoding);
591         break;
592       }
593       BackslashPos = FormatTok->TokenText.find('\\', BackslashPos + 1);
594     }
595   }
596 
597   // In case the token starts with escaped newlines, we want to
598   // take them into account as whitespace - this pattern is quite frequent
599   // in macro definitions.
600   // FIXME: Add a more explicit test.
601   while (FormatTok->TokenText.size() > 1 && FormatTok->TokenText[0] == '\\') {
602     unsigned SkippedWhitespace = 0;
603     if (FormatTok->TokenText.size() > 2 &&
604         (FormatTok->TokenText[1] == '\r' && FormatTok->TokenText[2] == '\n'))
605       SkippedWhitespace = 3;
606     else if (FormatTok->TokenText[1] == '\n')
607       SkippedWhitespace = 2;
608     else
609       break;
610 
611     ++FormatTok->NewlinesBefore;
612     WhitespaceLength += SkippedWhitespace;
613     FormatTok->LastNewlineOffset = SkippedWhitespace;
614     Column = 0;
615     FormatTok->TokenText = FormatTok->TokenText.substr(SkippedWhitespace);
616   }
617 
618   FormatTok->WhitespaceRange = SourceRange(
619       WhitespaceStart, WhitespaceStart.getLocWithOffset(WhitespaceLength));
620 
621   FormatTok->OriginalColumn = Column;
622 
623   TrailingWhitespace = 0;
624   if (FormatTok->Tok.is(tok::comment)) {
625     // FIXME: Add the trimmed whitespace to Column.
626     StringRef UntrimmedText = FormatTok->TokenText;
627     FormatTok->TokenText = FormatTok->TokenText.rtrim(" \t\v\f");
628     TrailingWhitespace = UntrimmedText.size() - FormatTok->TokenText.size();
629   } else if (FormatTok->Tok.is(tok::raw_identifier)) {
630     IdentifierInfo &Info = IdentTable.get(FormatTok->TokenText);
631     FormatTok->Tok.setIdentifierInfo(&Info);
632     FormatTok->Tok.setKind(Info.getTokenID());
633     if (Style.Language == FormatStyle::LK_Java &&
634         FormatTok->isOneOf(tok::kw_struct, tok::kw_union, tok::kw_delete,
635                            tok::kw_operator)) {
636       FormatTok->Tok.setKind(tok::identifier);
637       FormatTok->Tok.setIdentifierInfo(nullptr);
638     } else if (Style.Language == FormatStyle::LK_JavaScript &&
639                FormatTok->isOneOf(tok::kw_struct, tok::kw_union,
640                                   tok::kw_operator)) {
641       FormatTok->Tok.setKind(tok::identifier);
642       FormatTok->Tok.setIdentifierInfo(nullptr);
643     }
644   } else if (FormatTok->Tok.is(tok::greatergreater)) {
645     FormatTok->Tok.setKind(tok::greater);
646     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
647     ++Column;
648     StateStack.push(LexerState::TOKEN_STASHED);
649   } else if (FormatTok->Tok.is(tok::lessless)) {
650     FormatTok->Tok.setKind(tok::less);
651     FormatTok->TokenText = FormatTok->TokenText.substr(0, 1);
652     ++Column;
653     StateStack.push(LexerState::TOKEN_STASHED);
654   }
655 
656   // Now FormatTok is the next non-whitespace token.
657 
658   StringRef Text = FormatTok->TokenText;
659   size_t FirstNewlinePos = Text.find('\n');
660   if (FirstNewlinePos == StringRef::npos) {
661     // FIXME: ColumnWidth actually depends on the start column, we need to
662     // take this into account when the token is moved.
663     FormatTok->ColumnWidth =
664         encoding::columnWidthWithTabs(Text, Column, Style.TabWidth, Encoding);
665     Column += FormatTok->ColumnWidth;
666   } else {
667     FormatTok->IsMultiline = true;
668     // FIXME: ColumnWidth actually depends on the start column, we need to
669     // take this into account when the token is moved.
670     FormatTok->ColumnWidth = encoding::columnWidthWithTabs(
671         Text.substr(0, FirstNewlinePos), Column, Style.TabWidth, Encoding);
672 
673     // The last line of the token always starts in column 0.
674     // Thus, the length can be precomputed even in the presence of tabs.
675     FormatTok->LastLineColumnWidth = encoding::columnWidthWithTabs(
676         Text.substr(Text.find_last_of('\n') + 1), 0, Style.TabWidth, Encoding);
677     Column = FormatTok->LastLineColumnWidth;
678   }
679 
680   if (Style.isCpp()) {
681     auto it = Macros.find(FormatTok->Tok.getIdentifierInfo());
682     if (!(Tokens.size() > 0 && Tokens.back()->Tok.getIdentifierInfo() &&
683           Tokens.back()->Tok.getIdentifierInfo()->getPPKeywordID() ==
684               tok::pp_define) &&
685         it != Macros.end()) {
686       FormatTok->Type = it->second;
687     } else if (FormatTok->is(tok::identifier)) {
688       if (MacroBlockBeginRegex.match(Text)) {
689         FormatTok->Type = TT_MacroBlockBegin;
690       } else if (MacroBlockEndRegex.match(Text)) {
691         FormatTok->Type = TT_MacroBlockEnd;
692       }
693     }
694   }
695 
696   return FormatTok;
697 }
698 
699 void FormatTokenLexer::readRawToken(FormatToken &Tok) {
700   Lex->LexFromRawLexer(Tok.Tok);
701   Tok.TokenText = StringRef(SourceMgr.getCharacterData(Tok.Tok.getLocation()),
702                             Tok.Tok.getLength());
703   // For formatting, treat unterminated string literals like normal string
704   // literals.
705   if (Tok.is(tok::unknown)) {
706     if (!Tok.TokenText.empty() && Tok.TokenText[0] == '"') {
707       Tok.Tok.setKind(tok::string_literal);
708       Tok.IsUnterminatedLiteral = true;
709     } else if (Style.Language == FormatStyle::LK_JavaScript &&
710                Tok.TokenText == "''") {
711       Tok.Tok.setKind(tok::string_literal);
712     }
713   }
714 
715   if ((Style.Language == FormatStyle::LK_JavaScript ||
716        Style.Language == FormatStyle::LK_Proto ||
717        Style.Language == FormatStyle::LK_TextProto) &&
718       Tok.is(tok::char_constant)) {
719     Tok.Tok.setKind(tok::string_literal);
720   }
721 
722   if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format on" ||
723                                Tok.TokenText == "/* clang-format on */")) {
724     FormattingDisabled = false;
725   }
726 
727   Tok.Finalized = FormattingDisabled;
728 
729   if (Tok.is(tok::comment) && (Tok.TokenText == "// clang-format off" ||
730                                Tok.TokenText == "/* clang-format off */")) {
731     FormattingDisabled = true;
732   }
733 }
734 
735 void FormatTokenLexer::resetLexer(unsigned Offset) {
736   StringRef Buffer = SourceMgr.getBufferData(ID);
737   Lex.reset(new Lexer(SourceMgr.getLocForStartOfFile(ID),
738                       getFormattingLangOpts(Style), Buffer.begin(),
739                       Buffer.begin() + Offset, Buffer.end()));
740   Lex->SetKeepWhitespaceMode(true);
741   TrailingWhitespace = 0;
742 }
743 
744 } // namespace format
745 } // namespace clang
746