1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #define DEBUG_TYPE "format-parser"
22 
23 namespace clang {
24 namespace format {
25 
26 class FormatTokenSource {
27 public:
28   virtual ~FormatTokenSource() {}
29   virtual FormatToken *getNextToken() = 0;
30 
31   virtual unsigned getPosition() = 0;
32   virtual FormatToken *setPosition(unsigned Position) = 0;
33 };
34 
35 namespace {
36 
37 class ScopedDeclarationState {
38 public:
39   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
40                          bool MustBeDeclaration)
41       : Line(Line), Stack(Stack) {
42     Line.MustBeDeclaration = MustBeDeclaration;
43     Stack.push_back(MustBeDeclaration);
44   }
45   ~ScopedDeclarationState() {
46     Stack.pop_back();
47     if (!Stack.empty())
48       Line.MustBeDeclaration = Stack.back();
49     else
50       Line.MustBeDeclaration = true;
51   }
52 
53 private:
54   UnwrappedLine &Line;
55   std::vector<bool> &Stack;
56 };
57 
58 class ScopedMacroState : public FormatTokenSource {
59 public:
60   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
61                    FormatToken *&ResetToken)
62       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
63         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
64         Token(nullptr) {
65     TokenSource = this;
66     Line.Level = 0;
67     Line.InPPDirective = true;
68   }
69 
70   ~ScopedMacroState() override {
71     TokenSource = PreviousTokenSource;
72     ResetToken = Token;
73     Line.InPPDirective = false;
74     Line.Level = PreviousLineLevel;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113 
114   FormatToken *Token;
115 };
116 
117 } // end anonymous namespace
118 
119 class ScopedLineState {
120 public:
121   ScopedLineState(UnwrappedLineParser &Parser,
122                   bool SwitchToPreprocessorLines = false)
123       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
124     if (SwitchToPreprocessorLines)
125       Parser.CurrentLines = &Parser.PreprocessorDirectives;
126     else if (!Parser.Line->Tokens.empty())
127       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
128     PreBlockLine = std::move(Parser.Line);
129     Parser.Line = llvm::make_unique<UnwrappedLine>();
130     Parser.Line->Level = PreBlockLine->Level;
131     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
132   }
133 
134   ~ScopedLineState() {
135     if (!Parser.Line->Tokens.empty()) {
136       Parser.addUnwrappedLine();
137     }
138     assert(Parser.Line->Tokens.empty());
139     Parser.Line = std::move(PreBlockLine);
140     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
141       Parser.MustBreakBeforeNextToken = true;
142     Parser.CurrentLines = OriginalLines;
143   }
144 
145 private:
146   UnwrappedLineParser &Parser;
147 
148   std::unique_ptr<UnwrappedLine> PreBlockLine;
149   SmallVectorImpl<UnwrappedLine> *OriginalLines;
150 };
151 
152 class CompoundStatementIndenter {
153 public:
154   CompoundStatementIndenter(UnwrappedLineParser *Parser,
155                             const FormatStyle &Style, unsigned &LineLevel)
156       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
157     if (Style.BraceWrapping.AfterControlStatement)
158       Parser->addUnwrappedLine();
159     if (Style.BraceWrapping.IndentBraces)
160       ++LineLevel;
161   }
162   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
163 
164 private:
165   unsigned &LineLevel;
166   unsigned OldLineLevel;
167 };
168 
169 namespace {
170 
171 class IndexedTokenSource : public FormatTokenSource {
172 public:
173   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
174       : Tokens(Tokens), Position(-1) {}
175 
176   FormatToken *getNextToken() override {
177     ++Position;
178     return Tokens[Position];
179   }
180 
181   unsigned getPosition() override {
182     assert(Position >= 0);
183     return Position;
184   }
185 
186   FormatToken *setPosition(unsigned P) override {
187     Position = P;
188     return Tokens[Position];
189   }
190 
191   void reset() { Position = -1; }
192 
193 private:
194   ArrayRef<FormatToken *> Tokens;
195   int Position;
196 };
197 
198 } // end anonymous namespace
199 
200 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
201                                          const AdditionalKeywords &Keywords,
202                                          ArrayRef<FormatToken *> Tokens,
203                                          UnwrappedLineConsumer &Callback)
204     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
205       CurrentLines(&Lines), Style(Style), Keywords(Keywords), Tokens(nullptr),
206       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
207 
208 void UnwrappedLineParser::reset() {
209   PPBranchLevel = -1;
210   Line.reset(new UnwrappedLine);
211   CommentsBeforeNextToken.clear();
212   FormatTok = nullptr;
213   MustBreakBeforeNextToken = false;
214   PreprocessorDirectives.clear();
215   CurrentLines = &Lines;
216   DeclarationScopeStack.clear();
217   PPStack.clear();
218 }
219 
220 void UnwrappedLineParser::parse() {
221   IndexedTokenSource TokenSource(AllTokens);
222   do {
223     DEBUG(llvm::dbgs() << "----\n");
224     reset();
225     Tokens = &TokenSource;
226     TokenSource.reset();
227 
228     readToken();
229     parseFile();
230     // Create line with eof token.
231     pushToken(FormatTok);
232     addUnwrappedLine();
233 
234     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
235                                                   E = Lines.end();
236          I != E; ++I) {
237       Callback.consumeUnwrappedLine(*I);
238     }
239     Callback.finishRun();
240     Lines.clear();
241     while (!PPLevelBranchIndex.empty() &&
242            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
243       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
244       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
245     }
246     if (!PPLevelBranchIndex.empty()) {
247       ++PPLevelBranchIndex.back();
248       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
249       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
250     }
251   } while (!PPLevelBranchIndex.empty());
252 }
253 
254 void UnwrappedLineParser::parseFile() {
255   // The top-level context in a file always has declarations, except for pre-
256   // processor directives and JavaScript files.
257   bool MustBeDeclaration =
258       !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
259   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
260                                           MustBeDeclaration);
261   parseLevel(/*HasOpeningBrace=*/false);
262   // Make sure to format the remaining tokens.
263   flushComments(true);
264   addUnwrappedLine();
265 }
266 
267 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
268   bool SwitchLabelEncountered = false;
269   do {
270     tok::TokenKind kind = FormatTok->Tok.getKind();
271     if (FormatTok->Type == TT_MacroBlockBegin) {
272       kind = tok::l_brace;
273     } else if (FormatTok->Type == TT_MacroBlockEnd) {
274       kind = tok::r_brace;
275     }
276 
277     switch (kind) {
278     case tok::comment:
279       nextToken();
280       addUnwrappedLine();
281       break;
282     case tok::l_brace:
283       // FIXME: Add parameter whether this can happen - if this happens, we must
284       // be in a non-declaration context.
285       if (!FormatTok->is(TT_MacroBlockBegin) && tryToParseBracedList())
286         continue;
287       parseBlock(/*MustBeDeclaration=*/false);
288       addUnwrappedLine();
289       break;
290     case tok::r_brace:
291       if (HasOpeningBrace)
292         return;
293       nextToken();
294       addUnwrappedLine();
295       break;
296     case tok::kw_default:
297     case tok::kw_case:
298       if (!SwitchLabelEncountered &&
299           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
300         ++Line->Level;
301       SwitchLabelEncountered = true;
302       parseStructuralElement();
303       break;
304     default:
305       parseStructuralElement();
306       break;
307     }
308   } while (!eof());
309 }
310 
311 void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
312   // We'll parse forward through the tokens until we hit
313   // a closing brace or eof - note that getNextToken() will
314   // parse macros, so this will magically work inside macro
315   // definitions, too.
316   unsigned StoredPosition = Tokens->getPosition();
317   FormatToken *Tok = FormatTok;
318   const FormatToken *PrevTok = getPreviousToken();
319   // Keep a stack of positions of lbrace tokens. We will
320   // update information about whether an lbrace starts a
321   // braced init list or a different block during the loop.
322   SmallVector<FormatToken *, 8> LBraceStack;
323   assert(Tok->Tok.is(tok::l_brace));
324   do {
325     // Get next non-comment token.
326     FormatToken *NextTok;
327     unsigned ReadTokens = 0;
328     do {
329       NextTok = Tokens->getNextToken();
330       ++ReadTokens;
331     } while (NextTok->is(tok::comment));
332 
333     switch (Tok->Tok.getKind()) {
334     case tok::l_brace:
335       if (Style.Language == FormatStyle::LK_JavaScript && PrevTok &&
336           PrevTok->is(tok::colon))
337         // In TypeScript's TypeMemberLists, there can be semicolons between the
338         // individual members.
339         Tok->BlockKind = BK_BracedInit;
340       else
341         Tok->BlockKind = BK_Unknown;
342       LBraceStack.push_back(Tok);
343       break;
344     case tok::r_brace:
345       if (LBraceStack.empty())
346         break;
347       if (LBraceStack.back()->BlockKind == BK_Unknown) {
348         bool ProbablyBracedList = false;
349         if (Style.Language == FormatStyle::LK_Proto) {
350           ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
351         } else {
352           // Using OriginalColumn to distinguish between ObjC methods and
353           // binary operators is a bit hacky.
354           bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
355                                   NextTok->OriginalColumn == 0;
356 
357           // If there is a comma, semicolon or right paren after the closing
358           // brace, we assume this is a braced initializer list.  Note that
359           // regardless how we mark inner braces here, we will overwrite the
360           // BlockKind later if we parse a braced list (where all blocks
361           // inside are by default braced lists), or when we explicitly detect
362           // blocks (for example while parsing lambdas).
363           //
364           // We exclude + and - as they can be ObjC visibility modifiers.
365           ProbablyBracedList =
366               (Style.Language == FormatStyle::LK_JavaScript &&
367                NextTok->isOneOf(Keywords.kw_of, Keywords.kw_in)) ||
368               NextTok->isOneOf(tok::comma, tok::period, tok::colon,
369                                tok::r_paren, tok::r_square, tok::l_brace,
370                                tok::l_square, tok::l_paren, tok::ellipsis) ||
371               (NextTok->is(tok::semi) &&
372                (!ExpectClassBody || LBraceStack.size() != 1)) ||
373               (NextTok->isBinaryOperator() && !NextIsObjCMethod);
374         }
375         if (ProbablyBracedList) {
376           Tok->BlockKind = BK_BracedInit;
377           LBraceStack.back()->BlockKind = BK_BracedInit;
378         } else {
379           Tok->BlockKind = BK_Block;
380           LBraceStack.back()->BlockKind = BK_Block;
381         }
382       }
383       LBraceStack.pop_back();
384       break;
385     case tok::at:
386     case tok::semi:
387     case tok::kw_if:
388     case tok::kw_while:
389     case tok::kw_for:
390     case tok::kw_switch:
391     case tok::kw_try:
392     case tok::kw___try:
393       if (!LBraceStack.empty() && LBraceStack.back()->BlockKind == BK_Unknown)
394         LBraceStack.back()->BlockKind = BK_Block;
395       break;
396     default:
397       break;
398     }
399     PrevTok = Tok;
400     Tok = NextTok;
401   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
402 
403   // Assume other blocks for all unclosed opening braces.
404   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
405     if (LBraceStack[i]->BlockKind == BK_Unknown)
406       LBraceStack[i]->BlockKind = BK_Block;
407   }
408 
409   FormatTok = Tokens->setPosition(StoredPosition);
410 }
411 
412 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
413                                      bool MunchSemi) {
414   assert(FormatTok->isOneOf(tok::l_brace, TT_MacroBlockBegin) &&
415          "'{' or macro block token expected");
416   const bool MacroBlock = FormatTok->is(TT_MacroBlockBegin);
417   FormatTok->BlockKind = BK_Block;
418 
419   unsigned InitialLevel = Line->Level;
420   nextToken();
421 
422   if (MacroBlock && FormatTok->is(tok::l_paren))
423     parseParens();
424 
425   addUnwrappedLine();
426 
427   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
428                                           MustBeDeclaration);
429   if (AddLevel)
430     ++Line->Level;
431   parseLevel(/*HasOpeningBrace=*/true);
432 
433   if (eof())
434     return;
435 
436   if (MacroBlock ? !FormatTok->is(TT_MacroBlockEnd)
437                  : !FormatTok->is(tok::r_brace)) {
438     Line->Level = InitialLevel;
439     FormatTok->BlockKind = BK_Block;
440     return;
441   }
442 
443   nextToken(); // Munch the closing brace.
444 
445   if (MacroBlock && FormatTok->is(tok::l_paren))
446     parseParens();
447 
448   if (MunchSemi && FormatTok->Tok.is(tok::semi))
449     nextToken();
450   Line->Level = InitialLevel;
451 }
452 
453 static bool isGoogScope(const UnwrappedLine &Line) {
454   // FIXME: Closure-library specific stuff should not be hard-coded but be
455   // configurable.
456   if (Line.Tokens.size() < 4)
457     return false;
458   auto I = Line.Tokens.begin();
459   if (I->Tok->TokenText != "goog")
460     return false;
461   ++I;
462   if (I->Tok->isNot(tok::period))
463     return false;
464   ++I;
465   if (I->Tok->TokenText != "scope")
466     return false;
467   ++I;
468   return I->Tok->is(tok::l_paren);
469 }
470 
471 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
472                                    const FormatToken &InitialToken) {
473   if (InitialToken.is(tok::kw_namespace))
474     return Style.BraceWrapping.AfterNamespace;
475   if (InitialToken.is(tok::kw_class))
476     return Style.BraceWrapping.AfterClass;
477   if (InitialToken.is(tok::kw_union))
478     return Style.BraceWrapping.AfterUnion;
479   if (InitialToken.is(tok::kw_struct))
480     return Style.BraceWrapping.AfterStruct;
481   return false;
482 }
483 
484 void UnwrappedLineParser::parseChildBlock() {
485   FormatTok->BlockKind = BK_Block;
486   nextToken();
487   {
488     bool GoogScope =
489         Style.Language == FormatStyle::LK_JavaScript && isGoogScope(*Line);
490     ScopedLineState LineState(*this);
491     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
492                                             /*MustBeDeclaration=*/false);
493     Line->Level += GoogScope ? 0 : 1;
494     parseLevel(/*HasOpeningBrace=*/true);
495     flushComments(isOnNewLine(*FormatTok));
496     Line->Level -= GoogScope ? 0 : 1;
497   }
498   nextToken();
499 }
500 
501 void UnwrappedLineParser::parsePPDirective() {
502   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
503   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
504   nextToken();
505 
506   if (!FormatTok->Tok.getIdentifierInfo()) {
507     parsePPUnknown();
508     return;
509   }
510 
511   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
512   case tok::pp_define:
513     parsePPDefine();
514     return;
515   case tok::pp_if:
516     parsePPIf(/*IfDef=*/false);
517     break;
518   case tok::pp_ifdef:
519   case tok::pp_ifndef:
520     parsePPIf(/*IfDef=*/true);
521     break;
522   case tok::pp_else:
523     parsePPElse();
524     break;
525   case tok::pp_elif:
526     parsePPElIf();
527     break;
528   case tok::pp_endif:
529     parsePPEndIf();
530     break;
531   default:
532     parsePPUnknown();
533     break;
534   }
535 }
536 
537 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
538   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
539     PPStack.push_back(PP_Unreachable);
540   else
541     PPStack.push_back(PP_Conditional);
542 }
543 
544 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
545   ++PPBranchLevel;
546   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
547   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
548     PPLevelBranchIndex.push_back(0);
549     PPLevelBranchCount.push_back(0);
550   }
551   PPChainBranchIndex.push(0);
552   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
553   conditionalCompilationCondition(Unreachable || Skip);
554 }
555 
556 void UnwrappedLineParser::conditionalCompilationAlternative() {
557   if (!PPStack.empty())
558     PPStack.pop_back();
559   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
560   if (!PPChainBranchIndex.empty())
561     ++PPChainBranchIndex.top();
562   conditionalCompilationCondition(
563       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
564       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
565 }
566 
567 void UnwrappedLineParser::conditionalCompilationEnd() {
568   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
569   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
570     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
571       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
572     }
573   }
574   // Guard against #endif's without #if.
575   if (PPBranchLevel > 0)
576     --PPBranchLevel;
577   if (!PPChainBranchIndex.empty())
578     PPChainBranchIndex.pop();
579   if (!PPStack.empty())
580     PPStack.pop_back();
581 }
582 
583 void UnwrappedLineParser::parsePPIf(bool IfDef) {
584   nextToken();
585   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
586                          FormatTok->Tok.getLiteralData() != nullptr &&
587                          StringRef(FormatTok->Tok.getLiteralData(),
588                                    FormatTok->Tok.getLength()) == "0") ||
589                         FormatTok->Tok.is(tok::kw_false);
590   conditionalCompilationStart(!IfDef && IsLiteralFalse);
591   parsePPUnknown();
592 }
593 
594 void UnwrappedLineParser::parsePPElse() {
595   conditionalCompilationAlternative();
596   parsePPUnknown();
597 }
598 
599 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
600 
601 void UnwrappedLineParser::parsePPEndIf() {
602   conditionalCompilationEnd();
603   parsePPUnknown();
604 }
605 
606 void UnwrappedLineParser::parsePPDefine() {
607   nextToken();
608 
609   if (FormatTok->Tok.getKind() != tok::identifier) {
610     parsePPUnknown();
611     return;
612   }
613   nextToken();
614   if (FormatTok->Tok.getKind() == tok::l_paren &&
615       FormatTok->WhitespaceRange.getBegin() ==
616           FormatTok->WhitespaceRange.getEnd()) {
617     parseParens();
618   }
619   addUnwrappedLine();
620   Line->Level = 1;
621 
622   // Errors during a preprocessor directive can only affect the layout of the
623   // preprocessor directive, and thus we ignore them. An alternative approach
624   // would be to use the same approach we use on the file level (no
625   // re-indentation if there was a structural error) within the macro
626   // definition.
627   parseFile();
628 }
629 
630 void UnwrappedLineParser::parsePPUnknown() {
631   do {
632     nextToken();
633   } while (!eof());
634   addUnwrappedLine();
635 }
636 
637 // Here we blacklist certain tokens that are not usually the first token in an
638 // unwrapped line. This is used in attempt to distinguish macro calls without
639 // trailing semicolons from other constructs split to several lines.
640 static bool tokenCanStartNewLine(const clang::Token &Tok) {
641   // Semicolon can be a null-statement, l_square can be a start of a macro or
642   // a C++11 attribute, but this doesn't seem to be common.
643   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
644          Tok.isNot(tok::l_square) &&
645          // Tokens that can only be used as binary operators and a part of
646          // overloaded operator names.
647          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
648          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
649          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
650          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
651          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
652          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
653          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
654          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
655          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
656          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
657          Tok.isNot(tok::lesslessequal) &&
658          // Colon is used in labels, base class lists, initializer lists,
659          // range-based for loops, ternary operator, but should never be the
660          // first token in an unwrapped line.
661          Tok.isNot(tok::colon) &&
662          // 'noexcept' is a trailing annotation.
663          Tok.isNot(tok::kw_noexcept);
664 }
665 
666 static bool mustBeJSIdentOrValue(const AdditionalKeywords &Keywords,
667                                  const FormatToken *FormatTok) {
668   if (FormatTok->Tok.isLiteral())
669     return true;
670   // FIXME: This returns true for C/C++ keywords like 'struct'.
671   return FormatTok->is(tok::identifier) &&
672          (FormatTok->Tok.getIdentifierInfo() == nullptr ||
673           !FormatTok->isOneOf(Keywords.kw_in, Keywords.kw_of,
674                               Keywords.kw_finally, Keywords.kw_function,
675                               Keywords.kw_import, Keywords.kw_is,
676                               Keywords.kw_let, Keywords.kw_var,
677                               Keywords.kw_abstract, Keywords.kw_extends,
678                               Keywords.kw_implements, Keywords.kw_instanceof,
679                               Keywords.kw_interface, Keywords.kw_throws));
680 }
681 
682 // isJSDeclOrStmt returns true if |FormatTok| starts a declaration or statement
683 // when encountered after a value (see mustBeJSIdentOrValue).
684 static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
685                            const FormatToken *FormatTok) {
686   return FormatTok->isOneOf(
687       tok::kw_return,
688       // conditionals
689       tok::kw_if, tok::kw_else,
690       // loops
691       tok::kw_for, tok::kw_while, tok::kw_do, tok::kw_continue, tok::kw_break,
692       // switch/case
693       tok::kw_switch, tok::kw_case,
694       // exceptions
695       tok::kw_throw, tok::kw_try, tok::kw_catch, Keywords.kw_finally,
696       // declaration
697       tok::kw_const, tok::kw_class, Keywords.kw_var, Keywords.kw_let,
698       Keywords.kw_function);
699 }
700 
701 // readTokenWithJavaScriptASI reads the next token and terminates the current
702 // line if JavaScript Automatic Semicolon Insertion must
703 // happen between the current token and the next token.
704 //
705 // This method is conservative - it cannot cover all edge cases of JavaScript,
706 // but only aims to correctly handle certain well known cases. It *must not*
707 // return true in speculative cases.
708 void UnwrappedLineParser::readTokenWithJavaScriptASI() {
709   FormatToken *Previous = FormatTok;
710   readToken();
711   FormatToken *Next = FormatTok;
712 
713   bool IsOnSameLine =
714       CommentsBeforeNextToken.empty()
715           ? Next->NewlinesBefore == 0
716           : CommentsBeforeNextToken.front()->NewlinesBefore == 0;
717   if (IsOnSameLine)
718     return;
719 
720   bool PreviousMustBeValue = mustBeJSIdentOrValue(Keywords, Previous);
721   if (PreviousMustBeValue && Line && Line->Tokens.size() > 1) {
722     // If the token before the previous one is an '@', the previous token is an
723     // annotation and can precede another identifier/value.
724     const FormatToken *PrePrevious = std::prev(Line->Tokens.end(), 2)->Tok;
725     if (PrePrevious->is(tok::at))
726       return;
727   }
728   if (Next->is(tok::exclaim) && PreviousMustBeValue)
729     addUnwrappedLine();
730   bool NextMustBeValue = mustBeJSIdentOrValue(Keywords, Next);
731   if (NextMustBeValue && (PreviousMustBeValue ||
732                           Previous->isOneOf(tok::r_square, tok::r_paren,
733                                             tok::plusplus, tok::minusminus)))
734     addUnwrappedLine();
735   if (PreviousMustBeValue && isJSDeclOrStmt(Keywords, Next))
736     addUnwrappedLine();
737 }
738 
739 void UnwrappedLineParser::parseStructuralElement() {
740   assert(!FormatTok->is(tok::l_brace));
741   if (Style.Language == FormatStyle::LK_TableGen &&
742       FormatTok->is(tok::pp_include)) {
743     nextToken();
744     if (FormatTok->is(tok::string_literal))
745       nextToken();
746     addUnwrappedLine();
747     return;
748   }
749   switch (FormatTok->Tok.getKind()) {
750   case tok::at:
751     nextToken();
752     if (FormatTok->Tok.is(tok::l_brace)) {
753       parseBracedList();
754       break;
755     }
756     switch (FormatTok->Tok.getObjCKeywordID()) {
757     case tok::objc_public:
758     case tok::objc_protected:
759     case tok::objc_package:
760     case tok::objc_private:
761       return parseAccessSpecifier();
762     case tok::objc_interface:
763     case tok::objc_implementation:
764       return parseObjCInterfaceOrImplementation();
765     case tok::objc_protocol:
766       return parseObjCProtocol();
767     case tok::objc_end:
768       return; // Handled by the caller.
769     case tok::objc_optional:
770     case tok::objc_required:
771       nextToken();
772       addUnwrappedLine();
773       return;
774     case tok::objc_autoreleasepool:
775       nextToken();
776       if (FormatTok->Tok.is(tok::l_brace)) {
777         if (Style.BraceWrapping.AfterObjCDeclaration)
778           addUnwrappedLine();
779         parseBlock(/*MustBeDeclaration=*/false);
780       }
781       addUnwrappedLine();
782       return;
783     case tok::objc_try:
784       // This branch isn't strictly necessary (the kw_try case below would
785       // do this too after the tok::at is parsed above).  But be explicit.
786       parseTryCatch();
787       return;
788     default:
789       break;
790     }
791     break;
792   case tok::kw_asm:
793     nextToken();
794     if (FormatTok->is(tok::l_brace)) {
795       FormatTok->Type = TT_InlineASMBrace;
796       nextToken();
797       while (FormatTok && FormatTok->isNot(tok::eof)) {
798         if (FormatTok->is(tok::r_brace)) {
799           FormatTok->Type = TT_InlineASMBrace;
800           nextToken();
801           addUnwrappedLine();
802           break;
803         }
804         FormatTok->Finalized = true;
805         nextToken();
806       }
807     }
808     break;
809   case tok::kw_namespace:
810     parseNamespace();
811     return;
812   case tok::kw_inline:
813     nextToken();
814     if (FormatTok->Tok.is(tok::kw_namespace)) {
815       parseNamespace();
816       return;
817     }
818     break;
819   case tok::kw_public:
820   case tok::kw_protected:
821   case tok::kw_private:
822     if (Style.Language == FormatStyle::LK_Java ||
823         Style.Language == FormatStyle::LK_JavaScript)
824       nextToken();
825     else
826       parseAccessSpecifier();
827     return;
828   case tok::kw_if:
829     parseIfThenElse();
830     return;
831   case tok::kw_for:
832   case tok::kw_while:
833     parseForOrWhileLoop();
834     return;
835   case tok::kw_do:
836     parseDoWhile();
837     return;
838   case tok::kw_switch:
839     parseSwitch();
840     return;
841   case tok::kw_default:
842     nextToken();
843     parseLabel();
844     return;
845   case tok::kw_case:
846     parseCaseLabel();
847     return;
848   case tok::kw_try:
849   case tok::kw___try:
850     parseTryCatch();
851     return;
852   case tok::kw_extern:
853     nextToken();
854     if (FormatTok->Tok.is(tok::string_literal)) {
855       nextToken();
856       if (FormatTok->Tok.is(tok::l_brace)) {
857         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
858         addUnwrappedLine();
859         return;
860       }
861     }
862     break;
863   case tok::kw_export:
864     if (Style.Language == FormatStyle::LK_JavaScript) {
865       parseJavaScriptEs6ImportExport();
866       return;
867     }
868     break;
869   case tok::identifier:
870     if (FormatTok->is(TT_ForEachMacro)) {
871       parseForOrWhileLoop();
872       return;
873     }
874     if (FormatTok->is(TT_MacroBlockBegin)) {
875       parseBlock(/*MustBeDeclaration=*/false, /*AddLevel=*/true,
876                  /*MunchSemi=*/false);
877       return;
878     }
879     if (Style.Language == FormatStyle::LK_JavaScript &&
880         FormatTok->is(Keywords.kw_import)) {
881       parseJavaScriptEs6ImportExport();
882       return;
883     }
884     if (FormatTok->isOneOf(Keywords.kw_signals, Keywords.kw_qsignals,
885                            Keywords.kw_slots, Keywords.kw_qslots)) {
886       nextToken();
887       if (FormatTok->is(tok::colon)) {
888         nextToken();
889         addUnwrappedLine();
890       }
891       return;
892     }
893     // In all other cases, parse the declaration.
894     break;
895   default:
896     break;
897   }
898   do {
899     switch (FormatTok->Tok.getKind()) {
900     case tok::at:
901       nextToken();
902       if (FormatTok->Tok.is(tok::l_brace))
903         parseBracedList();
904       break;
905     case tok::kw_enum:
906       // parseEnum falls through and does not yet add an unwrapped line as an
907       // enum definition can start a structural element.
908       if (!parseEnum())
909         break;
910       // This only applies for C++.
911       if (Style.Language != FormatStyle::LK_Cpp) {
912         addUnwrappedLine();
913         return;
914       }
915       break;
916     case tok::kw_typedef:
917       nextToken();
918       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
919                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
920         parseEnum();
921       break;
922     case tok::kw_struct:
923     case tok::kw_union:
924     case tok::kw_class:
925       // parseRecord falls through and does not yet add an unwrapped line as a
926       // record declaration or definition can start a structural element.
927       parseRecord();
928       // This does not apply for Java and JavaScript.
929       if (Style.Language == FormatStyle::LK_Java ||
930           Style.Language == FormatStyle::LK_JavaScript) {
931         if (FormatTok->is(tok::semi))
932           nextToken();
933         addUnwrappedLine();
934         return;
935       }
936       break;
937     case tok::period:
938       nextToken();
939       // In Java, classes have an implicit static member "class".
940       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
941           FormatTok->is(tok::kw_class))
942         nextToken();
943       if (Style.Language == FormatStyle::LK_JavaScript && FormatTok &&
944           FormatTok->Tok.getIdentifierInfo())
945         // JavaScript only has pseudo keywords, all keywords are allowed to
946         // appear in "IdentifierName" positions. See http://es5.github.io/#x7.6
947         nextToken();
948       break;
949     case tok::semi:
950       nextToken();
951       addUnwrappedLine();
952       return;
953     case tok::r_brace:
954       addUnwrappedLine();
955       return;
956     case tok::l_paren:
957       parseParens();
958       break;
959     case tok::kw_operator:
960       nextToken();
961       if (FormatTok->isBinaryOperator())
962         nextToken();
963       break;
964     case tok::caret:
965       nextToken();
966       if (FormatTok->Tok.isAnyIdentifier() ||
967           FormatTok->isSimpleTypeSpecifier())
968         nextToken();
969       if (FormatTok->is(tok::l_paren))
970         parseParens();
971       if (FormatTok->is(tok::l_brace))
972         parseChildBlock();
973       break;
974     case tok::l_brace:
975       if (!tryToParseBracedList()) {
976         // A block outside of parentheses must be the last part of a
977         // structural element.
978         // FIXME: Figure out cases where this is not true, and add projections
979         // for them (the one we know is missing are lambdas).
980         if (Style.BraceWrapping.AfterFunction)
981           addUnwrappedLine();
982         FormatTok->Type = TT_FunctionLBrace;
983         parseBlock(/*MustBeDeclaration=*/false);
984         addUnwrappedLine();
985         return;
986       }
987       // Otherwise this was a braced init list, and the structural
988       // element continues.
989       break;
990     case tok::kw_try:
991       // We arrive here when parsing function-try blocks.
992       parseTryCatch();
993       return;
994     case tok::identifier: {
995       if (FormatTok->is(TT_MacroBlockEnd)) {
996         addUnwrappedLine();
997         return;
998       }
999 
1000       // Parse function literal unless 'function' is the first token in a line
1001       // in which case this should be treated as a free-standing function.
1002       if (Style.Language == FormatStyle::LK_JavaScript &&
1003           FormatTok->is(Keywords.kw_function) && Line->Tokens.size() > 0) {
1004         tryToParseJSFunction();
1005         break;
1006       }
1007       if ((Style.Language == FormatStyle::LK_JavaScript ||
1008            Style.Language == FormatStyle::LK_Java) &&
1009           FormatTok->is(Keywords.kw_interface)) {
1010         parseRecord();
1011         addUnwrappedLine();
1012         return;
1013       }
1014 
1015       // See if the following token should start a new unwrapped line.
1016       StringRef Text = FormatTok->TokenText;
1017       nextToken();
1018       if (Line->Tokens.size() == 1 &&
1019           // JS doesn't have macros, and within classes colons indicate fields,
1020           // not labels.
1021           Style.Language != FormatStyle::LK_JavaScript) {
1022         if (FormatTok->Tok.is(tok::colon) && !Line->MustBeDeclaration) {
1023           Line->Tokens.begin()->Tok->MustBreakBefore = true;
1024           parseLabel();
1025           return;
1026         }
1027         // Recognize function-like macro usages without trailing semicolon as
1028         // well as free-standing macros like Q_OBJECT.
1029         bool FunctionLike = FormatTok->is(tok::l_paren);
1030         if (FunctionLike)
1031           parseParens();
1032 
1033         bool FollowedByNewline =
1034             CommentsBeforeNextToken.empty()
1035                 ? FormatTok->NewlinesBefore > 0
1036                 : CommentsBeforeNextToken.front()->NewlinesBefore > 0;
1037 
1038         if (FollowedByNewline && (Text.size() >= 5 || FunctionLike) &&
1039             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
1040           addUnwrappedLine();
1041           return;
1042         }
1043       }
1044       break;
1045     }
1046     case tok::equal:
1047       // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
1048       // TT_JsFatArrow. The always start an expression or a child block if
1049       // followed by a curly.
1050       if (FormatTok->is(TT_JsFatArrow)) {
1051         nextToken();
1052         if (FormatTok->is(tok::l_brace))
1053           parseChildBlock();
1054         break;
1055       }
1056 
1057       nextToken();
1058       if (FormatTok->Tok.is(tok::l_brace)) {
1059         parseBracedList();
1060       }
1061       break;
1062     case tok::l_square:
1063       parseSquare();
1064       break;
1065     case tok::kw_new:
1066       parseNew();
1067       break;
1068     default:
1069       nextToken();
1070       break;
1071     }
1072   } while (!eof());
1073 }
1074 
1075 bool UnwrappedLineParser::tryToParseLambda() {
1076   if (Style.Language != FormatStyle::LK_Cpp) {
1077     nextToken();
1078     return false;
1079   }
1080   const FormatToken* Previous = getPreviousToken();
1081   if (Previous &&
1082       (Previous->isOneOf(tok::identifier, tok::kw_operator, tok::kw_new,
1083                          tok::kw_delete) ||
1084        Previous->closesScope() || Previous->isSimpleTypeSpecifier())) {
1085     nextToken();
1086     return false;
1087   }
1088   assert(FormatTok->is(tok::l_square));
1089   FormatToken &LSquare = *FormatTok;
1090   if (!tryToParseLambdaIntroducer())
1091     return false;
1092 
1093   while (FormatTok->isNot(tok::l_brace)) {
1094     if (FormatTok->isSimpleTypeSpecifier()) {
1095       nextToken();
1096       continue;
1097     }
1098     switch (FormatTok->Tok.getKind()) {
1099     case tok::l_brace:
1100       break;
1101     case tok::l_paren:
1102       parseParens();
1103       break;
1104     case tok::amp:
1105     case tok::star:
1106     case tok::kw_const:
1107     case tok::comma:
1108     case tok::less:
1109     case tok::greater:
1110     case tok::identifier:
1111     case tok::numeric_constant:
1112     case tok::coloncolon:
1113     case tok::kw_mutable:
1114       nextToken();
1115       break;
1116     case tok::arrow:
1117       FormatTok->Type = TT_LambdaArrow;
1118       nextToken();
1119       break;
1120     default:
1121       return true;
1122     }
1123   }
1124   LSquare.Type = TT_LambdaLSquare;
1125   parseChildBlock();
1126   return true;
1127 }
1128 
1129 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
1130   nextToken();
1131   if (FormatTok->is(tok::equal)) {
1132     nextToken();
1133     if (FormatTok->is(tok::r_square)) {
1134       nextToken();
1135       return true;
1136     }
1137     if (FormatTok->isNot(tok::comma))
1138       return false;
1139     nextToken();
1140   } else if (FormatTok->is(tok::amp)) {
1141     nextToken();
1142     if (FormatTok->is(tok::r_square)) {
1143       nextToken();
1144       return true;
1145     }
1146     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
1147       return false;
1148     }
1149     if (FormatTok->is(tok::comma))
1150       nextToken();
1151   } else if (FormatTok->is(tok::r_square)) {
1152     nextToken();
1153     return true;
1154   }
1155   do {
1156     if (FormatTok->is(tok::amp))
1157       nextToken();
1158     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
1159       return false;
1160     nextToken();
1161     if (FormatTok->is(tok::ellipsis))
1162       nextToken();
1163     if (FormatTok->is(tok::comma)) {
1164       nextToken();
1165     } else if (FormatTok->is(tok::r_square)) {
1166       nextToken();
1167       return true;
1168     } else {
1169       return false;
1170     }
1171   } while (!eof());
1172   return false;
1173 }
1174 
1175 void UnwrappedLineParser::tryToParseJSFunction() {
1176   nextToken();
1177 
1178   // Consume function name.
1179   if (FormatTok->is(tok::identifier))
1180     nextToken();
1181 
1182   if (FormatTok->isNot(tok::l_paren))
1183     return;
1184 
1185   // Parse formal parameter list.
1186   parseParens();
1187 
1188   if (FormatTok->is(tok::colon)) {
1189     // Parse a type definition.
1190     nextToken();
1191 
1192     // Eat the type declaration. For braced inline object types, balance braces,
1193     // otherwise just parse until finding an l_brace for the function body.
1194     if (FormatTok->is(tok::l_brace))
1195       tryToParseBracedList();
1196     else
1197       while (FormatTok->isNot(tok::l_brace) && !eof())
1198         nextToken();
1199   }
1200 
1201   parseChildBlock();
1202 }
1203 
1204 bool UnwrappedLineParser::tryToParseBracedList() {
1205   if (FormatTok->BlockKind == BK_Unknown)
1206     calculateBraceTypes();
1207   assert(FormatTok->BlockKind != BK_Unknown);
1208   if (FormatTok->BlockKind == BK_Block)
1209     return false;
1210   parseBracedList();
1211   return true;
1212 }
1213 
1214 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
1215   bool HasError = false;
1216   nextToken();
1217 
1218   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
1219   // replace this by using parseAssigmentExpression() inside.
1220   do {
1221     if (Style.Language == FormatStyle::LK_JavaScript) {
1222       if (FormatTok->is(Keywords.kw_function)) {
1223         tryToParseJSFunction();
1224         continue;
1225       }
1226       if (FormatTok->is(TT_JsFatArrow)) {
1227         nextToken();
1228         // Fat arrows can be followed by simple expressions or by child blocks
1229         // in curly braces.
1230         if (FormatTok->is(tok::l_brace)) {
1231           parseChildBlock();
1232           continue;
1233         }
1234       }
1235     }
1236     switch (FormatTok->Tok.getKind()) {
1237     case tok::caret:
1238       nextToken();
1239       if (FormatTok->is(tok::l_brace)) {
1240         parseChildBlock();
1241       }
1242       break;
1243     case tok::l_square:
1244       tryToParseLambda();
1245       break;
1246     case tok::l_brace:
1247       // Assume there are no blocks inside a braced init list apart
1248       // from the ones we explicitly parse out (like lambdas).
1249       FormatTok->BlockKind = BK_BracedInit;
1250       parseBracedList();
1251       break;
1252     case tok::l_paren:
1253       parseParens();
1254       // JavaScript can just have free standing methods and getters/setters in
1255       // object literals. Detect them by a "{" following ")".
1256       if (Style.Language == FormatStyle::LK_JavaScript) {
1257         if (FormatTok->is(tok::l_brace))
1258           parseChildBlock();
1259         break;
1260       }
1261       break;
1262     case tok::r_brace:
1263       nextToken();
1264       return !HasError;
1265     case tok::semi:
1266       // JavaScript (or more precisely TypeScript) can have semicolons in braced
1267       // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
1268       // used for error recovery if we have otherwise determined that this is
1269       // a braced list.
1270       if (Style.Language == FormatStyle::LK_JavaScript) {
1271         nextToken();
1272         break;
1273       }
1274       HasError = true;
1275       if (!ContinueOnSemicolons)
1276         return !HasError;
1277       nextToken();
1278       break;
1279     case tok::comma:
1280       nextToken();
1281       break;
1282     default:
1283       nextToken();
1284       break;
1285     }
1286   } while (!eof());
1287   return false;
1288 }
1289 
1290 void UnwrappedLineParser::parseParens() {
1291   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1292   nextToken();
1293   do {
1294     switch (FormatTok->Tok.getKind()) {
1295     case tok::l_paren:
1296       parseParens();
1297       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1298         parseChildBlock();
1299       break;
1300     case tok::r_paren:
1301       nextToken();
1302       return;
1303     case tok::r_brace:
1304       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1305       return;
1306     case tok::l_square:
1307       tryToParseLambda();
1308       break;
1309     case tok::l_brace:
1310       if (!tryToParseBracedList())
1311         parseChildBlock();
1312       break;
1313     case tok::at:
1314       nextToken();
1315       if (FormatTok->Tok.is(tok::l_brace))
1316         parseBracedList();
1317       break;
1318     case tok::identifier:
1319       if (Style.Language == FormatStyle::LK_JavaScript &&
1320           FormatTok->is(Keywords.kw_function))
1321         tryToParseJSFunction();
1322       else
1323         nextToken();
1324       break;
1325     default:
1326       nextToken();
1327       break;
1328     }
1329   } while (!eof());
1330 }
1331 
1332 void UnwrappedLineParser::parseSquare() {
1333   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1334   if (tryToParseLambda())
1335     return;
1336   do {
1337     switch (FormatTok->Tok.getKind()) {
1338     case tok::l_paren:
1339       parseParens();
1340       break;
1341     case tok::r_square:
1342       nextToken();
1343       return;
1344     case tok::r_brace:
1345       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1346       return;
1347     case tok::l_square:
1348       parseSquare();
1349       break;
1350     case tok::l_brace: {
1351       if (!tryToParseBracedList())
1352         parseChildBlock();
1353       break;
1354     }
1355     case tok::at:
1356       nextToken();
1357       if (FormatTok->Tok.is(tok::l_brace))
1358         parseBracedList();
1359       break;
1360     default:
1361       nextToken();
1362       break;
1363     }
1364   } while (!eof());
1365 }
1366 
1367 void UnwrappedLineParser::parseIfThenElse() {
1368   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1369   nextToken();
1370   if (FormatTok->Tok.is(tok::l_paren))
1371     parseParens();
1372   bool NeedsUnwrappedLine = false;
1373   if (FormatTok->Tok.is(tok::l_brace)) {
1374     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1375     parseBlock(/*MustBeDeclaration=*/false);
1376     if (Style.BraceWrapping.BeforeElse)
1377       addUnwrappedLine();
1378     else
1379       NeedsUnwrappedLine = true;
1380   } else {
1381     addUnwrappedLine();
1382     ++Line->Level;
1383     parseStructuralElement();
1384     --Line->Level;
1385   }
1386   if (FormatTok->Tok.is(tok::kw_else)) {
1387     nextToken();
1388     if (FormatTok->Tok.is(tok::l_brace)) {
1389       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1390       parseBlock(/*MustBeDeclaration=*/false);
1391       addUnwrappedLine();
1392     } else if (FormatTok->Tok.is(tok::kw_if)) {
1393       parseIfThenElse();
1394     } else {
1395       addUnwrappedLine();
1396       ++Line->Level;
1397       parseStructuralElement();
1398       --Line->Level;
1399     }
1400   } else if (NeedsUnwrappedLine) {
1401     addUnwrappedLine();
1402   }
1403 }
1404 
1405 void UnwrappedLineParser::parseTryCatch() {
1406   assert(FormatTok->isOneOf(tok::kw_try, tok::kw___try) && "'try' expected");
1407   nextToken();
1408   bool NeedsUnwrappedLine = false;
1409   if (FormatTok->is(tok::colon)) {
1410     // We are in a function try block, what comes is an initializer list.
1411     nextToken();
1412     while (FormatTok->is(tok::identifier)) {
1413       nextToken();
1414       if (FormatTok->is(tok::l_paren))
1415         parseParens();
1416       if (FormatTok->is(tok::comma))
1417         nextToken();
1418     }
1419   }
1420   // Parse try with resource.
1421   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1422     parseParens();
1423   }
1424   if (FormatTok->is(tok::l_brace)) {
1425     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1426     parseBlock(/*MustBeDeclaration=*/false);
1427     if (Style.BraceWrapping.BeforeCatch) {
1428       addUnwrappedLine();
1429     } else {
1430       NeedsUnwrappedLine = true;
1431     }
1432   } else if (!FormatTok->is(tok::kw_catch)) {
1433     // The C++ standard requires a compound-statement after a try.
1434     // If there's none, we try to assume there's a structuralElement
1435     // and try to continue.
1436     addUnwrappedLine();
1437     ++Line->Level;
1438     parseStructuralElement();
1439     --Line->Level;
1440   }
1441   while (1) {
1442     if (FormatTok->is(tok::at))
1443       nextToken();
1444     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
1445                              tok::kw___finally) ||
1446           ((Style.Language == FormatStyle::LK_Java ||
1447             Style.Language == FormatStyle::LK_JavaScript) &&
1448            FormatTok->is(Keywords.kw_finally)) ||
1449           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
1450            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
1451       break;
1452     nextToken();
1453     while (FormatTok->isNot(tok::l_brace)) {
1454       if (FormatTok->is(tok::l_paren)) {
1455         parseParens();
1456         continue;
1457       }
1458       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1459         return;
1460       nextToken();
1461     }
1462     NeedsUnwrappedLine = false;
1463     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1464     parseBlock(/*MustBeDeclaration=*/false);
1465     if (Style.BraceWrapping.BeforeCatch)
1466       addUnwrappedLine();
1467     else
1468       NeedsUnwrappedLine = true;
1469   }
1470   if (NeedsUnwrappedLine)
1471     addUnwrappedLine();
1472 }
1473 
1474 void UnwrappedLineParser::parseNamespace() {
1475   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1476 
1477   const FormatToken &InitialToken = *FormatTok;
1478   nextToken();
1479   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon))
1480     nextToken();
1481   if (FormatTok->Tok.is(tok::l_brace)) {
1482     if (ShouldBreakBeforeBrace(Style, InitialToken))
1483       addUnwrappedLine();
1484 
1485     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1486                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1487                      DeclarationScopeStack.size() > 1);
1488     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1489     // Munch the semicolon after a namespace. This is more common than one would
1490     // think. Puttin the semicolon into its own line is very ugly.
1491     if (FormatTok->Tok.is(tok::semi))
1492       nextToken();
1493     addUnwrappedLine();
1494   }
1495   // FIXME: Add error handling.
1496 }
1497 
1498 void UnwrappedLineParser::parseNew() {
1499   assert(FormatTok->is(tok::kw_new) && "'new' expected");
1500   nextToken();
1501   if (Style.Language != FormatStyle::LK_Java)
1502     return;
1503 
1504   // In Java, we can parse everything up to the parens, which aren't optional.
1505   do {
1506     // There should not be a ;, { or } before the new's open paren.
1507     if (FormatTok->isOneOf(tok::semi, tok::l_brace, tok::r_brace))
1508       return;
1509 
1510     // Consume the parens.
1511     if (FormatTok->is(tok::l_paren)) {
1512       parseParens();
1513 
1514       // If there is a class body of an anonymous class, consume that as child.
1515       if (FormatTok->is(tok::l_brace))
1516         parseChildBlock();
1517       return;
1518     }
1519     nextToken();
1520   } while (!eof());
1521 }
1522 
1523 void UnwrappedLineParser::parseForOrWhileLoop() {
1524   assert(FormatTok->isOneOf(tok::kw_for, tok::kw_while, TT_ForEachMacro) &&
1525          "'for', 'while' or foreach macro expected");
1526   nextToken();
1527   if (FormatTok->Tok.is(tok::l_paren))
1528     parseParens();
1529   if (FormatTok->Tok.is(tok::l_brace)) {
1530     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1531     parseBlock(/*MustBeDeclaration=*/false);
1532     addUnwrappedLine();
1533   } else {
1534     addUnwrappedLine();
1535     ++Line->Level;
1536     parseStructuralElement();
1537     --Line->Level;
1538   }
1539 }
1540 
1541 void UnwrappedLineParser::parseDoWhile() {
1542   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1543   nextToken();
1544   if (FormatTok->Tok.is(tok::l_brace)) {
1545     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1546     parseBlock(/*MustBeDeclaration=*/false);
1547     if (Style.BraceWrapping.IndentBraces)
1548       addUnwrappedLine();
1549   } else {
1550     addUnwrappedLine();
1551     ++Line->Level;
1552     parseStructuralElement();
1553     --Line->Level;
1554   }
1555 
1556   // FIXME: Add error handling.
1557   if (!FormatTok->Tok.is(tok::kw_while)) {
1558     addUnwrappedLine();
1559     return;
1560   }
1561 
1562   nextToken();
1563   parseStructuralElement();
1564 }
1565 
1566 void UnwrappedLineParser::parseLabel() {
1567   nextToken();
1568   unsigned OldLineLevel = Line->Level;
1569   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1570     --Line->Level;
1571   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1572     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1573     parseBlock(/*MustBeDeclaration=*/false);
1574     if (FormatTok->Tok.is(tok::kw_break)) {
1575       if (Style.BraceWrapping.AfterControlStatement)
1576         addUnwrappedLine();
1577       parseStructuralElement();
1578     }
1579     addUnwrappedLine();
1580   } else {
1581     if (FormatTok->is(tok::semi))
1582       nextToken();
1583     addUnwrappedLine();
1584   }
1585   Line->Level = OldLineLevel;
1586   if (FormatTok->isNot(tok::l_brace)) {
1587     parseStructuralElement();
1588     addUnwrappedLine();
1589   }
1590 }
1591 
1592 void UnwrappedLineParser::parseCaseLabel() {
1593   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1594   // FIXME: fix handling of complex expressions here.
1595   do {
1596     nextToken();
1597   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1598   parseLabel();
1599 }
1600 
1601 void UnwrappedLineParser::parseSwitch() {
1602   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1603   nextToken();
1604   if (FormatTok->Tok.is(tok::l_paren))
1605     parseParens();
1606   if (FormatTok->Tok.is(tok::l_brace)) {
1607     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1608     parseBlock(/*MustBeDeclaration=*/false);
1609     addUnwrappedLine();
1610   } else {
1611     addUnwrappedLine();
1612     ++Line->Level;
1613     parseStructuralElement();
1614     --Line->Level;
1615   }
1616 }
1617 
1618 void UnwrappedLineParser::parseAccessSpecifier() {
1619   nextToken();
1620   // Understand Qt's slots.
1621   if (FormatTok->isOneOf(Keywords.kw_slots, Keywords.kw_qslots))
1622     nextToken();
1623   // Otherwise, we don't know what it is, and we'd better keep the next token.
1624   if (FormatTok->Tok.is(tok::colon))
1625     nextToken();
1626   addUnwrappedLine();
1627 }
1628 
1629 bool UnwrappedLineParser::parseEnum() {
1630   // Won't be 'enum' for NS_ENUMs.
1631   if (FormatTok->Tok.is(tok::kw_enum))
1632     nextToken();
1633 
1634   // In TypeScript, "enum" can also be used as property name, e.g. in interface
1635   // declarations. An "enum" keyword followed by a colon would be a syntax
1636   // error and thus assume it is just an identifier.
1637   if (Style.Language == FormatStyle::LK_JavaScript &&
1638       FormatTok->isOneOf(tok::colon, tok::question))
1639     return false;
1640 
1641   // Eat up enum class ...
1642   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1643     nextToken();
1644 
1645   while (FormatTok->Tok.getIdentifierInfo() ||
1646          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1647                             tok::greater, tok::comma, tok::question)) {
1648     nextToken();
1649     // We can have macros or attributes in between 'enum' and the enum name.
1650     if (FormatTok->is(tok::l_paren))
1651       parseParens();
1652     if (FormatTok->is(tok::identifier)) {
1653       nextToken();
1654       // If there are two identifiers in a row, this is likely an elaborate
1655       // return type. In Java, this can be "implements", etc.
1656       if (Style.Language == FormatStyle::LK_Cpp &&
1657           FormatTok->is(tok::identifier))
1658         return false;
1659     }
1660   }
1661 
1662   // Just a declaration or something is wrong.
1663   if (FormatTok->isNot(tok::l_brace))
1664     return true;
1665   FormatTok->BlockKind = BK_Block;
1666 
1667   if (Style.Language == FormatStyle::LK_Java) {
1668     // Java enums are different.
1669     parseJavaEnumBody();
1670     return true;
1671   }
1672   if (Style.Language == FormatStyle::LK_Proto) {
1673     parseBlock(/*MustBeDeclaration=*/true);
1674     return true;
1675   }
1676 
1677   // Parse enum body.
1678   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1679   if (HasError) {
1680     if (FormatTok->is(tok::semi))
1681       nextToken();
1682     addUnwrappedLine();
1683   }
1684   return true;
1685 
1686   // There is no addUnwrappedLine() here so that we fall through to parsing a
1687   // structural element afterwards. Thus, in "enum A {} n, m;",
1688   // "} n, m;" will end up in one unwrapped line.
1689 }
1690 
1691 void UnwrappedLineParser::parseJavaEnumBody() {
1692   // Determine whether the enum is simple, i.e. does not have a semicolon or
1693   // constants with class bodies. Simple enums can be formatted like braced
1694   // lists, contracted to a single line, etc.
1695   unsigned StoredPosition = Tokens->getPosition();
1696   bool IsSimple = true;
1697   FormatToken *Tok = Tokens->getNextToken();
1698   while (Tok) {
1699     if (Tok->is(tok::r_brace))
1700       break;
1701     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1702       IsSimple = false;
1703       break;
1704     }
1705     // FIXME: This will also mark enums with braces in the arguments to enum
1706     // constants as "not simple". This is probably fine in practice, though.
1707     Tok = Tokens->getNextToken();
1708   }
1709   FormatTok = Tokens->setPosition(StoredPosition);
1710 
1711   if (IsSimple) {
1712     parseBracedList();
1713     addUnwrappedLine();
1714     return;
1715   }
1716 
1717   // Parse the body of a more complex enum.
1718   // First add a line for everything up to the "{".
1719   nextToken();
1720   addUnwrappedLine();
1721   ++Line->Level;
1722 
1723   // Parse the enum constants.
1724   while (FormatTok) {
1725     if (FormatTok->is(tok::l_brace)) {
1726       // Parse the constant's class body.
1727       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1728                  /*MunchSemi=*/false);
1729     } else if (FormatTok->is(tok::l_paren)) {
1730       parseParens();
1731     } else if (FormatTok->is(tok::comma)) {
1732       nextToken();
1733       addUnwrappedLine();
1734     } else if (FormatTok->is(tok::semi)) {
1735       nextToken();
1736       addUnwrappedLine();
1737       break;
1738     } else if (FormatTok->is(tok::r_brace)) {
1739       addUnwrappedLine();
1740       break;
1741     } else {
1742       nextToken();
1743     }
1744   }
1745 
1746   // Parse the class body after the enum's ";" if any.
1747   parseLevel(/*HasOpeningBrace=*/true);
1748   nextToken();
1749   --Line->Level;
1750   addUnwrappedLine();
1751 }
1752 
1753 void UnwrappedLineParser::parseRecord() {
1754   const FormatToken &InitialToken = *FormatTok;
1755   nextToken();
1756 
1757   // The actual identifier can be a nested name specifier, and in macros
1758   // it is often token-pasted.
1759   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::hashhash,
1760                             tok::kw___attribute, tok::kw___declspec,
1761                             tok::kw_alignas) ||
1762          ((Style.Language == FormatStyle::LK_Java ||
1763            Style.Language == FormatStyle::LK_JavaScript) &&
1764           FormatTok->isOneOf(tok::period, tok::comma))) {
1765     bool IsNonMacroIdentifier =
1766         FormatTok->is(tok::identifier) &&
1767         FormatTok->TokenText != FormatTok->TokenText.upper();
1768     nextToken();
1769     // We can have macros or attributes in between 'class' and the class name.
1770     if (!IsNonMacroIdentifier && FormatTok->Tok.is(tok::l_paren))
1771       parseParens();
1772   }
1773 
1774   // Note that parsing away template declarations here leads to incorrectly
1775   // accepting function declarations as record declarations.
1776   // In general, we cannot solve this problem. Consider:
1777   // class A<int> B() {}
1778   // which can be a function definition or a class definition when B() is a
1779   // macro. If we find enough real-world cases where this is a problem, we
1780   // can parse for the 'template' keyword in the beginning of the statement,
1781   // and thus rule out the record production in case there is no template
1782   // (this would still leave us with an ambiguity between template function
1783   // and class declarations).
1784   if (FormatTok->isOneOf(tok::colon, tok::less)) {
1785     while (!eof()) {
1786       if (FormatTok->is(tok::l_brace)) {
1787         calculateBraceTypes(/*ExpectClassBody=*/true);
1788         if (!tryToParseBracedList())
1789           break;
1790       }
1791       if (FormatTok->Tok.is(tok::semi))
1792         return;
1793       nextToken();
1794     }
1795   }
1796   if (FormatTok->Tok.is(tok::l_brace)) {
1797     if (ShouldBreakBeforeBrace(Style, InitialToken))
1798       addUnwrappedLine();
1799 
1800     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1801                /*MunchSemi=*/false);
1802   }
1803   // There is no addUnwrappedLine() here so that we fall through to parsing a
1804   // structural element afterwards. Thus, in "class A {} n, m;",
1805   // "} n, m;" will end up in one unwrapped line.
1806 }
1807 
1808 void UnwrappedLineParser::parseObjCProtocolList() {
1809   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1810   do
1811     nextToken();
1812   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1813   nextToken(); // Skip '>'.
1814 }
1815 
1816 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1817   do {
1818     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1819       nextToken();
1820       addUnwrappedLine();
1821       break;
1822     }
1823     if (FormatTok->is(tok::l_brace)) {
1824       parseBlock(/*MustBeDeclaration=*/false);
1825       // In ObjC interfaces, nothing should be following the "}".
1826       addUnwrappedLine();
1827     } else if (FormatTok->is(tok::r_brace)) {
1828       // Ignore stray "}". parseStructuralElement doesn't consume them.
1829       nextToken();
1830       addUnwrappedLine();
1831     } else {
1832       parseStructuralElement();
1833     }
1834   } while (!eof());
1835 }
1836 
1837 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1838   nextToken();
1839   nextToken(); // interface name
1840 
1841   // @interface can be followed by either a base class, or a category.
1842   if (FormatTok->Tok.is(tok::colon)) {
1843     nextToken();
1844     nextToken(); // base class name
1845   } else if (FormatTok->Tok.is(tok::l_paren))
1846     // Skip category, if present.
1847     parseParens();
1848 
1849   if (FormatTok->Tok.is(tok::less))
1850     parseObjCProtocolList();
1851 
1852   if (FormatTok->Tok.is(tok::l_brace)) {
1853     if (Style.BraceWrapping.AfterObjCDeclaration)
1854       addUnwrappedLine();
1855     parseBlock(/*MustBeDeclaration=*/true);
1856   }
1857 
1858   // With instance variables, this puts '}' on its own line.  Without instance
1859   // variables, this ends the @interface line.
1860   addUnwrappedLine();
1861 
1862   parseObjCUntilAtEnd();
1863 }
1864 
1865 void UnwrappedLineParser::parseObjCProtocol() {
1866   nextToken();
1867   nextToken(); // protocol name
1868 
1869   if (FormatTok->Tok.is(tok::less))
1870     parseObjCProtocolList();
1871 
1872   // Check for protocol declaration.
1873   if (FormatTok->Tok.is(tok::semi)) {
1874     nextToken();
1875     return addUnwrappedLine();
1876   }
1877 
1878   addUnwrappedLine();
1879   parseObjCUntilAtEnd();
1880 }
1881 
1882 void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
1883   assert(FormatTok->isOneOf(Keywords.kw_import, tok::kw_export));
1884   nextToken();
1885 
1886   // Consume the "default" in "export default class/function".
1887   if (FormatTok->is(tok::kw_default))
1888     nextToken();
1889 
1890   // Consume "function" and "default function", so that these get parsed as
1891   // free-standing JS functions, i.e. do not require a trailing semicolon.
1892   if (FormatTok->is(Keywords.kw_function)) {
1893     nextToken();
1894     return;
1895   }
1896 
1897   // Consume the "abstract" in "export abstract class".
1898   if (FormatTok->is(Keywords.kw_abstract))
1899     nextToken();
1900 
1901   if (FormatTok->isOneOf(tok::kw_const, tok::kw_class, tok::kw_enum,
1902                          Keywords.kw_interface, Keywords.kw_let,
1903                          Keywords.kw_var))
1904     return; // Fall through to parsing the corresponding structure.
1905 
1906   while (!eof() && FormatTok->isNot(tok::semi)) {
1907     if (FormatTok->is(tok::l_brace)) {
1908       FormatTok->BlockKind = BK_Block;
1909       parseBracedList();
1910     } else {
1911       nextToken();
1912     }
1913   }
1914 }
1915 
1916 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1917                                                  StringRef Prefix = "") {
1918   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1919                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1920   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1921                                                     E = Line.Tokens.end();
1922        I != E; ++I) {
1923     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1924   }
1925   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1926                                                     E = Line.Tokens.end();
1927        I != E; ++I) {
1928     const UnwrappedLineNode &Node = *I;
1929     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1930              I = Node.Children.begin(),
1931              E = Node.Children.end();
1932          I != E; ++I) {
1933       printDebugInfo(*I, "\nChild: ");
1934     }
1935   }
1936   llvm::dbgs() << "\n";
1937 }
1938 
1939 void UnwrappedLineParser::addUnwrappedLine() {
1940   if (Line->Tokens.empty())
1941     return;
1942   DEBUG({
1943     if (CurrentLines == &Lines)
1944       printDebugInfo(*Line);
1945   });
1946   CurrentLines->push_back(std::move(*Line));
1947   Line->Tokens.clear();
1948   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1949     CurrentLines->append(
1950         std::make_move_iterator(PreprocessorDirectives.begin()),
1951         std::make_move_iterator(PreprocessorDirectives.end()));
1952     PreprocessorDirectives.clear();
1953   }
1954 }
1955 
1956 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1957 
1958 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
1959   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1960          FormatTok.NewlinesBefore > 0;
1961 }
1962 
1963 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1964   bool JustComments = Line->Tokens.empty();
1965   for (SmallVectorImpl<FormatToken *>::const_iterator
1966            I = CommentsBeforeNextToken.begin(),
1967            E = CommentsBeforeNextToken.end();
1968        I != E; ++I) {
1969     if (isOnNewLine(**I) && JustComments)
1970       addUnwrappedLine();
1971     pushToken(*I);
1972   }
1973   if (NewlineBeforeNext && JustComments)
1974     addUnwrappedLine();
1975   CommentsBeforeNextToken.clear();
1976 }
1977 
1978 void UnwrappedLineParser::nextToken() {
1979   if (eof())
1980     return;
1981   flushComments(isOnNewLine(*FormatTok));
1982   pushToken(FormatTok);
1983   if (Style.Language != FormatStyle::LK_JavaScript)
1984     readToken();
1985   else
1986     readTokenWithJavaScriptASI();
1987 }
1988 
1989 const FormatToken *UnwrappedLineParser::getPreviousToken() {
1990   // FIXME: This is a dirty way to access the previous token. Find a better
1991   // solution.
1992   if (!Line || Line->Tokens.empty())
1993     return nullptr;
1994   return Line->Tokens.back().Tok;
1995 }
1996 
1997 void UnwrappedLineParser::readToken() {
1998   bool CommentsInCurrentLine = true;
1999   do {
2000     FormatTok = Tokens->getNextToken();
2001     assert(FormatTok);
2002     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
2003            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
2004       // If there is an unfinished unwrapped line, we flush the preprocessor
2005       // directives only after that unwrapped line was finished later.
2006       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
2007       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
2008       // Comments stored before the preprocessor directive need to be output
2009       // before the preprocessor directive, at the same level as the
2010       // preprocessor directive, as we consider them to apply to the directive.
2011       flushComments(isOnNewLine(*FormatTok));
2012       parsePPDirective();
2013     }
2014     while (FormatTok->Type == TT_ConflictStart ||
2015            FormatTok->Type == TT_ConflictEnd ||
2016            FormatTok->Type == TT_ConflictAlternative) {
2017       if (FormatTok->Type == TT_ConflictStart) {
2018         conditionalCompilationStart(/*Unreachable=*/false);
2019       } else if (FormatTok->Type == TT_ConflictAlternative) {
2020         conditionalCompilationAlternative();
2021       } else if (FormatTok->Type == TT_ConflictEnd) {
2022         conditionalCompilationEnd();
2023       }
2024       FormatTok = Tokens->getNextToken();
2025       FormatTok->MustBreakBefore = true;
2026     }
2027 
2028     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
2029         !Line->InPPDirective) {
2030       continue;
2031     }
2032 
2033     if (!FormatTok->Tok.is(tok::comment))
2034       return;
2035     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
2036       CommentsInCurrentLine = false;
2037     }
2038     if (CommentsInCurrentLine) {
2039       pushToken(FormatTok);
2040     } else {
2041       CommentsBeforeNextToken.push_back(FormatTok);
2042     }
2043   } while (!eof());
2044 }
2045 
2046 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
2047   Line->Tokens.push_back(UnwrappedLineNode(Tok));
2048   if (MustBreakBeforeNextToken) {
2049     Line->Tokens.back().Tok->MustBreakBefore = true;
2050     MustBreakBeforeNextToken = false;
2051   }
2052 }
2053 
2054 } // end namespace format
2055 } // end namespace clang
2056