1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #define DEBUG_TYPE "format-parser"
22 
23 namespace clang {
24 namespace format {
25 
26 class FormatTokenSource {
27 public:
28   virtual ~FormatTokenSource() {}
29   virtual FormatToken *getNextToken() = 0;
30 
31   virtual unsigned getPosition() = 0;
32   virtual FormatToken *setPosition(unsigned Position) = 0;
33 };
34 
35 namespace {
36 
37 class ScopedDeclarationState {
38 public:
39   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
40                          bool MustBeDeclaration)
41       : Line(Line), Stack(Stack) {
42     Line.MustBeDeclaration = MustBeDeclaration;
43     Stack.push_back(MustBeDeclaration);
44   }
45   ~ScopedDeclarationState() {
46     Stack.pop_back();
47     if (!Stack.empty())
48       Line.MustBeDeclaration = Stack.back();
49     else
50       Line.MustBeDeclaration = true;
51   }
52 
53 private:
54   UnwrappedLine &Line;
55   std::vector<bool> &Stack;
56 };
57 
58 class ScopedMacroState : public FormatTokenSource {
59 public:
60   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
61                    FormatToken *&ResetToken)
62       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
63         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
64         Token(nullptr) {
65     TokenSource = this;
66     Line.Level = 0;
67     Line.InPPDirective = true;
68   }
69 
70   ~ScopedMacroState() override {
71     TokenSource = PreviousTokenSource;
72     ResetToken = Token;
73     Line.InPPDirective = false;
74     Line.Level = PreviousLineLevel;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113 
114   FormatToken *Token;
115 };
116 
117 } // end anonymous namespace
118 
119 class ScopedLineState {
120 public:
121   ScopedLineState(UnwrappedLineParser &Parser,
122                   bool SwitchToPreprocessorLines = false)
123       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
124     if (SwitchToPreprocessorLines)
125       Parser.CurrentLines = &Parser.PreprocessorDirectives;
126     else if (!Parser.Line->Tokens.empty())
127       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
128     PreBlockLine = std::move(Parser.Line);
129     Parser.Line = llvm::make_unique<UnwrappedLine>();
130     Parser.Line->Level = PreBlockLine->Level;
131     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
132   }
133 
134   ~ScopedLineState() {
135     if (!Parser.Line->Tokens.empty()) {
136       Parser.addUnwrappedLine();
137     }
138     assert(Parser.Line->Tokens.empty());
139     Parser.Line = std::move(PreBlockLine);
140     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
141       Parser.MustBreakBeforeNextToken = true;
142     Parser.CurrentLines = OriginalLines;
143   }
144 
145 private:
146   UnwrappedLineParser &Parser;
147 
148   std::unique_ptr<UnwrappedLine> PreBlockLine;
149   SmallVectorImpl<UnwrappedLine> *OriginalLines;
150 };
151 
152 class CompoundStatementIndenter {
153 public:
154   CompoundStatementIndenter(UnwrappedLineParser *Parser,
155                             const FormatStyle &Style, unsigned &LineLevel)
156       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
157     if (Style.BraceWrapping.AfterControlStatement)
158       Parser->addUnwrappedLine();
159     if (Style.BraceWrapping.IndentBraces)
160       ++LineLevel;
161   }
162   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
163 
164 private:
165   unsigned &LineLevel;
166   unsigned OldLineLevel;
167 };
168 
169 namespace {
170 
171 class IndexedTokenSource : public FormatTokenSource {
172 public:
173   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
174       : Tokens(Tokens), Position(-1) {}
175 
176   FormatToken *getNextToken() override {
177     ++Position;
178     return Tokens[Position];
179   }
180 
181   unsigned getPosition() override {
182     assert(Position >= 0);
183     return Position;
184   }
185 
186   FormatToken *setPosition(unsigned P) override {
187     Position = P;
188     return Tokens[Position];
189   }
190 
191   void reset() { Position = -1; }
192 
193 private:
194   ArrayRef<FormatToken *> Tokens;
195   int Position;
196 };
197 
198 } // end anonymous namespace
199 
200 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
201                                          const AdditionalKeywords &Keywords,
202                                          ArrayRef<FormatToken *> Tokens,
203                                          UnwrappedLineConsumer &Callback)
204     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
205       CurrentLines(&Lines), Style(Style), Keywords(Keywords), Tokens(nullptr),
206       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
207 
208 void UnwrappedLineParser::reset() {
209   PPBranchLevel = -1;
210   Line.reset(new UnwrappedLine);
211   CommentsBeforeNextToken.clear();
212   FormatTok = nullptr;
213   MustBreakBeforeNextToken = false;
214   PreprocessorDirectives.clear();
215   CurrentLines = &Lines;
216   DeclarationScopeStack.clear();
217   PPStack.clear();
218 }
219 
220 void UnwrappedLineParser::parse() {
221   IndexedTokenSource TokenSource(AllTokens);
222   do {
223     DEBUG(llvm::dbgs() << "----\n");
224     reset();
225     Tokens = &TokenSource;
226     TokenSource.reset();
227 
228     readToken();
229     parseFile();
230     // Create line with eof token.
231     pushToken(FormatTok);
232     addUnwrappedLine();
233 
234     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
235                                                   E = Lines.end();
236          I != E; ++I) {
237       Callback.consumeUnwrappedLine(*I);
238     }
239     Callback.finishRun();
240     Lines.clear();
241     while (!PPLevelBranchIndex.empty() &&
242            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
243       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
244       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
245     }
246     if (!PPLevelBranchIndex.empty()) {
247       ++PPLevelBranchIndex.back();
248       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
249       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
250     }
251   } while (!PPLevelBranchIndex.empty());
252 }
253 
254 void UnwrappedLineParser::parseFile() {
255   // The top-level context in a file always has declarations, except for pre-
256   // processor directives and JavaScript files.
257   bool MustBeDeclaration =
258       !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
259   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
260                                           MustBeDeclaration);
261   parseLevel(/*HasOpeningBrace=*/false);
262   // Make sure to format the remaining tokens.
263   flushComments(true);
264   addUnwrappedLine();
265 }
266 
267 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
268   bool SwitchLabelEncountered = false;
269   do {
270     tok::TokenKind kind = FormatTok->Tok.getKind();
271     if (FormatTok->Type == TT_MacroBlockBegin) {
272       kind = tok::l_brace;
273     } else if (FormatTok->Type == TT_MacroBlockEnd) {
274       kind = tok::r_brace;
275     }
276 
277     switch (kind) {
278     case tok::comment:
279       nextToken();
280       addUnwrappedLine();
281       break;
282     case tok::l_brace:
283       // FIXME: Add parameter whether this can happen - if this happens, we must
284       // be in a non-declaration context.
285       if (!FormatTok->is(TT_MacroBlockBegin) && tryToParseBracedList())
286         continue;
287       parseBlock(/*MustBeDeclaration=*/false);
288       addUnwrappedLine();
289       break;
290     case tok::r_brace:
291       if (HasOpeningBrace)
292         return;
293       nextToken();
294       addUnwrappedLine();
295       break;
296     case tok::kw_default:
297     case tok::kw_case:
298       if (!SwitchLabelEncountered &&
299           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
300         ++Line->Level;
301       SwitchLabelEncountered = true;
302       parseStructuralElement();
303       break;
304     default:
305       parseStructuralElement();
306       break;
307     }
308   } while (!eof());
309 }
310 
311 void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
312   // We'll parse forward through the tokens until we hit
313   // a closing brace or eof - note that getNextToken() will
314   // parse macros, so this will magically work inside macro
315   // definitions, too.
316   unsigned StoredPosition = Tokens->getPosition();
317   FormatToken *Tok = FormatTok;
318   const FormatToken *PrevTok = getPreviousToken();
319   // Keep a stack of positions of lbrace tokens. We will
320   // update information about whether an lbrace starts a
321   // braced init list or a different block during the loop.
322   SmallVector<FormatToken *, 8> LBraceStack;
323   assert(Tok->Tok.is(tok::l_brace));
324   do {
325     // Get next non-comment token.
326     FormatToken *NextTok;
327     unsigned ReadTokens = 0;
328     do {
329       NextTok = Tokens->getNextToken();
330       ++ReadTokens;
331     } while (NextTok->is(tok::comment));
332 
333     switch (Tok->Tok.getKind()) {
334     case tok::l_brace:
335       if (Style.Language == FormatStyle::LK_JavaScript && PrevTok &&
336           PrevTok->is(tok::colon))
337         // In TypeScript's TypeMemberLists, there can be semicolons between the
338         // individual members.
339         Tok->BlockKind = BK_BracedInit;
340       else
341         Tok->BlockKind = BK_Unknown;
342       LBraceStack.push_back(Tok);
343       break;
344     case tok::r_brace:
345       if (LBraceStack.empty())
346         break;
347       if (LBraceStack.back()->BlockKind == BK_Unknown) {
348         bool ProbablyBracedList = false;
349         if (Style.Language == FormatStyle::LK_Proto) {
350           ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
351         } else {
352           // Using OriginalColumn to distinguish between ObjC methods and
353           // binary operators is a bit hacky.
354           bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
355                                   NextTok->OriginalColumn == 0;
356 
357           // If there is a comma, semicolon or right paren after the closing
358           // brace, we assume this is a braced initializer list.  Note that
359           // regardless how we mark inner braces here, we will overwrite the
360           // BlockKind later if we parse a braced list (where all blocks
361           // inside are by default braced lists), or when we explicitly detect
362           // blocks (for example while parsing lambdas).
363           ProbablyBracedList =
364               (Style.Language == FormatStyle::LK_JavaScript &&
365                NextTok->isOneOf(Keywords.kw_of, Keywords.kw_in,
366                                 Keywords.kw_as)) ||
367               NextTok->isOneOf(tok::comma, tok::period, tok::colon,
368                                tok::r_paren, tok::r_square, tok::l_brace,
369                                tok::l_square, tok::l_paren, tok::ellipsis) ||
370               (NextTok->is(tok::identifier) &&
371                !PrevTok->isOneOf(tok::semi, tok::r_brace, tok::l_brace)) ||
372               (NextTok->is(tok::semi) &&
373                (!ExpectClassBody || LBraceStack.size() != 1)) ||
374               (NextTok->isBinaryOperator() && !NextIsObjCMethod);
375         }
376         if (ProbablyBracedList) {
377           Tok->BlockKind = BK_BracedInit;
378           LBraceStack.back()->BlockKind = BK_BracedInit;
379         } else {
380           Tok->BlockKind = BK_Block;
381           LBraceStack.back()->BlockKind = BK_Block;
382         }
383       }
384       LBraceStack.pop_back();
385       break;
386     case tok::at:
387     case tok::semi:
388     case tok::kw_if:
389     case tok::kw_while:
390     case tok::kw_for:
391     case tok::kw_switch:
392     case tok::kw_try:
393     case tok::kw___try:
394       if (!LBraceStack.empty() && LBraceStack.back()->BlockKind == BK_Unknown)
395         LBraceStack.back()->BlockKind = BK_Block;
396       break;
397     default:
398       break;
399     }
400     PrevTok = Tok;
401     Tok = NextTok;
402   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
403 
404   // Assume other blocks for all unclosed opening braces.
405   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
406     if (LBraceStack[i]->BlockKind == BK_Unknown)
407       LBraceStack[i]->BlockKind = BK_Block;
408   }
409 
410   FormatTok = Tokens->setPosition(StoredPosition);
411 }
412 
413 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
414                                      bool MunchSemi) {
415   assert(FormatTok->isOneOf(tok::l_brace, TT_MacroBlockBegin) &&
416          "'{' or macro block token expected");
417   const bool MacroBlock = FormatTok->is(TT_MacroBlockBegin);
418   FormatTok->BlockKind = BK_Block;
419 
420   unsigned InitialLevel = Line->Level;
421   nextToken();
422 
423   if (MacroBlock && FormatTok->is(tok::l_paren))
424     parseParens();
425 
426   addUnwrappedLine();
427 
428   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
429                                           MustBeDeclaration);
430   if (AddLevel)
431     ++Line->Level;
432   parseLevel(/*HasOpeningBrace=*/true);
433 
434   if (eof())
435     return;
436 
437   if (MacroBlock ? !FormatTok->is(TT_MacroBlockEnd)
438                  : !FormatTok->is(tok::r_brace)) {
439     Line->Level = InitialLevel;
440     FormatTok->BlockKind = BK_Block;
441     return;
442   }
443 
444   nextToken(); // Munch the closing brace.
445 
446   if (MacroBlock && FormatTok->is(tok::l_paren))
447     parseParens();
448 
449   if (MunchSemi && FormatTok->Tok.is(tok::semi))
450     nextToken();
451   Line->Level = InitialLevel;
452 }
453 
454 static bool isGoogScope(const UnwrappedLine &Line) {
455   // FIXME: Closure-library specific stuff should not be hard-coded but be
456   // configurable.
457   if (Line.Tokens.size() < 4)
458     return false;
459   auto I = Line.Tokens.begin();
460   if (I->Tok->TokenText != "goog")
461     return false;
462   ++I;
463   if (I->Tok->isNot(tok::period))
464     return false;
465   ++I;
466   if (I->Tok->TokenText != "scope")
467     return false;
468   ++I;
469   return I->Tok->is(tok::l_paren);
470 }
471 
472 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
473                                    const FormatToken &InitialToken) {
474   if (InitialToken.is(tok::kw_namespace))
475     return Style.BraceWrapping.AfterNamespace;
476   if (InitialToken.is(tok::kw_class))
477     return Style.BraceWrapping.AfterClass;
478   if (InitialToken.is(tok::kw_union))
479     return Style.BraceWrapping.AfterUnion;
480   if (InitialToken.is(tok::kw_struct))
481     return Style.BraceWrapping.AfterStruct;
482   return false;
483 }
484 
485 void UnwrappedLineParser::parseChildBlock() {
486   FormatTok->BlockKind = BK_Block;
487   nextToken();
488   {
489     bool GoogScope =
490         Style.Language == FormatStyle::LK_JavaScript && isGoogScope(*Line);
491     ScopedLineState LineState(*this);
492     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
493                                             /*MustBeDeclaration=*/false);
494     Line->Level += GoogScope ? 0 : 1;
495     parseLevel(/*HasOpeningBrace=*/true);
496     flushComments(isOnNewLine(*FormatTok));
497     Line->Level -= GoogScope ? 0 : 1;
498   }
499   nextToken();
500 }
501 
502 void UnwrappedLineParser::parsePPDirective() {
503   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
504   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
505   nextToken();
506 
507   if (!FormatTok->Tok.getIdentifierInfo()) {
508     parsePPUnknown();
509     return;
510   }
511 
512   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
513   case tok::pp_define:
514     parsePPDefine();
515     return;
516   case tok::pp_if:
517     parsePPIf(/*IfDef=*/false);
518     break;
519   case tok::pp_ifdef:
520   case tok::pp_ifndef:
521     parsePPIf(/*IfDef=*/true);
522     break;
523   case tok::pp_else:
524     parsePPElse();
525     break;
526   case tok::pp_elif:
527     parsePPElIf();
528     break;
529   case tok::pp_endif:
530     parsePPEndIf();
531     break;
532   default:
533     parsePPUnknown();
534     break;
535   }
536 }
537 
538 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
539   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
540     PPStack.push_back(PP_Unreachable);
541   else
542     PPStack.push_back(PP_Conditional);
543 }
544 
545 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
546   ++PPBranchLevel;
547   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
548   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
549     PPLevelBranchIndex.push_back(0);
550     PPLevelBranchCount.push_back(0);
551   }
552   PPChainBranchIndex.push(0);
553   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
554   conditionalCompilationCondition(Unreachable || Skip);
555 }
556 
557 void UnwrappedLineParser::conditionalCompilationAlternative() {
558   if (!PPStack.empty())
559     PPStack.pop_back();
560   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
561   if (!PPChainBranchIndex.empty())
562     ++PPChainBranchIndex.top();
563   conditionalCompilationCondition(
564       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
565       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
566 }
567 
568 void UnwrappedLineParser::conditionalCompilationEnd() {
569   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
570   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
571     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
572       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
573     }
574   }
575   // Guard against #endif's without #if.
576   if (PPBranchLevel > 0)
577     --PPBranchLevel;
578   if (!PPChainBranchIndex.empty())
579     PPChainBranchIndex.pop();
580   if (!PPStack.empty())
581     PPStack.pop_back();
582 }
583 
584 void UnwrappedLineParser::parsePPIf(bool IfDef) {
585   nextToken();
586   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
587                          FormatTok->Tok.getLiteralData() != nullptr &&
588                          StringRef(FormatTok->Tok.getLiteralData(),
589                                    FormatTok->Tok.getLength()) == "0") ||
590                         FormatTok->Tok.is(tok::kw_false);
591   conditionalCompilationStart(!IfDef && IsLiteralFalse);
592   parsePPUnknown();
593 }
594 
595 void UnwrappedLineParser::parsePPElse() {
596   conditionalCompilationAlternative();
597   parsePPUnknown();
598 }
599 
600 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
601 
602 void UnwrappedLineParser::parsePPEndIf() {
603   conditionalCompilationEnd();
604   parsePPUnknown();
605 }
606 
607 void UnwrappedLineParser::parsePPDefine() {
608   nextToken();
609 
610   if (FormatTok->Tok.getKind() != tok::identifier) {
611     parsePPUnknown();
612     return;
613   }
614   nextToken();
615   if (FormatTok->Tok.getKind() == tok::l_paren &&
616       FormatTok->WhitespaceRange.getBegin() ==
617           FormatTok->WhitespaceRange.getEnd()) {
618     parseParens();
619   }
620   addUnwrappedLine();
621   Line->Level = 1;
622 
623   // Errors during a preprocessor directive can only affect the layout of the
624   // preprocessor directive, and thus we ignore them. An alternative approach
625   // would be to use the same approach we use on the file level (no
626   // re-indentation if there was a structural error) within the macro
627   // definition.
628   parseFile();
629 }
630 
631 void UnwrappedLineParser::parsePPUnknown() {
632   do {
633     nextToken();
634   } while (!eof());
635   addUnwrappedLine();
636 }
637 
638 // Here we blacklist certain tokens that are not usually the first token in an
639 // unwrapped line. This is used in attempt to distinguish macro calls without
640 // trailing semicolons from other constructs split to several lines.
641 static bool tokenCanStartNewLine(const clang::Token &Tok) {
642   // Semicolon can be a null-statement, l_square can be a start of a macro or
643   // a C++11 attribute, but this doesn't seem to be common.
644   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
645          Tok.isNot(tok::l_square) &&
646          // Tokens that can only be used as binary operators and a part of
647          // overloaded operator names.
648          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
649          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
650          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
651          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
652          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
653          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
654          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
655          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
656          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
657          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
658          Tok.isNot(tok::lesslessequal) &&
659          // Colon is used in labels, base class lists, initializer lists,
660          // range-based for loops, ternary operator, but should never be the
661          // first token in an unwrapped line.
662          Tok.isNot(tok::colon) &&
663          // 'noexcept' is a trailing annotation.
664          Tok.isNot(tok::kw_noexcept);
665 }
666 
667 static bool mustBeJSIdent(const AdditionalKeywords &Keywords,
668                           const FormatToken *FormatTok) {
669   // FIXME: This returns true for C/C++ keywords like 'struct'.
670   return FormatTok->is(tok::identifier) &&
671          (FormatTok->Tok.getIdentifierInfo() == nullptr ||
672           !FormatTok->isOneOf(
673               Keywords.kw_in, Keywords.kw_of, Keywords.kw_as, Keywords.kw_async,
674               Keywords.kw_await, Keywords.kw_yield, Keywords.kw_finally,
675               Keywords.kw_function, Keywords.kw_import, Keywords.kw_is,
676               Keywords.kw_let, Keywords.kw_var, tok::kw_const,
677               Keywords.kw_abstract, Keywords.kw_extends, Keywords.kw_implements,
678               Keywords.kw_instanceof, Keywords.kw_interface,
679               Keywords.kw_throws));
680 }
681 
682 static bool mustBeJSIdentOrValue(const AdditionalKeywords &Keywords,
683                                  const FormatToken *FormatTok) {
684   return FormatTok->Tok.isLiteral() ||
685          FormatTok->isOneOf(tok::kw_true, tok::kw_false) ||
686          mustBeJSIdent(Keywords, FormatTok);
687 }
688 
689 // isJSDeclOrStmt returns true if |FormatTok| starts a declaration or statement
690 // when encountered after a value (see mustBeJSIdentOrValue).
691 static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
692                            const FormatToken *FormatTok) {
693   return FormatTok->isOneOf(
694       tok::kw_return, Keywords.kw_yield,
695       // conditionals
696       tok::kw_if, tok::kw_else,
697       // loops
698       tok::kw_for, tok::kw_while, tok::kw_do, tok::kw_continue, tok::kw_break,
699       // switch/case
700       tok::kw_switch, tok::kw_case,
701       // exceptions
702       tok::kw_throw, tok::kw_try, tok::kw_catch, Keywords.kw_finally,
703       // declaration
704       tok::kw_const, tok::kw_class, Keywords.kw_var, Keywords.kw_let,
705       Keywords.kw_async, Keywords.kw_function,
706       // import/export
707       Keywords.kw_import, tok::kw_export);
708 }
709 
710 // readTokenWithJavaScriptASI reads the next token and terminates the current
711 // line if JavaScript Automatic Semicolon Insertion must
712 // happen between the current token and the next token.
713 //
714 // This method is conservative - it cannot cover all edge cases of JavaScript,
715 // but only aims to correctly handle certain well known cases. It *must not*
716 // return true in speculative cases.
717 void UnwrappedLineParser::readTokenWithJavaScriptASI() {
718   FormatToken *Previous = FormatTok;
719   readToken();
720   FormatToken *Next = FormatTok;
721 
722   bool IsOnSameLine =
723       CommentsBeforeNextToken.empty()
724           ? Next->NewlinesBefore == 0
725           : CommentsBeforeNextToken.front()->NewlinesBefore == 0;
726   if (IsOnSameLine)
727     return;
728 
729   bool PreviousMustBeValue = mustBeJSIdentOrValue(Keywords, Previous);
730   bool PreviousStartsTemplateExpr =
731       Previous->is(TT_TemplateString) && Previous->TokenText.endswith("${");
732   if (PreviousMustBeValue && Line && Line->Tokens.size() > 1) {
733     // If the token before the previous one is an '@', the previous token is an
734     // annotation and can precede another identifier/value.
735     const FormatToken *PrePrevious = std::prev(Line->Tokens.end(), 2)->Tok;
736     if (PrePrevious->is(tok::at))
737       return;
738   }
739   if (Next->is(tok::exclaim) && PreviousMustBeValue)
740     return addUnwrappedLine();
741   bool NextMustBeValue = mustBeJSIdentOrValue(Keywords, Next);
742   bool NextEndsTemplateExpr =
743       Next->is(TT_TemplateString) && Next->TokenText.startswith("}");
744   if (NextMustBeValue && !NextEndsTemplateExpr && !PreviousStartsTemplateExpr &&
745       (PreviousMustBeValue ||
746        Previous->isOneOf(tok::r_square, tok::r_paren, tok::plusplus,
747                          tok::minusminus)))
748     return addUnwrappedLine();
749   if (PreviousMustBeValue && isJSDeclOrStmt(Keywords, Next))
750     return addUnwrappedLine();
751 }
752 
753 void UnwrappedLineParser::parseStructuralElement() {
754   assert(!FormatTok->is(tok::l_brace));
755   if (Style.Language == FormatStyle::LK_TableGen &&
756       FormatTok->is(tok::pp_include)) {
757     nextToken();
758     if (FormatTok->is(tok::string_literal))
759       nextToken();
760     addUnwrappedLine();
761     return;
762   }
763   switch (FormatTok->Tok.getKind()) {
764   case tok::at:
765     nextToken();
766     if (FormatTok->Tok.is(tok::l_brace)) {
767       parseBracedList();
768       break;
769     }
770     switch (FormatTok->Tok.getObjCKeywordID()) {
771     case tok::objc_public:
772     case tok::objc_protected:
773     case tok::objc_package:
774     case tok::objc_private:
775       return parseAccessSpecifier();
776     case tok::objc_interface:
777     case tok::objc_implementation:
778       return parseObjCInterfaceOrImplementation();
779     case tok::objc_protocol:
780       return parseObjCProtocol();
781     case tok::objc_end:
782       return; // Handled by the caller.
783     case tok::objc_optional:
784     case tok::objc_required:
785       nextToken();
786       addUnwrappedLine();
787       return;
788     case tok::objc_autoreleasepool:
789       nextToken();
790       if (FormatTok->Tok.is(tok::l_brace)) {
791         if (Style.BraceWrapping.AfterObjCDeclaration)
792           addUnwrappedLine();
793         parseBlock(/*MustBeDeclaration=*/false);
794       }
795       addUnwrappedLine();
796       return;
797     case tok::objc_try:
798       // This branch isn't strictly necessary (the kw_try case below would
799       // do this too after the tok::at is parsed above).  But be explicit.
800       parseTryCatch();
801       return;
802     default:
803       break;
804     }
805     break;
806   case tok::kw_asm:
807     nextToken();
808     if (FormatTok->is(tok::l_brace)) {
809       FormatTok->Type = TT_InlineASMBrace;
810       nextToken();
811       while (FormatTok && FormatTok->isNot(tok::eof)) {
812         if (FormatTok->is(tok::r_brace)) {
813           FormatTok->Type = TT_InlineASMBrace;
814           nextToken();
815           addUnwrappedLine();
816           break;
817         }
818         FormatTok->Finalized = true;
819         nextToken();
820       }
821     }
822     break;
823   case tok::kw_namespace:
824     parseNamespace();
825     return;
826   case tok::kw_inline:
827     nextToken();
828     if (FormatTok->Tok.is(tok::kw_namespace)) {
829       parseNamespace();
830       return;
831     }
832     break;
833   case tok::kw_public:
834   case tok::kw_protected:
835   case tok::kw_private:
836     if (Style.Language == FormatStyle::LK_Java ||
837         Style.Language == FormatStyle::LK_JavaScript)
838       nextToken();
839     else
840       parseAccessSpecifier();
841     return;
842   case tok::kw_if:
843     parseIfThenElse();
844     return;
845   case tok::kw_for:
846   case tok::kw_while:
847     parseForOrWhileLoop();
848     return;
849   case tok::kw_do:
850     parseDoWhile();
851     return;
852   case tok::kw_switch:
853     parseSwitch();
854     return;
855   case tok::kw_default:
856     nextToken();
857     parseLabel();
858     return;
859   case tok::kw_case:
860     parseCaseLabel();
861     return;
862   case tok::kw_try:
863   case tok::kw___try:
864     parseTryCatch();
865     return;
866   case tok::kw_extern:
867     nextToken();
868     if (FormatTok->Tok.is(tok::string_literal)) {
869       nextToken();
870       if (FormatTok->Tok.is(tok::l_brace)) {
871         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
872         addUnwrappedLine();
873         return;
874       }
875     }
876     break;
877   case tok::kw_export:
878     if (Style.Language == FormatStyle::LK_JavaScript) {
879       parseJavaScriptEs6ImportExport();
880       return;
881     }
882     break;
883   case tok::identifier:
884     if (FormatTok->is(TT_ForEachMacro)) {
885       parseForOrWhileLoop();
886       return;
887     }
888     if (FormatTok->is(TT_MacroBlockBegin)) {
889       parseBlock(/*MustBeDeclaration=*/false, /*AddLevel=*/true,
890                  /*MunchSemi=*/false);
891       return;
892     }
893     if (FormatTok->is(Keywords.kw_import)) {
894       if (Style.Language == FormatStyle::LK_JavaScript) {
895         parseJavaScriptEs6ImportExport();
896         return;
897       }
898       if (Style.Language == FormatStyle::LK_Proto) {
899         nextToken();
900         if (FormatTok->is(tok::kw_public))
901           nextToken();
902         if (!FormatTok->is(tok::string_literal))
903           return;
904         nextToken();
905         if (FormatTok->is(tok::semi))
906           nextToken();
907         addUnwrappedLine();
908         return;
909       }
910     }
911     if (FormatTok->isOneOf(Keywords.kw_signals, Keywords.kw_qsignals,
912                            Keywords.kw_slots, Keywords.kw_qslots)) {
913       nextToken();
914       if (FormatTok->is(tok::colon)) {
915         nextToken();
916         addUnwrappedLine();
917         return;
918       }
919     }
920     // In all other cases, parse the declaration.
921     break;
922   default:
923     break;
924   }
925   do {
926     const FormatToken *Previous = getPreviousToken();
927     switch (FormatTok->Tok.getKind()) {
928     case tok::at:
929       nextToken();
930       if (FormatTok->Tok.is(tok::l_brace))
931         parseBracedList();
932       break;
933     case tok::kw_enum:
934       // Ignore if this is part of "template <enum ...".
935       if (Previous && Previous->is(tok::less)) {
936         nextToken();
937         break;
938       }
939 
940       // parseEnum falls through and does not yet add an unwrapped line as an
941       // enum definition can start a structural element.
942       if (!parseEnum())
943         break;
944       // This only applies for C++.
945       if (Style.Language != FormatStyle::LK_Cpp) {
946         addUnwrappedLine();
947         return;
948       }
949       break;
950     case tok::kw_typedef:
951       nextToken();
952       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
953                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
954         parseEnum();
955       break;
956     case tok::kw_struct:
957     case tok::kw_union:
958     case tok::kw_class:
959       // parseRecord falls through and does not yet add an unwrapped line as a
960       // record declaration or definition can start a structural element.
961       parseRecord();
962       // This does not apply for Java and JavaScript.
963       if (Style.Language == FormatStyle::LK_Java ||
964           Style.Language == FormatStyle::LK_JavaScript) {
965         if (FormatTok->is(tok::semi))
966           nextToken();
967         addUnwrappedLine();
968         return;
969       }
970       break;
971     case tok::period:
972       nextToken();
973       // In Java, classes have an implicit static member "class".
974       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
975           FormatTok->is(tok::kw_class))
976         nextToken();
977       if (Style.Language == FormatStyle::LK_JavaScript && FormatTok &&
978           FormatTok->Tok.getIdentifierInfo())
979         // JavaScript only has pseudo keywords, all keywords are allowed to
980         // appear in "IdentifierName" positions. See http://es5.github.io/#x7.6
981         nextToken();
982       break;
983     case tok::semi:
984       nextToken();
985       addUnwrappedLine();
986       return;
987     case tok::r_brace:
988       addUnwrappedLine();
989       return;
990     case tok::l_paren:
991       parseParens();
992       break;
993     case tok::kw_operator:
994       nextToken();
995       if (FormatTok->isBinaryOperator())
996         nextToken();
997       break;
998     case tok::caret:
999       nextToken();
1000       if (FormatTok->Tok.isAnyIdentifier() ||
1001           FormatTok->isSimpleTypeSpecifier())
1002         nextToken();
1003       if (FormatTok->is(tok::l_paren))
1004         parseParens();
1005       if (FormatTok->is(tok::l_brace))
1006         parseChildBlock();
1007       break;
1008     case tok::l_brace:
1009       if (!tryToParseBracedList()) {
1010         // A block outside of parentheses must be the last part of a
1011         // structural element.
1012         // FIXME: Figure out cases where this is not true, and add projections
1013         // for them (the one we know is missing are lambdas).
1014         if (Style.BraceWrapping.AfterFunction)
1015           addUnwrappedLine();
1016         FormatTok->Type = TT_FunctionLBrace;
1017         parseBlock(/*MustBeDeclaration=*/false);
1018         addUnwrappedLine();
1019         return;
1020       }
1021       // Otherwise this was a braced init list, and the structural
1022       // element continues.
1023       break;
1024     case tok::kw_try:
1025       // We arrive here when parsing function-try blocks.
1026       parseTryCatch();
1027       return;
1028     case tok::identifier: {
1029       if (FormatTok->is(TT_MacroBlockEnd)) {
1030         addUnwrappedLine();
1031         return;
1032       }
1033 
1034       // Parse function literal unless 'function' is the first token in a line
1035       // in which case this should be treated as a free-standing function.
1036       if (Style.Language == FormatStyle::LK_JavaScript &&
1037           (FormatTok->is(Keywords.kw_function) ||
1038            FormatTok->startsSequence(Keywords.kw_async,
1039                                      Keywords.kw_function)) &&
1040           Line->Tokens.size() > 0) {
1041         tryToParseJSFunction();
1042         break;
1043       }
1044       if ((Style.Language == FormatStyle::LK_JavaScript ||
1045            Style.Language == FormatStyle::LK_Java) &&
1046           FormatTok->is(Keywords.kw_interface)) {
1047         if (Style.Language == FormatStyle::LK_JavaScript) {
1048           // In JavaScript/TypeScript, "interface" can be used as a standalone
1049           // identifier, e.g. in `var interface = 1;`. If "interface" is
1050           // followed by another identifier, it is very like to be an actual
1051           // interface declaration.
1052           unsigned StoredPosition = Tokens->getPosition();
1053           FormatToken *Next = Tokens->getNextToken();
1054           FormatTok = Tokens->setPosition(StoredPosition);
1055           if (Next && !mustBeJSIdent(Keywords, Next)) {
1056             nextToken();
1057             break;
1058           }
1059         }
1060         parseRecord();
1061         addUnwrappedLine();
1062         return;
1063       }
1064 
1065       // See if the following token should start a new unwrapped line.
1066       StringRef Text = FormatTok->TokenText;
1067       nextToken();
1068       if (Line->Tokens.size() == 1 &&
1069           // JS doesn't have macros, and within classes colons indicate fields,
1070           // not labels.
1071           Style.Language != FormatStyle::LK_JavaScript) {
1072         if (FormatTok->Tok.is(tok::colon) && !Line->MustBeDeclaration) {
1073           Line->Tokens.begin()->Tok->MustBreakBefore = true;
1074           parseLabel();
1075           return;
1076         }
1077         // Recognize function-like macro usages without trailing semicolon as
1078         // well as free-standing macros like Q_OBJECT.
1079         bool FunctionLike = FormatTok->is(tok::l_paren);
1080         if (FunctionLike)
1081           parseParens();
1082 
1083         bool FollowedByNewline =
1084             CommentsBeforeNextToken.empty()
1085                 ? FormatTok->NewlinesBefore > 0
1086                 : CommentsBeforeNextToken.front()->NewlinesBefore > 0;
1087 
1088         if (FollowedByNewline && (Text.size() >= 5 || FunctionLike) &&
1089             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
1090           addUnwrappedLine();
1091           return;
1092         }
1093       }
1094       break;
1095     }
1096     case tok::equal:
1097       // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
1098       // TT_JsFatArrow. The always start an expression or a child block if
1099       // followed by a curly.
1100       if (FormatTok->is(TT_JsFatArrow)) {
1101         nextToken();
1102         if (FormatTok->is(tok::l_brace))
1103           parseChildBlock();
1104         break;
1105       }
1106 
1107       nextToken();
1108       if (FormatTok->Tok.is(tok::l_brace)) {
1109         parseBracedList();
1110       }
1111       break;
1112     case tok::l_square:
1113       parseSquare();
1114       break;
1115     case tok::kw_new:
1116       parseNew();
1117       break;
1118     default:
1119       nextToken();
1120       break;
1121     }
1122   } while (!eof());
1123 }
1124 
1125 bool UnwrappedLineParser::tryToParseLambda() {
1126   if (Style.Language != FormatStyle::LK_Cpp) {
1127     nextToken();
1128     return false;
1129   }
1130   const FormatToken* Previous = getPreviousToken();
1131   if (Previous &&
1132       (Previous->isOneOf(tok::identifier, tok::kw_operator, tok::kw_new,
1133                          tok::kw_delete) ||
1134        Previous->closesScope() || Previous->isSimpleTypeSpecifier())) {
1135     nextToken();
1136     return false;
1137   }
1138   assert(FormatTok->is(tok::l_square));
1139   FormatToken &LSquare = *FormatTok;
1140   if (!tryToParseLambdaIntroducer())
1141     return false;
1142 
1143   while (FormatTok->isNot(tok::l_brace)) {
1144     if (FormatTok->isSimpleTypeSpecifier()) {
1145       nextToken();
1146       continue;
1147     }
1148     switch (FormatTok->Tok.getKind()) {
1149     case tok::l_brace:
1150       break;
1151     case tok::l_paren:
1152       parseParens();
1153       break;
1154     case tok::amp:
1155     case tok::star:
1156     case tok::kw_const:
1157     case tok::comma:
1158     case tok::less:
1159     case tok::greater:
1160     case tok::identifier:
1161     case tok::numeric_constant:
1162     case tok::coloncolon:
1163     case tok::kw_mutable:
1164       nextToken();
1165       break;
1166     case tok::arrow:
1167       FormatTok->Type = TT_LambdaArrow;
1168       nextToken();
1169       break;
1170     default:
1171       return true;
1172     }
1173   }
1174   LSquare.Type = TT_LambdaLSquare;
1175   parseChildBlock();
1176   return true;
1177 }
1178 
1179 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
1180   nextToken();
1181   if (FormatTok->is(tok::equal)) {
1182     nextToken();
1183     if (FormatTok->is(tok::r_square)) {
1184       nextToken();
1185       return true;
1186     }
1187     if (FormatTok->isNot(tok::comma))
1188       return false;
1189     nextToken();
1190   } else if (FormatTok->is(tok::amp)) {
1191     nextToken();
1192     if (FormatTok->is(tok::r_square)) {
1193       nextToken();
1194       return true;
1195     }
1196     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
1197       return false;
1198     }
1199     if (FormatTok->is(tok::comma))
1200       nextToken();
1201   } else if (FormatTok->is(tok::r_square)) {
1202     nextToken();
1203     return true;
1204   }
1205   do {
1206     if (FormatTok->is(tok::amp))
1207       nextToken();
1208     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
1209       return false;
1210     nextToken();
1211     if (FormatTok->is(tok::ellipsis))
1212       nextToken();
1213     if (FormatTok->is(tok::comma)) {
1214       nextToken();
1215     } else if (FormatTok->is(tok::r_square)) {
1216       nextToken();
1217       return true;
1218     } else {
1219       return false;
1220     }
1221   } while (!eof());
1222   return false;
1223 }
1224 
1225 void UnwrappedLineParser::tryToParseJSFunction() {
1226   assert(FormatTok->is(Keywords.kw_function) ||
1227          FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function));
1228   if (FormatTok->is(Keywords.kw_async))
1229     nextToken();
1230   // Consume "function".
1231   nextToken();
1232 
1233   // Consume * (generator function). Treat it like C++'s overloaded operators.
1234   if (FormatTok->is(tok::star)) {
1235     FormatTok->Type = TT_OverloadedOperator;
1236     nextToken();
1237   }
1238 
1239   // Consume function name.
1240   if (FormatTok->is(tok::identifier))
1241     nextToken();
1242 
1243   if (FormatTok->isNot(tok::l_paren))
1244     return;
1245 
1246   // Parse formal parameter list.
1247   parseParens();
1248 
1249   if (FormatTok->is(tok::colon)) {
1250     // Parse a type definition.
1251     nextToken();
1252 
1253     // Eat the type declaration. For braced inline object types, balance braces,
1254     // otherwise just parse until finding an l_brace for the function body.
1255     if (FormatTok->is(tok::l_brace))
1256       tryToParseBracedList();
1257     else
1258       while (!FormatTok->isOneOf(tok::l_brace, tok::semi) && !eof())
1259         nextToken();
1260   }
1261 
1262   if (FormatTok->is(tok::semi))
1263     return;
1264 
1265   parseChildBlock();
1266 }
1267 
1268 bool UnwrappedLineParser::tryToParseBracedList() {
1269   if (FormatTok->BlockKind == BK_Unknown)
1270     calculateBraceTypes();
1271   assert(FormatTok->BlockKind != BK_Unknown);
1272   if (FormatTok->BlockKind == BK_Block)
1273     return false;
1274   parseBracedList();
1275   return true;
1276 }
1277 
1278 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
1279   bool HasError = false;
1280   nextToken();
1281 
1282   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
1283   // replace this by using parseAssigmentExpression() inside.
1284   do {
1285     if (Style.Language == FormatStyle::LK_JavaScript) {
1286       if (FormatTok->is(Keywords.kw_function) ||
1287           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
1288         tryToParseJSFunction();
1289         continue;
1290       }
1291       if (FormatTok->is(TT_JsFatArrow)) {
1292         nextToken();
1293         // Fat arrows can be followed by simple expressions or by child blocks
1294         // in curly braces.
1295         if (FormatTok->is(tok::l_brace)) {
1296           parseChildBlock();
1297           continue;
1298         }
1299       }
1300     }
1301     switch (FormatTok->Tok.getKind()) {
1302     case tok::caret:
1303       nextToken();
1304       if (FormatTok->is(tok::l_brace)) {
1305         parseChildBlock();
1306       }
1307       break;
1308     case tok::l_square:
1309       tryToParseLambda();
1310       break;
1311     case tok::l_brace:
1312       // Assume there are no blocks inside a braced init list apart
1313       // from the ones we explicitly parse out (like lambdas).
1314       FormatTok->BlockKind = BK_BracedInit;
1315       parseBracedList();
1316       break;
1317     case tok::l_paren:
1318       parseParens();
1319       // JavaScript can just have free standing methods and getters/setters in
1320       // object literals. Detect them by a "{" following ")".
1321       if (Style.Language == FormatStyle::LK_JavaScript) {
1322         if (FormatTok->is(tok::l_brace))
1323           parseChildBlock();
1324         break;
1325       }
1326       break;
1327     case tok::r_brace:
1328       nextToken();
1329       return !HasError;
1330     case tok::semi:
1331       // JavaScript (or more precisely TypeScript) can have semicolons in braced
1332       // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
1333       // used for error recovery if we have otherwise determined that this is
1334       // a braced list.
1335       if (Style.Language == FormatStyle::LK_JavaScript) {
1336         nextToken();
1337         break;
1338       }
1339       HasError = true;
1340       if (!ContinueOnSemicolons)
1341         return !HasError;
1342       nextToken();
1343       break;
1344     case tok::comma:
1345       nextToken();
1346       break;
1347     default:
1348       nextToken();
1349       break;
1350     }
1351   } while (!eof());
1352   return false;
1353 }
1354 
1355 void UnwrappedLineParser::parseParens() {
1356   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1357   nextToken();
1358   do {
1359     switch (FormatTok->Tok.getKind()) {
1360     case tok::l_paren:
1361       parseParens();
1362       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1363         parseChildBlock();
1364       break;
1365     case tok::r_paren:
1366       nextToken();
1367       return;
1368     case tok::r_brace:
1369       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1370       return;
1371     case tok::l_square:
1372       tryToParseLambda();
1373       break;
1374     case tok::l_brace:
1375       if (!tryToParseBracedList())
1376         parseChildBlock();
1377       break;
1378     case tok::at:
1379       nextToken();
1380       if (FormatTok->Tok.is(tok::l_brace))
1381         parseBracedList();
1382       break;
1383     case tok::identifier:
1384       if (Style.Language == FormatStyle::LK_JavaScript &&
1385           (FormatTok->is(Keywords.kw_function) ||
1386            FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)))
1387         tryToParseJSFunction();
1388       else
1389         nextToken();
1390       break;
1391     default:
1392       nextToken();
1393       break;
1394     }
1395   } while (!eof());
1396 }
1397 
1398 void UnwrappedLineParser::parseSquare() {
1399   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1400   if (tryToParseLambda())
1401     return;
1402   do {
1403     switch (FormatTok->Tok.getKind()) {
1404     case tok::l_paren:
1405       parseParens();
1406       break;
1407     case tok::r_square:
1408       nextToken();
1409       return;
1410     case tok::r_brace:
1411       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1412       return;
1413     case tok::l_square:
1414       parseSquare();
1415       break;
1416     case tok::l_brace: {
1417       if (!tryToParseBracedList())
1418         parseChildBlock();
1419       break;
1420     }
1421     case tok::at:
1422       nextToken();
1423       if (FormatTok->Tok.is(tok::l_brace))
1424         parseBracedList();
1425       break;
1426     default:
1427       nextToken();
1428       break;
1429     }
1430   } while (!eof());
1431 }
1432 
1433 void UnwrappedLineParser::parseIfThenElse() {
1434   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1435   nextToken();
1436   if (FormatTok->Tok.is(tok::l_paren))
1437     parseParens();
1438   bool NeedsUnwrappedLine = false;
1439   if (FormatTok->Tok.is(tok::l_brace)) {
1440     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1441     parseBlock(/*MustBeDeclaration=*/false);
1442     if (Style.BraceWrapping.BeforeElse)
1443       addUnwrappedLine();
1444     else
1445       NeedsUnwrappedLine = true;
1446   } else {
1447     addUnwrappedLine();
1448     ++Line->Level;
1449     parseStructuralElement();
1450     --Line->Level;
1451   }
1452   if (FormatTok->Tok.is(tok::kw_else)) {
1453     nextToken();
1454     if (FormatTok->Tok.is(tok::l_brace)) {
1455       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1456       parseBlock(/*MustBeDeclaration=*/false);
1457       addUnwrappedLine();
1458     } else if (FormatTok->Tok.is(tok::kw_if)) {
1459       parseIfThenElse();
1460     } else {
1461       addUnwrappedLine();
1462       ++Line->Level;
1463       parseStructuralElement();
1464       if (FormatTok->is(tok::eof))
1465         addUnwrappedLine();
1466       --Line->Level;
1467     }
1468   } else if (NeedsUnwrappedLine) {
1469     addUnwrappedLine();
1470   }
1471 }
1472 
1473 void UnwrappedLineParser::parseTryCatch() {
1474   assert(FormatTok->isOneOf(tok::kw_try, tok::kw___try) && "'try' expected");
1475   nextToken();
1476   bool NeedsUnwrappedLine = false;
1477   if (FormatTok->is(tok::colon)) {
1478     // We are in a function try block, what comes is an initializer list.
1479     nextToken();
1480     while (FormatTok->is(tok::identifier)) {
1481       nextToken();
1482       if (FormatTok->is(tok::l_paren))
1483         parseParens();
1484       if (FormatTok->is(tok::comma))
1485         nextToken();
1486     }
1487   }
1488   // Parse try with resource.
1489   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1490     parseParens();
1491   }
1492   if (FormatTok->is(tok::l_brace)) {
1493     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1494     parseBlock(/*MustBeDeclaration=*/false);
1495     if (Style.BraceWrapping.BeforeCatch) {
1496       addUnwrappedLine();
1497     } else {
1498       NeedsUnwrappedLine = true;
1499     }
1500   } else if (!FormatTok->is(tok::kw_catch)) {
1501     // The C++ standard requires a compound-statement after a try.
1502     // If there's none, we try to assume there's a structuralElement
1503     // and try to continue.
1504     addUnwrappedLine();
1505     ++Line->Level;
1506     parseStructuralElement();
1507     --Line->Level;
1508   }
1509   while (1) {
1510     if (FormatTok->is(tok::at))
1511       nextToken();
1512     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
1513                              tok::kw___finally) ||
1514           ((Style.Language == FormatStyle::LK_Java ||
1515             Style.Language == FormatStyle::LK_JavaScript) &&
1516            FormatTok->is(Keywords.kw_finally)) ||
1517           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
1518            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
1519       break;
1520     nextToken();
1521     while (FormatTok->isNot(tok::l_brace)) {
1522       if (FormatTok->is(tok::l_paren)) {
1523         parseParens();
1524         continue;
1525       }
1526       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1527         return;
1528       nextToken();
1529     }
1530     NeedsUnwrappedLine = false;
1531     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1532     parseBlock(/*MustBeDeclaration=*/false);
1533     if (Style.BraceWrapping.BeforeCatch)
1534       addUnwrappedLine();
1535     else
1536       NeedsUnwrappedLine = true;
1537   }
1538   if (NeedsUnwrappedLine)
1539     addUnwrappedLine();
1540 }
1541 
1542 void UnwrappedLineParser::parseNamespace() {
1543   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1544 
1545   const FormatToken &InitialToken = *FormatTok;
1546   nextToken();
1547   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon))
1548     nextToken();
1549   if (FormatTok->Tok.is(tok::l_brace)) {
1550     if (ShouldBreakBeforeBrace(Style, InitialToken))
1551       addUnwrappedLine();
1552 
1553     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1554                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1555                      DeclarationScopeStack.size() > 1);
1556     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1557     // Munch the semicolon after a namespace. This is more common than one would
1558     // think. Puttin the semicolon into its own line is very ugly.
1559     if (FormatTok->Tok.is(tok::semi))
1560       nextToken();
1561     addUnwrappedLine();
1562   }
1563   // FIXME: Add error handling.
1564 }
1565 
1566 void UnwrappedLineParser::parseNew() {
1567   assert(FormatTok->is(tok::kw_new) && "'new' expected");
1568   nextToken();
1569   if (Style.Language != FormatStyle::LK_Java)
1570     return;
1571 
1572   // In Java, we can parse everything up to the parens, which aren't optional.
1573   do {
1574     // There should not be a ;, { or } before the new's open paren.
1575     if (FormatTok->isOneOf(tok::semi, tok::l_brace, tok::r_brace))
1576       return;
1577 
1578     // Consume the parens.
1579     if (FormatTok->is(tok::l_paren)) {
1580       parseParens();
1581 
1582       // If there is a class body of an anonymous class, consume that as child.
1583       if (FormatTok->is(tok::l_brace))
1584         parseChildBlock();
1585       return;
1586     }
1587     nextToken();
1588   } while (!eof());
1589 }
1590 
1591 void UnwrappedLineParser::parseForOrWhileLoop() {
1592   assert(FormatTok->isOneOf(tok::kw_for, tok::kw_while, TT_ForEachMacro) &&
1593          "'for', 'while' or foreach macro expected");
1594   nextToken();
1595   if (FormatTok->Tok.is(tok::l_paren))
1596     parseParens();
1597   if (FormatTok->Tok.is(tok::l_brace)) {
1598     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1599     parseBlock(/*MustBeDeclaration=*/false);
1600     addUnwrappedLine();
1601   } else {
1602     addUnwrappedLine();
1603     ++Line->Level;
1604     parseStructuralElement();
1605     --Line->Level;
1606   }
1607 }
1608 
1609 void UnwrappedLineParser::parseDoWhile() {
1610   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1611   nextToken();
1612   if (FormatTok->Tok.is(tok::l_brace)) {
1613     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1614     parseBlock(/*MustBeDeclaration=*/false);
1615     if (Style.BraceWrapping.IndentBraces)
1616       addUnwrappedLine();
1617   } else {
1618     addUnwrappedLine();
1619     ++Line->Level;
1620     parseStructuralElement();
1621     --Line->Level;
1622   }
1623 
1624   // FIXME: Add error handling.
1625   if (!FormatTok->Tok.is(tok::kw_while)) {
1626     addUnwrappedLine();
1627     return;
1628   }
1629 
1630   nextToken();
1631   parseStructuralElement();
1632 }
1633 
1634 void UnwrappedLineParser::parseLabel() {
1635   nextToken();
1636   unsigned OldLineLevel = Line->Level;
1637   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1638     --Line->Level;
1639   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1640     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1641     parseBlock(/*MustBeDeclaration=*/false);
1642     if (FormatTok->Tok.is(tok::kw_break)) {
1643       if (Style.BraceWrapping.AfterControlStatement)
1644         addUnwrappedLine();
1645       parseStructuralElement();
1646     }
1647     addUnwrappedLine();
1648   } else {
1649     if (FormatTok->is(tok::semi))
1650       nextToken();
1651     addUnwrappedLine();
1652   }
1653   Line->Level = OldLineLevel;
1654   if (FormatTok->isNot(tok::l_brace)) {
1655     parseStructuralElement();
1656     addUnwrappedLine();
1657   }
1658 }
1659 
1660 void UnwrappedLineParser::parseCaseLabel() {
1661   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1662   // FIXME: fix handling of complex expressions here.
1663   do {
1664     nextToken();
1665   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1666   parseLabel();
1667 }
1668 
1669 void UnwrappedLineParser::parseSwitch() {
1670   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1671   nextToken();
1672   if (FormatTok->Tok.is(tok::l_paren))
1673     parseParens();
1674   if (FormatTok->Tok.is(tok::l_brace)) {
1675     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1676     parseBlock(/*MustBeDeclaration=*/false);
1677     addUnwrappedLine();
1678   } else {
1679     addUnwrappedLine();
1680     ++Line->Level;
1681     parseStructuralElement();
1682     --Line->Level;
1683   }
1684 }
1685 
1686 void UnwrappedLineParser::parseAccessSpecifier() {
1687   nextToken();
1688   // Understand Qt's slots.
1689   if (FormatTok->isOneOf(Keywords.kw_slots, Keywords.kw_qslots))
1690     nextToken();
1691   // Otherwise, we don't know what it is, and we'd better keep the next token.
1692   if (FormatTok->Tok.is(tok::colon))
1693     nextToken();
1694   addUnwrappedLine();
1695 }
1696 
1697 bool UnwrappedLineParser::parseEnum() {
1698   // Won't be 'enum' for NS_ENUMs.
1699   if (FormatTok->Tok.is(tok::kw_enum))
1700     nextToken();
1701 
1702   // In TypeScript, "enum" can also be used as property name, e.g. in interface
1703   // declarations. An "enum" keyword followed by a colon would be a syntax
1704   // error and thus assume it is just an identifier.
1705   if (Style.Language == FormatStyle::LK_JavaScript &&
1706       FormatTok->isOneOf(tok::colon, tok::question))
1707     return false;
1708 
1709   // Eat up enum class ...
1710   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1711     nextToken();
1712 
1713   while (FormatTok->Tok.getIdentifierInfo() ||
1714          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1715                             tok::greater, tok::comma, tok::question)) {
1716     nextToken();
1717     // We can have macros or attributes in between 'enum' and the enum name.
1718     if (FormatTok->is(tok::l_paren))
1719       parseParens();
1720     if (FormatTok->is(tok::identifier)) {
1721       nextToken();
1722       // If there are two identifiers in a row, this is likely an elaborate
1723       // return type. In Java, this can be "implements", etc.
1724       if (Style.Language == FormatStyle::LK_Cpp &&
1725           FormatTok->is(tok::identifier))
1726         return false;
1727     }
1728   }
1729 
1730   // Just a declaration or something is wrong.
1731   if (FormatTok->isNot(tok::l_brace))
1732     return true;
1733   FormatTok->BlockKind = BK_Block;
1734 
1735   if (Style.Language == FormatStyle::LK_Java) {
1736     // Java enums are different.
1737     parseJavaEnumBody();
1738     return true;
1739   }
1740   if (Style.Language == FormatStyle::LK_Proto) {
1741     parseBlock(/*MustBeDeclaration=*/true);
1742     return true;
1743   }
1744 
1745   // Parse enum body.
1746   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1747   if (HasError) {
1748     if (FormatTok->is(tok::semi))
1749       nextToken();
1750     addUnwrappedLine();
1751   }
1752   return true;
1753 
1754   // There is no addUnwrappedLine() here so that we fall through to parsing a
1755   // structural element afterwards. Thus, in "enum A {} n, m;",
1756   // "} n, m;" will end up in one unwrapped line.
1757 }
1758 
1759 void UnwrappedLineParser::parseJavaEnumBody() {
1760   // Determine whether the enum is simple, i.e. does not have a semicolon or
1761   // constants with class bodies. Simple enums can be formatted like braced
1762   // lists, contracted to a single line, etc.
1763   unsigned StoredPosition = Tokens->getPosition();
1764   bool IsSimple = true;
1765   FormatToken *Tok = Tokens->getNextToken();
1766   while (Tok) {
1767     if (Tok->is(tok::r_brace))
1768       break;
1769     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1770       IsSimple = false;
1771       break;
1772     }
1773     // FIXME: This will also mark enums with braces in the arguments to enum
1774     // constants as "not simple". This is probably fine in practice, though.
1775     Tok = Tokens->getNextToken();
1776   }
1777   FormatTok = Tokens->setPosition(StoredPosition);
1778 
1779   if (IsSimple) {
1780     parseBracedList();
1781     addUnwrappedLine();
1782     return;
1783   }
1784 
1785   // Parse the body of a more complex enum.
1786   // First add a line for everything up to the "{".
1787   nextToken();
1788   addUnwrappedLine();
1789   ++Line->Level;
1790 
1791   // Parse the enum constants.
1792   while (FormatTok) {
1793     if (FormatTok->is(tok::l_brace)) {
1794       // Parse the constant's class body.
1795       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1796                  /*MunchSemi=*/false);
1797     } else if (FormatTok->is(tok::l_paren)) {
1798       parseParens();
1799     } else if (FormatTok->is(tok::comma)) {
1800       nextToken();
1801       addUnwrappedLine();
1802     } else if (FormatTok->is(tok::semi)) {
1803       nextToken();
1804       addUnwrappedLine();
1805       break;
1806     } else if (FormatTok->is(tok::r_brace)) {
1807       addUnwrappedLine();
1808       break;
1809     } else {
1810       nextToken();
1811     }
1812   }
1813 
1814   // Parse the class body after the enum's ";" if any.
1815   parseLevel(/*HasOpeningBrace=*/true);
1816   nextToken();
1817   --Line->Level;
1818   addUnwrappedLine();
1819 }
1820 
1821 void UnwrappedLineParser::parseRecord() {
1822   const FormatToken &InitialToken = *FormatTok;
1823   nextToken();
1824 
1825   // The actual identifier can be a nested name specifier, and in macros
1826   // it is often token-pasted.
1827   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::hashhash,
1828                             tok::kw___attribute, tok::kw___declspec,
1829                             tok::kw_alignas) ||
1830          ((Style.Language == FormatStyle::LK_Java ||
1831            Style.Language == FormatStyle::LK_JavaScript) &&
1832           FormatTok->isOneOf(tok::period, tok::comma))) {
1833     bool IsNonMacroIdentifier =
1834         FormatTok->is(tok::identifier) &&
1835         FormatTok->TokenText != FormatTok->TokenText.upper();
1836     nextToken();
1837     // We can have macros or attributes in between 'class' and the class name.
1838     if (!IsNonMacroIdentifier && FormatTok->Tok.is(tok::l_paren))
1839       parseParens();
1840   }
1841 
1842   // Note that parsing away template declarations here leads to incorrectly
1843   // accepting function declarations as record declarations.
1844   // In general, we cannot solve this problem. Consider:
1845   // class A<int> B() {}
1846   // which can be a function definition or a class definition when B() is a
1847   // macro. If we find enough real-world cases where this is a problem, we
1848   // can parse for the 'template' keyword in the beginning of the statement,
1849   // and thus rule out the record production in case there is no template
1850   // (this would still leave us with an ambiguity between template function
1851   // and class declarations).
1852   if (FormatTok->isOneOf(tok::colon, tok::less)) {
1853     while (!eof()) {
1854       if (FormatTok->is(tok::l_brace)) {
1855         calculateBraceTypes(/*ExpectClassBody=*/true);
1856         if (!tryToParseBracedList())
1857           break;
1858       }
1859       if (FormatTok->Tok.is(tok::semi))
1860         return;
1861       nextToken();
1862     }
1863   }
1864   if (FormatTok->Tok.is(tok::l_brace)) {
1865     if (ShouldBreakBeforeBrace(Style, InitialToken))
1866       addUnwrappedLine();
1867 
1868     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1869                /*MunchSemi=*/false);
1870   }
1871   // There is no addUnwrappedLine() here so that we fall through to parsing a
1872   // structural element afterwards. Thus, in "class A {} n, m;",
1873   // "} n, m;" will end up in one unwrapped line.
1874 }
1875 
1876 void UnwrappedLineParser::parseObjCProtocolList() {
1877   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1878   do
1879     nextToken();
1880   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1881   nextToken(); // Skip '>'.
1882 }
1883 
1884 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1885   do {
1886     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1887       nextToken();
1888       addUnwrappedLine();
1889       break;
1890     }
1891     if (FormatTok->is(tok::l_brace)) {
1892       parseBlock(/*MustBeDeclaration=*/false);
1893       // In ObjC interfaces, nothing should be following the "}".
1894       addUnwrappedLine();
1895     } else if (FormatTok->is(tok::r_brace)) {
1896       // Ignore stray "}". parseStructuralElement doesn't consume them.
1897       nextToken();
1898       addUnwrappedLine();
1899     } else {
1900       parseStructuralElement();
1901     }
1902   } while (!eof());
1903 }
1904 
1905 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1906   nextToken();
1907   nextToken(); // interface name
1908 
1909   // @interface can be followed by either a base class, or a category.
1910   if (FormatTok->Tok.is(tok::colon)) {
1911     nextToken();
1912     nextToken(); // base class name
1913   } else if (FormatTok->Tok.is(tok::l_paren))
1914     // Skip category, if present.
1915     parseParens();
1916 
1917   if (FormatTok->Tok.is(tok::less))
1918     parseObjCProtocolList();
1919 
1920   if (FormatTok->Tok.is(tok::l_brace)) {
1921     if (Style.BraceWrapping.AfterObjCDeclaration)
1922       addUnwrappedLine();
1923     parseBlock(/*MustBeDeclaration=*/true);
1924   }
1925 
1926   // With instance variables, this puts '}' on its own line.  Without instance
1927   // variables, this ends the @interface line.
1928   addUnwrappedLine();
1929 
1930   parseObjCUntilAtEnd();
1931 }
1932 
1933 void UnwrappedLineParser::parseObjCProtocol() {
1934   nextToken();
1935   nextToken(); // protocol name
1936 
1937   if (FormatTok->Tok.is(tok::less))
1938     parseObjCProtocolList();
1939 
1940   // Check for protocol declaration.
1941   if (FormatTok->Tok.is(tok::semi)) {
1942     nextToken();
1943     return addUnwrappedLine();
1944   }
1945 
1946   addUnwrappedLine();
1947   parseObjCUntilAtEnd();
1948 }
1949 
1950 void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
1951   bool IsImport = FormatTok->is(Keywords.kw_import);
1952   assert(IsImport || FormatTok->is(tok::kw_export));
1953   nextToken();
1954 
1955   // Consume the "default" in "export default class/function".
1956   if (FormatTok->is(tok::kw_default))
1957     nextToken();
1958 
1959   // Consume "async function", "function" and "default function", so that these
1960   // get parsed as free-standing JS functions, i.e. do not require a trailing
1961   // semicolon.
1962   if (FormatTok->is(Keywords.kw_async))
1963     nextToken();
1964   if (FormatTok->is(Keywords.kw_function)) {
1965     nextToken();
1966     return;
1967   }
1968 
1969   // For imports, `export *`, `export {...}`, consume the rest of the line up
1970   // to the terminating `;`. For everything else, just return and continue
1971   // parsing the structural element, i.e. the declaration or expression for
1972   // `export default`.
1973   if (!IsImport && !FormatTok->isOneOf(tok::l_brace, tok::star) &&
1974       !FormatTok->isStringLiteral())
1975     return;
1976 
1977   while (!eof()) {
1978     if (FormatTok->is(tok::semi))
1979       return;
1980     if (Line->Tokens.size() == 0) {
1981       // Common issue: Automatic Semicolon Insertion wrapped the line, so the
1982       // import statement should terminate.
1983       return;
1984     }
1985     if (FormatTok->is(tok::l_brace)) {
1986       FormatTok->BlockKind = BK_Block;
1987       parseBracedList();
1988     } else {
1989       nextToken();
1990     }
1991   }
1992 }
1993 
1994 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1995                                                  StringRef Prefix = "") {
1996   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1997                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1998   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1999                                                     E = Line.Tokens.end();
2000        I != E; ++I) {
2001     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
2002   }
2003   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
2004                                                     E = Line.Tokens.end();
2005        I != E; ++I) {
2006     const UnwrappedLineNode &Node = *I;
2007     for (SmallVectorImpl<UnwrappedLine>::const_iterator
2008              I = Node.Children.begin(),
2009              E = Node.Children.end();
2010          I != E; ++I) {
2011       printDebugInfo(*I, "\nChild: ");
2012     }
2013   }
2014   llvm::dbgs() << "\n";
2015 }
2016 
2017 void UnwrappedLineParser::addUnwrappedLine() {
2018   if (Line->Tokens.empty())
2019     return;
2020   DEBUG({
2021     if (CurrentLines == &Lines)
2022       printDebugInfo(*Line);
2023   });
2024   CurrentLines->push_back(std::move(*Line));
2025   Line->Tokens.clear();
2026   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
2027     CurrentLines->append(
2028         std::make_move_iterator(PreprocessorDirectives.begin()),
2029         std::make_move_iterator(PreprocessorDirectives.end()));
2030     PreprocessorDirectives.clear();
2031   }
2032 }
2033 
2034 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
2035 
2036 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
2037   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
2038          FormatTok.NewlinesBefore > 0;
2039 }
2040 
2041 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
2042   bool JustComments = Line->Tokens.empty();
2043   for (SmallVectorImpl<FormatToken *>::const_iterator
2044            I = CommentsBeforeNextToken.begin(),
2045            E = CommentsBeforeNextToken.end();
2046        I != E; ++I) {
2047     if (isOnNewLine(**I) && JustComments)
2048       addUnwrappedLine();
2049     pushToken(*I);
2050   }
2051   if (NewlineBeforeNext && JustComments)
2052     addUnwrappedLine();
2053   CommentsBeforeNextToken.clear();
2054 }
2055 
2056 void UnwrappedLineParser::nextToken() {
2057   if (eof())
2058     return;
2059   flushComments(isOnNewLine(*FormatTok));
2060   pushToken(FormatTok);
2061   if (Style.Language != FormatStyle::LK_JavaScript)
2062     readToken();
2063   else
2064     readTokenWithJavaScriptASI();
2065 }
2066 
2067 const FormatToken *UnwrappedLineParser::getPreviousToken() {
2068   // FIXME: This is a dirty way to access the previous token. Find a better
2069   // solution.
2070   if (!Line || Line->Tokens.empty())
2071     return nullptr;
2072   return Line->Tokens.back().Tok;
2073 }
2074 
2075 void UnwrappedLineParser::readToken() {
2076   bool CommentsInCurrentLine = true;
2077   do {
2078     FormatTok = Tokens->getNextToken();
2079     assert(FormatTok);
2080     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
2081            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
2082       // If there is an unfinished unwrapped line, we flush the preprocessor
2083       // directives only after that unwrapped line was finished later.
2084       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
2085       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
2086       // Comments stored before the preprocessor directive need to be output
2087       // before the preprocessor directive, at the same level as the
2088       // preprocessor directive, as we consider them to apply to the directive.
2089       flushComments(isOnNewLine(*FormatTok));
2090       parsePPDirective();
2091     }
2092     while (FormatTok->Type == TT_ConflictStart ||
2093            FormatTok->Type == TT_ConflictEnd ||
2094            FormatTok->Type == TT_ConflictAlternative) {
2095       if (FormatTok->Type == TT_ConflictStart) {
2096         conditionalCompilationStart(/*Unreachable=*/false);
2097       } else if (FormatTok->Type == TT_ConflictAlternative) {
2098         conditionalCompilationAlternative();
2099       } else if (FormatTok->Type == TT_ConflictEnd) {
2100         conditionalCompilationEnd();
2101       }
2102       FormatTok = Tokens->getNextToken();
2103       FormatTok->MustBreakBefore = true;
2104     }
2105 
2106     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
2107         !Line->InPPDirective) {
2108       continue;
2109     }
2110 
2111     if (!FormatTok->Tok.is(tok::comment))
2112       return;
2113     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
2114       CommentsInCurrentLine = false;
2115     }
2116     if (CommentsInCurrentLine) {
2117       pushToken(FormatTok);
2118     } else {
2119       CommentsBeforeNextToken.push_back(FormatTok);
2120     }
2121   } while (!eof());
2122 }
2123 
2124 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
2125   Line->Tokens.push_back(UnwrappedLineNode(Tok));
2126   if (MustBreakBeforeNextToken) {
2127     Line->Tokens.back().Tok->MustBreakBefore = true;
2128     MustBreakBeforeNextToken = false;
2129   }
2130 }
2131 
2132 } // end namespace format
2133 } // end namespace clang
2134