1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #define DEBUG_TYPE "format-parser"
22 
23 namespace clang {
24 namespace format {
25 
26 class FormatTokenSource {
27 public:
28   virtual ~FormatTokenSource() {}
29   virtual FormatToken *getNextToken() = 0;
30 
31   virtual unsigned getPosition() = 0;
32   virtual FormatToken *setPosition(unsigned Position) = 0;
33 };
34 
35 namespace {
36 
37 class ScopedDeclarationState {
38 public:
39   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
40                          bool MustBeDeclaration)
41       : Line(Line), Stack(Stack) {
42     Line.MustBeDeclaration = MustBeDeclaration;
43     Stack.push_back(MustBeDeclaration);
44   }
45   ~ScopedDeclarationState() {
46     Stack.pop_back();
47     if (!Stack.empty())
48       Line.MustBeDeclaration = Stack.back();
49     else
50       Line.MustBeDeclaration = true;
51   }
52 
53 private:
54   UnwrappedLine &Line;
55   std::vector<bool> &Stack;
56 };
57 
58 class ScopedMacroState : public FormatTokenSource {
59 public:
60   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
61                    FormatToken *&ResetToken)
62       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
63         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
64         Token(nullptr) {
65     TokenSource = this;
66     Line.Level = 0;
67     Line.InPPDirective = true;
68   }
69 
70   ~ScopedMacroState() override {
71     TokenSource = PreviousTokenSource;
72     ResetToken = Token;
73     Line.InPPDirective = false;
74     Line.Level = PreviousLineLevel;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113 
114   FormatToken *Token;
115 };
116 
117 } // end anonymous namespace
118 
119 class ScopedLineState {
120 public:
121   ScopedLineState(UnwrappedLineParser &Parser,
122                   bool SwitchToPreprocessorLines = false)
123       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
124     if (SwitchToPreprocessorLines)
125       Parser.CurrentLines = &Parser.PreprocessorDirectives;
126     else if (!Parser.Line->Tokens.empty())
127       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
128     PreBlockLine = std::move(Parser.Line);
129     Parser.Line = llvm::make_unique<UnwrappedLine>();
130     Parser.Line->Level = PreBlockLine->Level;
131     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
132   }
133 
134   ~ScopedLineState() {
135     if (!Parser.Line->Tokens.empty()) {
136       Parser.addUnwrappedLine();
137     }
138     assert(Parser.Line->Tokens.empty());
139     Parser.Line = std::move(PreBlockLine);
140     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
141       Parser.MustBreakBeforeNextToken = true;
142     Parser.CurrentLines = OriginalLines;
143   }
144 
145 private:
146   UnwrappedLineParser &Parser;
147 
148   std::unique_ptr<UnwrappedLine> PreBlockLine;
149   SmallVectorImpl<UnwrappedLine> *OriginalLines;
150 };
151 
152 class CompoundStatementIndenter {
153 public:
154   CompoundStatementIndenter(UnwrappedLineParser *Parser,
155                             const FormatStyle &Style, unsigned &LineLevel)
156       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
157     if (Style.BraceWrapping.AfterControlStatement)
158       Parser->addUnwrappedLine();
159     if (Style.BraceWrapping.IndentBraces)
160       ++LineLevel;
161   }
162   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
163 
164 private:
165   unsigned &LineLevel;
166   unsigned OldLineLevel;
167 };
168 
169 namespace {
170 
171 class IndexedTokenSource : public FormatTokenSource {
172 public:
173   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
174       : Tokens(Tokens), Position(-1) {}
175 
176   FormatToken *getNextToken() override {
177     ++Position;
178     return Tokens[Position];
179   }
180 
181   unsigned getPosition() override {
182     assert(Position >= 0);
183     return Position;
184   }
185 
186   FormatToken *setPosition(unsigned P) override {
187     Position = P;
188     return Tokens[Position];
189   }
190 
191   void reset() { Position = -1; }
192 
193 private:
194   ArrayRef<FormatToken *> Tokens;
195   int Position;
196 };
197 
198 } // end anonymous namespace
199 
200 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
201                                          const AdditionalKeywords &Keywords,
202                                          ArrayRef<FormatToken *> Tokens,
203                                          UnwrappedLineConsumer &Callback)
204     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
205       CurrentLines(&Lines), Style(Style), Keywords(Keywords),
206       CommentPragmasRegex(Style.CommentPragmas), Tokens(nullptr),
207       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
208 
209 void UnwrappedLineParser::reset() {
210   PPBranchLevel = -1;
211   Line.reset(new UnwrappedLine);
212   CommentsBeforeNextToken.clear();
213   FormatTok = nullptr;
214   MustBreakBeforeNextToken = false;
215   PreprocessorDirectives.clear();
216   CurrentLines = &Lines;
217   DeclarationScopeStack.clear();
218   PPStack.clear();
219 }
220 
221 void UnwrappedLineParser::parse() {
222   IndexedTokenSource TokenSource(AllTokens);
223   do {
224     DEBUG(llvm::dbgs() << "----\n");
225     reset();
226     Tokens = &TokenSource;
227     TokenSource.reset();
228 
229     readToken();
230     parseFile();
231     // Create line with eof token.
232     pushToken(FormatTok);
233     addUnwrappedLine();
234 
235     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
236                                                   E = Lines.end();
237          I != E; ++I) {
238       Callback.consumeUnwrappedLine(*I);
239     }
240     Callback.finishRun();
241     Lines.clear();
242     while (!PPLevelBranchIndex.empty() &&
243            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
244       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
245       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
246     }
247     if (!PPLevelBranchIndex.empty()) {
248       ++PPLevelBranchIndex.back();
249       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
250       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
251     }
252   } while (!PPLevelBranchIndex.empty());
253 }
254 
255 void UnwrappedLineParser::parseFile() {
256   // The top-level context in a file always has declarations, except for pre-
257   // processor directives and JavaScript files.
258   bool MustBeDeclaration =
259       !Line->InPPDirective && Style.Language != FormatStyle::LK_JavaScript;
260   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
261                                           MustBeDeclaration);
262   parseLevel(/*HasOpeningBrace=*/false);
263   // Make sure to format the remaining tokens.
264   flushComments(true);
265   addUnwrappedLine();
266 }
267 
268 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
269   bool SwitchLabelEncountered = false;
270   do {
271     tok::TokenKind kind = FormatTok->Tok.getKind();
272     if (FormatTok->Type == TT_MacroBlockBegin) {
273       kind = tok::l_brace;
274     } else if (FormatTok->Type == TT_MacroBlockEnd) {
275       kind = tok::r_brace;
276     }
277 
278     switch (kind) {
279     case tok::comment:
280       nextToken();
281       addUnwrappedLine();
282       break;
283     case tok::l_brace:
284       // FIXME: Add parameter whether this can happen - if this happens, we must
285       // be in a non-declaration context.
286       if (!FormatTok->is(TT_MacroBlockBegin) && tryToParseBracedList())
287         continue;
288       parseBlock(/*MustBeDeclaration=*/false);
289       addUnwrappedLine();
290       break;
291     case tok::r_brace:
292       if (HasOpeningBrace)
293         return;
294       nextToken();
295       addUnwrappedLine();
296       break;
297     case tok::kw_default:
298     case tok::kw_case:
299       if (!SwitchLabelEncountered &&
300           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
301         ++Line->Level;
302       SwitchLabelEncountered = true;
303       parseStructuralElement();
304       break;
305     default:
306       parseStructuralElement();
307       break;
308     }
309   } while (!eof());
310 }
311 
312 void UnwrappedLineParser::calculateBraceTypes(bool ExpectClassBody) {
313   // We'll parse forward through the tokens until we hit
314   // a closing brace or eof - note that getNextToken() will
315   // parse macros, so this will magically work inside macro
316   // definitions, too.
317   unsigned StoredPosition = Tokens->getPosition();
318   FormatToken *Tok = FormatTok;
319   const FormatToken *PrevTok = getPreviousToken();
320   // Keep a stack of positions of lbrace tokens. We will
321   // update information about whether an lbrace starts a
322   // braced init list or a different block during the loop.
323   SmallVector<FormatToken *, 8> LBraceStack;
324   assert(Tok->Tok.is(tok::l_brace));
325   do {
326     // Get next non-comment token.
327     FormatToken *NextTok;
328     unsigned ReadTokens = 0;
329     do {
330       NextTok = Tokens->getNextToken();
331       ++ReadTokens;
332     } while (NextTok->is(tok::comment));
333 
334     switch (Tok->Tok.getKind()) {
335     case tok::l_brace:
336       if (Style.Language == FormatStyle::LK_JavaScript && PrevTok &&
337           PrevTok->is(tok::colon))
338         // In TypeScript's TypeMemberLists, there can be semicolons between the
339         // individual members.
340         Tok->BlockKind = BK_BracedInit;
341       else
342         Tok->BlockKind = BK_Unknown;
343       LBraceStack.push_back(Tok);
344       break;
345     case tok::r_brace:
346       if (LBraceStack.empty())
347         break;
348       if (LBraceStack.back()->BlockKind == BK_Unknown) {
349         bool ProbablyBracedList = false;
350         if (Style.Language == FormatStyle::LK_Proto) {
351           ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
352         } else {
353           // Using OriginalColumn to distinguish between ObjC methods and
354           // binary operators is a bit hacky.
355           bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
356                                   NextTok->OriginalColumn == 0;
357 
358           // If there is a comma, semicolon or right paren after the closing
359           // brace, we assume this is a braced initializer list.  Note that
360           // regardless how we mark inner braces here, we will overwrite the
361           // BlockKind later if we parse a braced list (where all blocks
362           // inside are by default braced lists), or when we explicitly detect
363           // blocks (for example while parsing lambdas).
364           ProbablyBracedList =
365               (Style.Language == FormatStyle::LK_JavaScript &&
366                NextTok->isOneOf(Keywords.kw_of, Keywords.kw_in,
367                                 Keywords.kw_as)) ||
368               NextTok->isOneOf(tok::comma, tok::period, tok::colon,
369                                tok::r_paren, tok::r_square, tok::l_brace,
370                                tok::l_square, tok::l_paren, tok::ellipsis) ||
371               (NextTok->is(tok::identifier) &&
372                !PrevTok->isOneOf(tok::semi, tok::r_brace, tok::l_brace)) ||
373               (NextTok->is(tok::semi) &&
374                (!ExpectClassBody || LBraceStack.size() != 1)) ||
375               (NextTok->isBinaryOperator() && !NextIsObjCMethod);
376         }
377         if (ProbablyBracedList) {
378           Tok->BlockKind = BK_BracedInit;
379           LBraceStack.back()->BlockKind = BK_BracedInit;
380         } else {
381           Tok->BlockKind = BK_Block;
382           LBraceStack.back()->BlockKind = BK_Block;
383         }
384       }
385       LBraceStack.pop_back();
386       break;
387     case tok::at:
388     case tok::semi:
389     case tok::kw_if:
390     case tok::kw_while:
391     case tok::kw_for:
392     case tok::kw_switch:
393     case tok::kw_try:
394     case tok::kw___try:
395       if (!LBraceStack.empty() && LBraceStack.back()->BlockKind == BK_Unknown)
396         LBraceStack.back()->BlockKind = BK_Block;
397       break;
398     default:
399       break;
400     }
401     PrevTok = Tok;
402     Tok = NextTok;
403   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
404 
405   // Assume other blocks for all unclosed opening braces.
406   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
407     if (LBraceStack[i]->BlockKind == BK_Unknown)
408       LBraceStack[i]->BlockKind = BK_Block;
409   }
410 
411   FormatTok = Tokens->setPosition(StoredPosition);
412 }
413 
414 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
415                                      bool MunchSemi) {
416   assert(FormatTok->isOneOf(tok::l_brace, TT_MacroBlockBegin) &&
417          "'{' or macro block token expected");
418   const bool MacroBlock = FormatTok->is(TT_MacroBlockBegin);
419   FormatTok->BlockKind = BK_Block;
420 
421   unsigned InitialLevel = Line->Level;
422   nextToken();
423 
424   if (MacroBlock && FormatTok->is(tok::l_paren))
425     parseParens();
426 
427   addUnwrappedLine();
428 
429   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
430                                           MustBeDeclaration);
431   if (AddLevel)
432     ++Line->Level;
433   parseLevel(/*HasOpeningBrace=*/true);
434 
435   if (eof())
436     return;
437 
438   if (MacroBlock ? !FormatTok->is(TT_MacroBlockEnd)
439                  : !FormatTok->is(tok::r_brace)) {
440     Line->Level = InitialLevel;
441     FormatTok->BlockKind = BK_Block;
442     return;
443   }
444 
445   nextToken(); // Munch the closing brace.
446 
447   if (MacroBlock && FormatTok->is(tok::l_paren))
448     parseParens();
449 
450   if (MunchSemi && FormatTok->Tok.is(tok::semi))
451     nextToken();
452   Line->Level = InitialLevel;
453 }
454 
455 static bool isGoogScope(const UnwrappedLine &Line) {
456   // FIXME: Closure-library specific stuff should not be hard-coded but be
457   // configurable.
458   if (Line.Tokens.size() < 4)
459     return false;
460   auto I = Line.Tokens.begin();
461   if (I->Tok->TokenText != "goog")
462     return false;
463   ++I;
464   if (I->Tok->isNot(tok::period))
465     return false;
466   ++I;
467   if (I->Tok->TokenText != "scope")
468     return false;
469   ++I;
470   return I->Tok->is(tok::l_paren);
471 }
472 
473 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
474                                    const FormatToken &InitialToken) {
475   if (InitialToken.is(tok::kw_namespace))
476     return Style.BraceWrapping.AfterNamespace;
477   if (InitialToken.is(tok::kw_class))
478     return Style.BraceWrapping.AfterClass;
479   if (InitialToken.is(tok::kw_union))
480     return Style.BraceWrapping.AfterUnion;
481   if (InitialToken.is(tok::kw_struct))
482     return Style.BraceWrapping.AfterStruct;
483   return false;
484 }
485 
486 void UnwrappedLineParser::parseChildBlock() {
487   FormatTok->BlockKind = BK_Block;
488   nextToken();
489   {
490     bool GoogScope =
491         Style.Language == FormatStyle::LK_JavaScript && isGoogScope(*Line);
492     ScopedLineState LineState(*this);
493     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
494                                             /*MustBeDeclaration=*/false);
495     Line->Level += GoogScope ? 0 : 1;
496     parseLevel(/*HasOpeningBrace=*/true);
497     flushComments(isOnNewLine(*FormatTok));
498     Line->Level -= GoogScope ? 0 : 1;
499   }
500   nextToken();
501 }
502 
503 void UnwrappedLineParser::parsePPDirective() {
504   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
505   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
506   nextToken();
507 
508   if (!FormatTok->Tok.getIdentifierInfo()) {
509     parsePPUnknown();
510     return;
511   }
512 
513   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
514   case tok::pp_define:
515     parsePPDefine();
516     return;
517   case tok::pp_if:
518     parsePPIf(/*IfDef=*/false);
519     break;
520   case tok::pp_ifdef:
521   case tok::pp_ifndef:
522     parsePPIf(/*IfDef=*/true);
523     break;
524   case tok::pp_else:
525     parsePPElse();
526     break;
527   case tok::pp_elif:
528     parsePPElIf();
529     break;
530   case tok::pp_endif:
531     parsePPEndIf();
532     break;
533   default:
534     parsePPUnknown();
535     break;
536   }
537 }
538 
539 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
540   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
541     PPStack.push_back(PP_Unreachable);
542   else
543     PPStack.push_back(PP_Conditional);
544 }
545 
546 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
547   ++PPBranchLevel;
548   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
549   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
550     PPLevelBranchIndex.push_back(0);
551     PPLevelBranchCount.push_back(0);
552   }
553   PPChainBranchIndex.push(0);
554   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
555   conditionalCompilationCondition(Unreachable || Skip);
556 }
557 
558 void UnwrappedLineParser::conditionalCompilationAlternative() {
559   if (!PPStack.empty())
560     PPStack.pop_back();
561   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
562   if (!PPChainBranchIndex.empty())
563     ++PPChainBranchIndex.top();
564   conditionalCompilationCondition(
565       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
566       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
567 }
568 
569 void UnwrappedLineParser::conditionalCompilationEnd() {
570   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
571   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
572     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
573       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
574     }
575   }
576   // Guard against #endif's without #if.
577   if (PPBranchLevel > 0)
578     --PPBranchLevel;
579   if (!PPChainBranchIndex.empty())
580     PPChainBranchIndex.pop();
581   if (!PPStack.empty())
582     PPStack.pop_back();
583 }
584 
585 void UnwrappedLineParser::parsePPIf(bool IfDef) {
586   nextToken();
587   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
588                          FormatTok->Tok.getLiteralData() != nullptr &&
589                          StringRef(FormatTok->Tok.getLiteralData(),
590                                    FormatTok->Tok.getLength()) == "0") ||
591                         FormatTok->Tok.is(tok::kw_false);
592   conditionalCompilationStart(!IfDef && IsLiteralFalse);
593   parsePPUnknown();
594 }
595 
596 void UnwrappedLineParser::parsePPElse() {
597   conditionalCompilationAlternative();
598   parsePPUnknown();
599 }
600 
601 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
602 
603 void UnwrappedLineParser::parsePPEndIf() {
604   conditionalCompilationEnd();
605   parsePPUnknown();
606 }
607 
608 void UnwrappedLineParser::parsePPDefine() {
609   nextToken();
610 
611   if (FormatTok->Tok.getKind() != tok::identifier) {
612     parsePPUnknown();
613     return;
614   }
615   nextToken();
616   if (FormatTok->Tok.getKind() == tok::l_paren &&
617       FormatTok->WhitespaceRange.getBegin() ==
618           FormatTok->WhitespaceRange.getEnd()) {
619     parseParens();
620   }
621   addUnwrappedLine();
622   Line->Level = 1;
623 
624   // Errors during a preprocessor directive can only affect the layout of the
625   // preprocessor directive, and thus we ignore them. An alternative approach
626   // would be to use the same approach we use on the file level (no
627   // re-indentation if there was a structural error) within the macro
628   // definition.
629   parseFile();
630 }
631 
632 void UnwrappedLineParser::parsePPUnknown() {
633   do {
634     nextToken();
635   } while (!eof());
636   addUnwrappedLine();
637 }
638 
639 // Here we blacklist certain tokens that are not usually the first token in an
640 // unwrapped line. This is used in attempt to distinguish macro calls without
641 // trailing semicolons from other constructs split to several lines.
642 static bool tokenCanStartNewLine(const clang::Token &Tok) {
643   // Semicolon can be a null-statement, l_square can be a start of a macro or
644   // a C++11 attribute, but this doesn't seem to be common.
645   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
646          Tok.isNot(tok::l_square) &&
647          // Tokens that can only be used as binary operators and a part of
648          // overloaded operator names.
649          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
650          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
651          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
652          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
653          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
654          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
655          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
656          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
657          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
658          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
659          Tok.isNot(tok::lesslessequal) &&
660          // Colon is used in labels, base class lists, initializer lists,
661          // range-based for loops, ternary operator, but should never be the
662          // first token in an unwrapped line.
663          Tok.isNot(tok::colon) &&
664          // 'noexcept' is a trailing annotation.
665          Tok.isNot(tok::kw_noexcept);
666 }
667 
668 static bool mustBeJSIdent(const AdditionalKeywords &Keywords,
669                           const FormatToken *FormatTok) {
670   // FIXME: This returns true for C/C++ keywords like 'struct'.
671   return FormatTok->is(tok::identifier) &&
672          (FormatTok->Tok.getIdentifierInfo() == nullptr ||
673           !FormatTok->isOneOf(
674               Keywords.kw_in, Keywords.kw_of, Keywords.kw_as, Keywords.kw_async,
675               Keywords.kw_await, Keywords.kw_yield, Keywords.kw_finally,
676               Keywords.kw_function, Keywords.kw_import, Keywords.kw_is,
677               Keywords.kw_let, Keywords.kw_var, tok::kw_const,
678               Keywords.kw_abstract, Keywords.kw_extends, Keywords.kw_implements,
679               Keywords.kw_instanceof, Keywords.kw_interface,
680               Keywords.kw_throws));
681 }
682 
683 static bool mustBeJSIdentOrValue(const AdditionalKeywords &Keywords,
684                                  const FormatToken *FormatTok) {
685   return FormatTok->Tok.isLiteral() ||
686          FormatTok->isOneOf(tok::kw_true, tok::kw_false) ||
687          mustBeJSIdent(Keywords, FormatTok);
688 }
689 
690 // isJSDeclOrStmt returns true if |FormatTok| starts a declaration or statement
691 // when encountered after a value (see mustBeJSIdentOrValue).
692 static bool isJSDeclOrStmt(const AdditionalKeywords &Keywords,
693                            const FormatToken *FormatTok) {
694   return FormatTok->isOneOf(
695       tok::kw_return, Keywords.kw_yield,
696       // conditionals
697       tok::kw_if, tok::kw_else,
698       // loops
699       tok::kw_for, tok::kw_while, tok::kw_do, tok::kw_continue, tok::kw_break,
700       // switch/case
701       tok::kw_switch, tok::kw_case,
702       // exceptions
703       tok::kw_throw, tok::kw_try, tok::kw_catch, Keywords.kw_finally,
704       // declaration
705       tok::kw_const, tok::kw_class, Keywords.kw_var, Keywords.kw_let,
706       Keywords.kw_async, Keywords.kw_function,
707       // import/export
708       Keywords.kw_import, tok::kw_export);
709 }
710 
711 // readTokenWithJavaScriptASI reads the next token and terminates the current
712 // line if JavaScript Automatic Semicolon Insertion must
713 // happen between the current token and the next token.
714 //
715 // This method is conservative - it cannot cover all edge cases of JavaScript,
716 // but only aims to correctly handle certain well known cases. It *must not*
717 // return true in speculative cases.
718 void UnwrappedLineParser::readTokenWithJavaScriptASI() {
719   FormatToken *Previous = FormatTok;
720   readToken();
721   FormatToken *Next = FormatTok;
722 
723   bool IsOnSameLine =
724       CommentsBeforeNextToken.empty()
725           ? Next->NewlinesBefore == 0
726           : CommentsBeforeNextToken.front()->NewlinesBefore == 0;
727   if (IsOnSameLine)
728     return;
729 
730   bool PreviousMustBeValue = mustBeJSIdentOrValue(Keywords, Previous);
731   bool PreviousStartsTemplateExpr =
732       Previous->is(TT_TemplateString) && Previous->TokenText.endswith("${");
733   if (PreviousMustBeValue && Line && Line->Tokens.size() > 1) {
734     // If the token before the previous one is an '@', the previous token is an
735     // annotation and can precede another identifier/value.
736     const FormatToken *PrePrevious = std::prev(Line->Tokens.end(), 2)->Tok;
737     if (PrePrevious->is(tok::at))
738       return;
739   }
740   if (Next->is(tok::exclaim) && PreviousMustBeValue)
741     return addUnwrappedLine();
742   bool NextMustBeValue = mustBeJSIdentOrValue(Keywords, Next);
743   bool NextEndsTemplateExpr =
744       Next->is(TT_TemplateString) && Next->TokenText.startswith("}");
745   if (NextMustBeValue && !NextEndsTemplateExpr && !PreviousStartsTemplateExpr &&
746       (PreviousMustBeValue ||
747        Previous->isOneOf(tok::r_square, tok::r_paren, tok::plusplus,
748                          tok::minusminus)))
749     return addUnwrappedLine();
750   if (PreviousMustBeValue && isJSDeclOrStmt(Keywords, Next))
751     return addUnwrappedLine();
752 }
753 
754 void UnwrappedLineParser::parseStructuralElement() {
755   assert(!FormatTok->is(tok::l_brace));
756   if (Style.Language == FormatStyle::LK_TableGen &&
757       FormatTok->is(tok::pp_include)) {
758     nextToken();
759     if (FormatTok->is(tok::string_literal))
760       nextToken();
761     addUnwrappedLine();
762     return;
763   }
764   switch (FormatTok->Tok.getKind()) {
765   case tok::at:
766     nextToken();
767     if (FormatTok->Tok.is(tok::l_brace)) {
768       parseBracedList();
769       break;
770     }
771     switch (FormatTok->Tok.getObjCKeywordID()) {
772     case tok::objc_public:
773     case tok::objc_protected:
774     case tok::objc_package:
775     case tok::objc_private:
776       return parseAccessSpecifier();
777     case tok::objc_interface:
778     case tok::objc_implementation:
779       return parseObjCInterfaceOrImplementation();
780     case tok::objc_protocol:
781       return parseObjCProtocol();
782     case tok::objc_end:
783       return; // Handled by the caller.
784     case tok::objc_optional:
785     case tok::objc_required:
786       nextToken();
787       addUnwrappedLine();
788       return;
789     case tok::objc_autoreleasepool:
790       nextToken();
791       if (FormatTok->Tok.is(tok::l_brace)) {
792         if (Style.BraceWrapping.AfterObjCDeclaration)
793           addUnwrappedLine();
794         parseBlock(/*MustBeDeclaration=*/false);
795       }
796       addUnwrappedLine();
797       return;
798     case tok::objc_try:
799       // This branch isn't strictly necessary (the kw_try case below would
800       // do this too after the tok::at is parsed above).  But be explicit.
801       parseTryCatch();
802       return;
803     default:
804       break;
805     }
806     break;
807   case tok::kw_asm:
808     nextToken();
809     if (FormatTok->is(tok::l_brace)) {
810       FormatTok->Type = TT_InlineASMBrace;
811       nextToken();
812       while (FormatTok && FormatTok->isNot(tok::eof)) {
813         if (FormatTok->is(tok::r_brace)) {
814           FormatTok->Type = TT_InlineASMBrace;
815           nextToken();
816           addUnwrappedLine();
817           break;
818         }
819         FormatTok->Finalized = true;
820         nextToken();
821       }
822     }
823     break;
824   case tok::kw_namespace:
825     parseNamespace();
826     return;
827   case tok::kw_inline:
828     nextToken();
829     if (FormatTok->Tok.is(tok::kw_namespace)) {
830       parseNamespace();
831       return;
832     }
833     break;
834   case tok::kw_public:
835   case tok::kw_protected:
836   case tok::kw_private:
837     if (Style.Language == FormatStyle::LK_Java ||
838         Style.Language == FormatStyle::LK_JavaScript)
839       nextToken();
840     else
841       parseAccessSpecifier();
842     return;
843   case tok::kw_if:
844     parseIfThenElse();
845     return;
846   case tok::kw_for:
847   case tok::kw_while:
848     parseForOrWhileLoop();
849     return;
850   case tok::kw_do:
851     parseDoWhile();
852     return;
853   case tok::kw_switch:
854     parseSwitch();
855     return;
856   case tok::kw_default:
857     nextToken();
858     parseLabel();
859     return;
860   case tok::kw_case:
861     parseCaseLabel();
862     return;
863   case tok::kw_try:
864   case tok::kw___try:
865     parseTryCatch();
866     return;
867   case tok::kw_extern:
868     nextToken();
869     if (FormatTok->Tok.is(tok::string_literal)) {
870       nextToken();
871       if (FormatTok->Tok.is(tok::l_brace)) {
872         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
873         addUnwrappedLine();
874         return;
875       }
876     }
877     break;
878   case tok::kw_export:
879     if (Style.Language == FormatStyle::LK_JavaScript) {
880       parseJavaScriptEs6ImportExport();
881       return;
882     }
883     break;
884   case tok::identifier:
885     if (FormatTok->is(TT_ForEachMacro)) {
886       parseForOrWhileLoop();
887       return;
888     }
889     if (FormatTok->is(TT_MacroBlockBegin)) {
890       parseBlock(/*MustBeDeclaration=*/false, /*AddLevel=*/true,
891                  /*MunchSemi=*/false);
892       return;
893     }
894     if (FormatTok->is(Keywords.kw_import)) {
895       if (Style.Language == FormatStyle::LK_JavaScript) {
896         parseJavaScriptEs6ImportExport();
897         return;
898       }
899       if (Style.Language == FormatStyle::LK_Proto) {
900         nextToken();
901         if (FormatTok->is(tok::kw_public))
902           nextToken();
903         if (!FormatTok->is(tok::string_literal))
904           return;
905         nextToken();
906         if (FormatTok->is(tok::semi))
907           nextToken();
908         addUnwrappedLine();
909         return;
910       }
911     }
912     if (FormatTok->isOneOf(Keywords.kw_signals, Keywords.kw_qsignals,
913                            Keywords.kw_slots, Keywords.kw_qslots)) {
914       nextToken();
915       if (FormatTok->is(tok::colon)) {
916         nextToken();
917         addUnwrappedLine();
918         return;
919       }
920     }
921     // In all other cases, parse the declaration.
922     break;
923   default:
924     break;
925   }
926   do {
927     const FormatToken *Previous = getPreviousToken();
928     switch (FormatTok->Tok.getKind()) {
929     case tok::at:
930       nextToken();
931       if (FormatTok->Tok.is(tok::l_brace))
932         parseBracedList();
933       break;
934     case tok::kw_enum:
935       // Ignore if this is part of "template <enum ...".
936       if (Previous && Previous->is(tok::less)) {
937         nextToken();
938         break;
939       }
940 
941       // parseEnum falls through and does not yet add an unwrapped line as an
942       // enum definition can start a structural element.
943       if (!parseEnum())
944         break;
945       // This only applies for C++.
946       if (Style.Language != FormatStyle::LK_Cpp) {
947         addUnwrappedLine();
948         return;
949       }
950       break;
951     case tok::kw_typedef:
952       nextToken();
953       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
954                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
955         parseEnum();
956       break;
957     case tok::kw_struct:
958     case tok::kw_union:
959     case tok::kw_class:
960       // parseRecord falls through and does not yet add an unwrapped line as a
961       // record declaration or definition can start a structural element.
962       parseRecord();
963       // This does not apply for Java and JavaScript.
964       if (Style.Language == FormatStyle::LK_Java ||
965           Style.Language == FormatStyle::LK_JavaScript) {
966         if (FormatTok->is(tok::semi))
967           nextToken();
968         addUnwrappedLine();
969         return;
970       }
971       break;
972     case tok::period:
973       nextToken();
974       // In Java, classes have an implicit static member "class".
975       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
976           FormatTok->is(tok::kw_class))
977         nextToken();
978       if (Style.Language == FormatStyle::LK_JavaScript && FormatTok &&
979           FormatTok->Tok.getIdentifierInfo())
980         // JavaScript only has pseudo keywords, all keywords are allowed to
981         // appear in "IdentifierName" positions. See http://es5.github.io/#x7.6
982         nextToken();
983       break;
984     case tok::semi:
985       nextToken();
986       addUnwrappedLine();
987       return;
988     case tok::r_brace:
989       addUnwrappedLine();
990       return;
991     case tok::l_paren:
992       parseParens();
993       break;
994     case tok::kw_operator:
995       nextToken();
996       if (FormatTok->isBinaryOperator())
997         nextToken();
998       break;
999     case tok::caret:
1000       nextToken();
1001       if (FormatTok->Tok.isAnyIdentifier() ||
1002           FormatTok->isSimpleTypeSpecifier())
1003         nextToken();
1004       if (FormatTok->is(tok::l_paren))
1005         parseParens();
1006       if (FormatTok->is(tok::l_brace))
1007         parseChildBlock();
1008       break;
1009     case tok::l_brace:
1010       if (!tryToParseBracedList()) {
1011         // A block outside of parentheses must be the last part of a
1012         // structural element.
1013         // FIXME: Figure out cases where this is not true, and add projections
1014         // for them (the one we know is missing are lambdas).
1015         if (Style.BraceWrapping.AfterFunction)
1016           addUnwrappedLine();
1017         FormatTok->Type = TT_FunctionLBrace;
1018         parseBlock(/*MustBeDeclaration=*/false);
1019         addUnwrappedLine();
1020         return;
1021       }
1022       // Otherwise this was a braced init list, and the structural
1023       // element continues.
1024       break;
1025     case tok::kw_try:
1026       // We arrive here when parsing function-try blocks.
1027       parseTryCatch();
1028       return;
1029     case tok::identifier: {
1030       if (FormatTok->is(TT_MacroBlockEnd)) {
1031         addUnwrappedLine();
1032         return;
1033       }
1034 
1035       // Parse function literal unless 'function' is the first token in a line
1036       // in which case this should be treated as a free-standing function.
1037       if (Style.Language == FormatStyle::LK_JavaScript &&
1038           (FormatTok->is(Keywords.kw_function) ||
1039            FormatTok->startsSequence(Keywords.kw_async,
1040                                      Keywords.kw_function)) &&
1041           Line->Tokens.size() > 0) {
1042         tryToParseJSFunction();
1043         break;
1044       }
1045       if ((Style.Language == FormatStyle::LK_JavaScript ||
1046            Style.Language == FormatStyle::LK_Java) &&
1047           FormatTok->is(Keywords.kw_interface)) {
1048         if (Style.Language == FormatStyle::LK_JavaScript) {
1049           // In JavaScript/TypeScript, "interface" can be used as a standalone
1050           // identifier, e.g. in `var interface = 1;`. If "interface" is
1051           // followed by another identifier, it is very like to be an actual
1052           // interface declaration.
1053           unsigned StoredPosition = Tokens->getPosition();
1054           FormatToken *Next = Tokens->getNextToken();
1055           FormatTok = Tokens->setPosition(StoredPosition);
1056           if (Next && !mustBeJSIdent(Keywords, Next)) {
1057             nextToken();
1058             break;
1059           }
1060         }
1061         parseRecord();
1062         addUnwrappedLine();
1063         return;
1064       }
1065 
1066       // See if the following token should start a new unwrapped line.
1067       StringRef Text = FormatTok->TokenText;
1068       nextToken();
1069       if (Line->Tokens.size() == 1 &&
1070           // JS doesn't have macros, and within classes colons indicate fields,
1071           // not labels.
1072           Style.Language != FormatStyle::LK_JavaScript) {
1073         if (FormatTok->Tok.is(tok::colon) && !Line->MustBeDeclaration) {
1074           Line->Tokens.begin()->Tok->MustBreakBefore = true;
1075           parseLabel();
1076           return;
1077         }
1078         // Recognize function-like macro usages without trailing semicolon as
1079         // well as free-standing macros like Q_OBJECT.
1080         bool FunctionLike = FormatTok->is(tok::l_paren);
1081         if (FunctionLike)
1082           parseParens();
1083 
1084         bool FollowedByNewline =
1085             CommentsBeforeNextToken.empty()
1086                 ? FormatTok->NewlinesBefore > 0
1087                 : CommentsBeforeNextToken.front()->NewlinesBefore > 0;
1088 
1089         if (FollowedByNewline && (Text.size() >= 5 || FunctionLike) &&
1090             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
1091           addUnwrappedLine();
1092           return;
1093         }
1094       }
1095       break;
1096     }
1097     case tok::equal:
1098       // Fat arrows (=>) have tok::TokenKind tok::equal but TokenType
1099       // TT_JsFatArrow. The always start an expression or a child block if
1100       // followed by a curly.
1101       if (FormatTok->is(TT_JsFatArrow)) {
1102         nextToken();
1103         if (FormatTok->is(tok::l_brace))
1104           parseChildBlock();
1105         break;
1106       }
1107 
1108       nextToken();
1109       if (FormatTok->Tok.is(tok::l_brace)) {
1110         parseBracedList();
1111       }
1112       break;
1113     case tok::l_square:
1114       parseSquare();
1115       break;
1116     case tok::kw_new:
1117       parseNew();
1118       break;
1119     default:
1120       nextToken();
1121       break;
1122     }
1123   } while (!eof());
1124 }
1125 
1126 bool UnwrappedLineParser::tryToParseLambda() {
1127   if (Style.Language != FormatStyle::LK_Cpp) {
1128     nextToken();
1129     return false;
1130   }
1131   const FormatToken* Previous = getPreviousToken();
1132   if (Previous &&
1133       (Previous->isOneOf(tok::identifier, tok::kw_operator, tok::kw_new,
1134                          tok::kw_delete) ||
1135        Previous->closesScope() || Previous->isSimpleTypeSpecifier())) {
1136     nextToken();
1137     return false;
1138   }
1139   assert(FormatTok->is(tok::l_square));
1140   FormatToken &LSquare = *FormatTok;
1141   if (!tryToParseLambdaIntroducer())
1142     return false;
1143 
1144   while (FormatTok->isNot(tok::l_brace)) {
1145     if (FormatTok->isSimpleTypeSpecifier()) {
1146       nextToken();
1147       continue;
1148     }
1149     switch (FormatTok->Tok.getKind()) {
1150     case tok::l_brace:
1151       break;
1152     case tok::l_paren:
1153       parseParens();
1154       break;
1155     case tok::amp:
1156     case tok::star:
1157     case tok::kw_const:
1158     case tok::comma:
1159     case tok::less:
1160     case tok::greater:
1161     case tok::identifier:
1162     case tok::numeric_constant:
1163     case tok::coloncolon:
1164     case tok::kw_mutable:
1165       nextToken();
1166       break;
1167     case tok::arrow:
1168       FormatTok->Type = TT_LambdaArrow;
1169       nextToken();
1170       break;
1171     default:
1172       return true;
1173     }
1174   }
1175   LSquare.Type = TT_LambdaLSquare;
1176   parseChildBlock();
1177   return true;
1178 }
1179 
1180 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
1181   nextToken();
1182   if (FormatTok->is(tok::equal)) {
1183     nextToken();
1184     if (FormatTok->is(tok::r_square)) {
1185       nextToken();
1186       return true;
1187     }
1188     if (FormatTok->isNot(tok::comma))
1189       return false;
1190     nextToken();
1191   } else if (FormatTok->is(tok::amp)) {
1192     nextToken();
1193     if (FormatTok->is(tok::r_square)) {
1194       nextToken();
1195       return true;
1196     }
1197     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
1198       return false;
1199     }
1200     if (FormatTok->is(tok::comma))
1201       nextToken();
1202   } else if (FormatTok->is(tok::r_square)) {
1203     nextToken();
1204     return true;
1205   }
1206   do {
1207     if (FormatTok->is(tok::amp))
1208       nextToken();
1209     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
1210       return false;
1211     nextToken();
1212     if (FormatTok->is(tok::ellipsis))
1213       nextToken();
1214     if (FormatTok->is(tok::comma)) {
1215       nextToken();
1216     } else if (FormatTok->is(tok::r_square)) {
1217       nextToken();
1218       return true;
1219     } else {
1220       return false;
1221     }
1222   } while (!eof());
1223   return false;
1224 }
1225 
1226 void UnwrappedLineParser::tryToParseJSFunction() {
1227   assert(FormatTok->is(Keywords.kw_function) ||
1228          FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function));
1229   if (FormatTok->is(Keywords.kw_async))
1230     nextToken();
1231   // Consume "function".
1232   nextToken();
1233 
1234   // Consume * (generator function). Treat it like C++'s overloaded operators.
1235   if (FormatTok->is(tok::star)) {
1236     FormatTok->Type = TT_OverloadedOperator;
1237     nextToken();
1238   }
1239 
1240   // Consume function name.
1241   if (FormatTok->is(tok::identifier))
1242     nextToken();
1243 
1244   if (FormatTok->isNot(tok::l_paren))
1245     return;
1246 
1247   // Parse formal parameter list.
1248   parseParens();
1249 
1250   if (FormatTok->is(tok::colon)) {
1251     // Parse a type definition.
1252     nextToken();
1253 
1254     // Eat the type declaration. For braced inline object types, balance braces,
1255     // otherwise just parse until finding an l_brace for the function body.
1256     if (FormatTok->is(tok::l_brace))
1257       tryToParseBracedList();
1258     else
1259       while (!FormatTok->isOneOf(tok::l_brace, tok::semi) && !eof())
1260         nextToken();
1261   }
1262 
1263   if (FormatTok->is(tok::semi))
1264     return;
1265 
1266   parseChildBlock();
1267 }
1268 
1269 bool UnwrappedLineParser::tryToParseBracedList() {
1270   if (FormatTok->BlockKind == BK_Unknown)
1271     calculateBraceTypes();
1272   assert(FormatTok->BlockKind != BK_Unknown);
1273   if (FormatTok->BlockKind == BK_Block)
1274     return false;
1275   parseBracedList();
1276   return true;
1277 }
1278 
1279 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
1280   bool HasError = false;
1281   nextToken();
1282 
1283   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
1284   // replace this by using parseAssigmentExpression() inside.
1285   do {
1286     if (Style.Language == FormatStyle::LK_JavaScript) {
1287       if (FormatTok->is(Keywords.kw_function) ||
1288           FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)) {
1289         tryToParseJSFunction();
1290         continue;
1291       }
1292       if (FormatTok->is(TT_JsFatArrow)) {
1293         nextToken();
1294         // Fat arrows can be followed by simple expressions or by child blocks
1295         // in curly braces.
1296         if (FormatTok->is(tok::l_brace)) {
1297           parseChildBlock();
1298           continue;
1299         }
1300       }
1301     }
1302     switch (FormatTok->Tok.getKind()) {
1303     case tok::caret:
1304       nextToken();
1305       if (FormatTok->is(tok::l_brace)) {
1306         parseChildBlock();
1307       }
1308       break;
1309     case tok::l_square:
1310       tryToParseLambda();
1311       break;
1312     case tok::l_brace:
1313       // Assume there are no blocks inside a braced init list apart
1314       // from the ones we explicitly parse out (like lambdas).
1315       FormatTok->BlockKind = BK_BracedInit;
1316       parseBracedList();
1317       break;
1318     case tok::l_paren:
1319       parseParens();
1320       // JavaScript can just have free standing methods and getters/setters in
1321       // object literals. Detect them by a "{" following ")".
1322       if (Style.Language == FormatStyle::LK_JavaScript) {
1323         if (FormatTok->is(tok::l_brace))
1324           parseChildBlock();
1325         break;
1326       }
1327       break;
1328     case tok::r_brace:
1329       nextToken();
1330       return !HasError;
1331     case tok::semi:
1332       // JavaScript (or more precisely TypeScript) can have semicolons in braced
1333       // lists (in so-called TypeMemberLists). Thus, the semicolon cannot be
1334       // used for error recovery if we have otherwise determined that this is
1335       // a braced list.
1336       if (Style.Language == FormatStyle::LK_JavaScript) {
1337         nextToken();
1338         break;
1339       }
1340       HasError = true;
1341       if (!ContinueOnSemicolons)
1342         return !HasError;
1343       nextToken();
1344       break;
1345     case tok::comma:
1346       nextToken();
1347       break;
1348     default:
1349       nextToken();
1350       break;
1351     }
1352   } while (!eof());
1353   return false;
1354 }
1355 
1356 void UnwrappedLineParser::parseParens() {
1357   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1358   nextToken();
1359   do {
1360     switch (FormatTok->Tok.getKind()) {
1361     case tok::l_paren:
1362       parseParens();
1363       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1364         parseChildBlock();
1365       break;
1366     case tok::r_paren:
1367       nextToken();
1368       return;
1369     case tok::r_brace:
1370       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1371       return;
1372     case tok::l_square:
1373       tryToParseLambda();
1374       break;
1375     case tok::l_brace:
1376       if (!tryToParseBracedList())
1377         parseChildBlock();
1378       break;
1379     case tok::at:
1380       nextToken();
1381       if (FormatTok->Tok.is(tok::l_brace))
1382         parseBracedList();
1383       break;
1384     case tok::identifier:
1385       if (Style.Language == FormatStyle::LK_JavaScript &&
1386           (FormatTok->is(Keywords.kw_function) ||
1387            FormatTok->startsSequence(Keywords.kw_async, Keywords.kw_function)))
1388         tryToParseJSFunction();
1389       else
1390         nextToken();
1391       break;
1392     default:
1393       nextToken();
1394       break;
1395     }
1396   } while (!eof());
1397 }
1398 
1399 void UnwrappedLineParser::parseSquare() {
1400   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1401   if (tryToParseLambda())
1402     return;
1403   do {
1404     switch (FormatTok->Tok.getKind()) {
1405     case tok::l_paren:
1406       parseParens();
1407       break;
1408     case tok::r_square:
1409       nextToken();
1410       return;
1411     case tok::r_brace:
1412       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1413       return;
1414     case tok::l_square:
1415       parseSquare();
1416       break;
1417     case tok::l_brace: {
1418       if (!tryToParseBracedList())
1419         parseChildBlock();
1420       break;
1421     }
1422     case tok::at:
1423       nextToken();
1424       if (FormatTok->Tok.is(tok::l_brace))
1425         parseBracedList();
1426       break;
1427     default:
1428       nextToken();
1429       break;
1430     }
1431   } while (!eof());
1432 }
1433 
1434 void UnwrappedLineParser::parseIfThenElse() {
1435   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1436   nextToken();
1437   if (FormatTok->Tok.is(tok::l_paren))
1438     parseParens();
1439   bool NeedsUnwrappedLine = false;
1440   if (FormatTok->Tok.is(tok::l_brace)) {
1441     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1442     parseBlock(/*MustBeDeclaration=*/false);
1443     if (Style.BraceWrapping.BeforeElse)
1444       addUnwrappedLine();
1445     else
1446       NeedsUnwrappedLine = true;
1447   } else {
1448     addUnwrappedLine();
1449     ++Line->Level;
1450     parseStructuralElement();
1451     --Line->Level;
1452   }
1453   if (FormatTok->Tok.is(tok::kw_else)) {
1454     nextToken();
1455     if (FormatTok->Tok.is(tok::l_brace)) {
1456       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1457       parseBlock(/*MustBeDeclaration=*/false);
1458       addUnwrappedLine();
1459     } else if (FormatTok->Tok.is(tok::kw_if)) {
1460       parseIfThenElse();
1461     } else {
1462       addUnwrappedLine();
1463       ++Line->Level;
1464       parseStructuralElement();
1465       if (FormatTok->is(tok::eof))
1466         addUnwrappedLine();
1467       --Line->Level;
1468     }
1469   } else if (NeedsUnwrappedLine) {
1470     addUnwrappedLine();
1471   }
1472 }
1473 
1474 void UnwrappedLineParser::parseTryCatch() {
1475   assert(FormatTok->isOneOf(tok::kw_try, tok::kw___try) && "'try' expected");
1476   nextToken();
1477   bool NeedsUnwrappedLine = false;
1478   if (FormatTok->is(tok::colon)) {
1479     // We are in a function try block, what comes is an initializer list.
1480     nextToken();
1481     while (FormatTok->is(tok::identifier)) {
1482       nextToken();
1483       if (FormatTok->is(tok::l_paren))
1484         parseParens();
1485       if (FormatTok->is(tok::comma))
1486         nextToken();
1487     }
1488   }
1489   // Parse try with resource.
1490   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1491     parseParens();
1492   }
1493   if (FormatTok->is(tok::l_brace)) {
1494     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1495     parseBlock(/*MustBeDeclaration=*/false);
1496     if (Style.BraceWrapping.BeforeCatch) {
1497       addUnwrappedLine();
1498     } else {
1499       NeedsUnwrappedLine = true;
1500     }
1501   } else if (!FormatTok->is(tok::kw_catch)) {
1502     // The C++ standard requires a compound-statement after a try.
1503     // If there's none, we try to assume there's a structuralElement
1504     // and try to continue.
1505     addUnwrappedLine();
1506     ++Line->Level;
1507     parseStructuralElement();
1508     --Line->Level;
1509   }
1510   while (1) {
1511     if (FormatTok->is(tok::at))
1512       nextToken();
1513     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
1514                              tok::kw___finally) ||
1515           ((Style.Language == FormatStyle::LK_Java ||
1516             Style.Language == FormatStyle::LK_JavaScript) &&
1517            FormatTok->is(Keywords.kw_finally)) ||
1518           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
1519            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
1520       break;
1521     nextToken();
1522     while (FormatTok->isNot(tok::l_brace)) {
1523       if (FormatTok->is(tok::l_paren)) {
1524         parseParens();
1525         continue;
1526       }
1527       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1528         return;
1529       nextToken();
1530     }
1531     NeedsUnwrappedLine = false;
1532     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1533     parseBlock(/*MustBeDeclaration=*/false);
1534     if (Style.BraceWrapping.BeforeCatch)
1535       addUnwrappedLine();
1536     else
1537       NeedsUnwrappedLine = true;
1538   }
1539   if (NeedsUnwrappedLine)
1540     addUnwrappedLine();
1541 }
1542 
1543 void UnwrappedLineParser::parseNamespace() {
1544   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1545 
1546   const FormatToken &InitialToken = *FormatTok;
1547   nextToken();
1548   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon))
1549     nextToken();
1550   if (FormatTok->Tok.is(tok::l_brace)) {
1551     if (ShouldBreakBeforeBrace(Style, InitialToken))
1552       addUnwrappedLine();
1553 
1554     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1555                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1556                      DeclarationScopeStack.size() > 1);
1557     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1558     // Munch the semicolon after a namespace. This is more common than one would
1559     // think. Puttin the semicolon into its own line is very ugly.
1560     if (FormatTok->Tok.is(tok::semi))
1561       nextToken();
1562     addUnwrappedLine();
1563   }
1564   // FIXME: Add error handling.
1565 }
1566 
1567 void UnwrappedLineParser::parseNew() {
1568   assert(FormatTok->is(tok::kw_new) && "'new' expected");
1569   nextToken();
1570   if (Style.Language != FormatStyle::LK_Java)
1571     return;
1572 
1573   // In Java, we can parse everything up to the parens, which aren't optional.
1574   do {
1575     // There should not be a ;, { or } before the new's open paren.
1576     if (FormatTok->isOneOf(tok::semi, tok::l_brace, tok::r_brace))
1577       return;
1578 
1579     // Consume the parens.
1580     if (FormatTok->is(tok::l_paren)) {
1581       parseParens();
1582 
1583       // If there is a class body of an anonymous class, consume that as child.
1584       if (FormatTok->is(tok::l_brace))
1585         parseChildBlock();
1586       return;
1587     }
1588     nextToken();
1589   } while (!eof());
1590 }
1591 
1592 void UnwrappedLineParser::parseForOrWhileLoop() {
1593   assert(FormatTok->isOneOf(tok::kw_for, tok::kw_while, TT_ForEachMacro) &&
1594          "'for', 'while' or foreach macro expected");
1595   nextToken();
1596   if (FormatTok->Tok.is(tok::l_paren))
1597     parseParens();
1598   if (FormatTok->Tok.is(tok::l_brace)) {
1599     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1600     parseBlock(/*MustBeDeclaration=*/false);
1601     addUnwrappedLine();
1602   } else {
1603     addUnwrappedLine();
1604     ++Line->Level;
1605     parseStructuralElement();
1606     --Line->Level;
1607   }
1608 }
1609 
1610 void UnwrappedLineParser::parseDoWhile() {
1611   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1612   nextToken();
1613   if (FormatTok->Tok.is(tok::l_brace)) {
1614     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1615     parseBlock(/*MustBeDeclaration=*/false);
1616     if (Style.BraceWrapping.IndentBraces)
1617       addUnwrappedLine();
1618   } else {
1619     addUnwrappedLine();
1620     ++Line->Level;
1621     parseStructuralElement();
1622     --Line->Level;
1623   }
1624 
1625   // FIXME: Add error handling.
1626   if (!FormatTok->Tok.is(tok::kw_while)) {
1627     addUnwrappedLine();
1628     return;
1629   }
1630 
1631   nextToken();
1632   parseStructuralElement();
1633 }
1634 
1635 void UnwrappedLineParser::parseLabel() {
1636   nextToken();
1637   unsigned OldLineLevel = Line->Level;
1638   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1639     --Line->Level;
1640   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1641     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1642     parseBlock(/*MustBeDeclaration=*/false);
1643     if (FormatTok->Tok.is(tok::kw_break)) {
1644       if (Style.BraceWrapping.AfterControlStatement)
1645         addUnwrappedLine();
1646       parseStructuralElement();
1647     }
1648     addUnwrappedLine();
1649   } else {
1650     if (FormatTok->is(tok::semi))
1651       nextToken();
1652     addUnwrappedLine();
1653   }
1654   Line->Level = OldLineLevel;
1655   if (FormatTok->isNot(tok::l_brace)) {
1656     parseStructuralElement();
1657     addUnwrappedLine();
1658   }
1659 }
1660 
1661 void UnwrappedLineParser::parseCaseLabel() {
1662   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1663   // FIXME: fix handling of complex expressions here.
1664   do {
1665     nextToken();
1666   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1667   parseLabel();
1668 }
1669 
1670 void UnwrappedLineParser::parseSwitch() {
1671   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1672   nextToken();
1673   if (FormatTok->Tok.is(tok::l_paren))
1674     parseParens();
1675   if (FormatTok->Tok.is(tok::l_brace)) {
1676     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1677     parseBlock(/*MustBeDeclaration=*/false);
1678     addUnwrappedLine();
1679   } else {
1680     addUnwrappedLine();
1681     ++Line->Level;
1682     parseStructuralElement();
1683     --Line->Level;
1684   }
1685 }
1686 
1687 void UnwrappedLineParser::parseAccessSpecifier() {
1688   nextToken();
1689   // Understand Qt's slots.
1690   if (FormatTok->isOneOf(Keywords.kw_slots, Keywords.kw_qslots))
1691     nextToken();
1692   // Otherwise, we don't know what it is, and we'd better keep the next token.
1693   if (FormatTok->Tok.is(tok::colon))
1694     nextToken();
1695   addUnwrappedLine();
1696 }
1697 
1698 bool UnwrappedLineParser::parseEnum() {
1699   // Won't be 'enum' for NS_ENUMs.
1700   if (FormatTok->Tok.is(tok::kw_enum))
1701     nextToken();
1702 
1703   // In TypeScript, "enum" can also be used as property name, e.g. in interface
1704   // declarations. An "enum" keyword followed by a colon would be a syntax
1705   // error and thus assume it is just an identifier.
1706   if (Style.Language == FormatStyle::LK_JavaScript &&
1707       FormatTok->isOneOf(tok::colon, tok::question))
1708     return false;
1709 
1710   // Eat up enum class ...
1711   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1712     nextToken();
1713 
1714   while (FormatTok->Tok.getIdentifierInfo() ||
1715          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1716                             tok::greater, tok::comma, tok::question)) {
1717     nextToken();
1718     // We can have macros or attributes in between 'enum' and the enum name.
1719     if (FormatTok->is(tok::l_paren))
1720       parseParens();
1721     if (FormatTok->is(tok::identifier)) {
1722       nextToken();
1723       // If there are two identifiers in a row, this is likely an elaborate
1724       // return type. In Java, this can be "implements", etc.
1725       if (Style.Language == FormatStyle::LK_Cpp &&
1726           FormatTok->is(tok::identifier))
1727         return false;
1728     }
1729   }
1730 
1731   // Just a declaration or something is wrong.
1732   if (FormatTok->isNot(tok::l_brace))
1733     return true;
1734   FormatTok->BlockKind = BK_Block;
1735 
1736   if (Style.Language == FormatStyle::LK_Java) {
1737     // Java enums are different.
1738     parseJavaEnumBody();
1739     return true;
1740   }
1741   if (Style.Language == FormatStyle::LK_Proto) {
1742     parseBlock(/*MustBeDeclaration=*/true);
1743     return true;
1744   }
1745 
1746   // Parse enum body.
1747   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1748   if (HasError) {
1749     if (FormatTok->is(tok::semi))
1750       nextToken();
1751     addUnwrappedLine();
1752   }
1753   return true;
1754 
1755   // There is no addUnwrappedLine() here so that we fall through to parsing a
1756   // structural element afterwards. Thus, in "enum A {} n, m;",
1757   // "} n, m;" will end up in one unwrapped line.
1758 }
1759 
1760 void UnwrappedLineParser::parseJavaEnumBody() {
1761   // Determine whether the enum is simple, i.e. does not have a semicolon or
1762   // constants with class bodies. Simple enums can be formatted like braced
1763   // lists, contracted to a single line, etc.
1764   unsigned StoredPosition = Tokens->getPosition();
1765   bool IsSimple = true;
1766   FormatToken *Tok = Tokens->getNextToken();
1767   while (Tok) {
1768     if (Tok->is(tok::r_brace))
1769       break;
1770     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1771       IsSimple = false;
1772       break;
1773     }
1774     // FIXME: This will also mark enums with braces in the arguments to enum
1775     // constants as "not simple". This is probably fine in practice, though.
1776     Tok = Tokens->getNextToken();
1777   }
1778   FormatTok = Tokens->setPosition(StoredPosition);
1779 
1780   if (IsSimple) {
1781     parseBracedList();
1782     addUnwrappedLine();
1783     return;
1784   }
1785 
1786   // Parse the body of a more complex enum.
1787   // First add a line for everything up to the "{".
1788   nextToken();
1789   addUnwrappedLine();
1790   ++Line->Level;
1791 
1792   // Parse the enum constants.
1793   while (FormatTok) {
1794     if (FormatTok->is(tok::l_brace)) {
1795       // Parse the constant's class body.
1796       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1797                  /*MunchSemi=*/false);
1798     } else if (FormatTok->is(tok::l_paren)) {
1799       parseParens();
1800     } else if (FormatTok->is(tok::comma)) {
1801       nextToken();
1802       addUnwrappedLine();
1803     } else if (FormatTok->is(tok::semi)) {
1804       nextToken();
1805       addUnwrappedLine();
1806       break;
1807     } else if (FormatTok->is(tok::r_brace)) {
1808       addUnwrappedLine();
1809       break;
1810     } else {
1811       nextToken();
1812     }
1813   }
1814 
1815   // Parse the class body after the enum's ";" if any.
1816   parseLevel(/*HasOpeningBrace=*/true);
1817   nextToken();
1818   --Line->Level;
1819   addUnwrappedLine();
1820 }
1821 
1822 void UnwrappedLineParser::parseRecord() {
1823   const FormatToken &InitialToken = *FormatTok;
1824   nextToken();
1825 
1826   // The actual identifier can be a nested name specifier, and in macros
1827   // it is often token-pasted.
1828   while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::hashhash,
1829                             tok::kw___attribute, tok::kw___declspec,
1830                             tok::kw_alignas) ||
1831          ((Style.Language == FormatStyle::LK_Java ||
1832            Style.Language == FormatStyle::LK_JavaScript) &&
1833           FormatTok->isOneOf(tok::period, tok::comma))) {
1834     bool IsNonMacroIdentifier =
1835         FormatTok->is(tok::identifier) &&
1836         FormatTok->TokenText != FormatTok->TokenText.upper();
1837     nextToken();
1838     // We can have macros or attributes in between 'class' and the class name.
1839     if (!IsNonMacroIdentifier && FormatTok->Tok.is(tok::l_paren))
1840       parseParens();
1841   }
1842 
1843   // Note that parsing away template declarations here leads to incorrectly
1844   // accepting function declarations as record declarations.
1845   // In general, we cannot solve this problem. Consider:
1846   // class A<int> B() {}
1847   // which can be a function definition or a class definition when B() is a
1848   // macro. If we find enough real-world cases where this is a problem, we
1849   // can parse for the 'template' keyword in the beginning of the statement,
1850   // and thus rule out the record production in case there is no template
1851   // (this would still leave us with an ambiguity between template function
1852   // and class declarations).
1853   if (FormatTok->isOneOf(tok::colon, tok::less)) {
1854     while (!eof()) {
1855       if (FormatTok->is(tok::l_brace)) {
1856         calculateBraceTypes(/*ExpectClassBody=*/true);
1857         if (!tryToParseBracedList())
1858           break;
1859       }
1860       if (FormatTok->Tok.is(tok::semi))
1861         return;
1862       nextToken();
1863     }
1864   }
1865   if (FormatTok->Tok.is(tok::l_brace)) {
1866     if (ShouldBreakBeforeBrace(Style, InitialToken))
1867       addUnwrappedLine();
1868 
1869     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1870                /*MunchSemi=*/false);
1871   }
1872   // There is no addUnwrappedLine() here so that we fall through to parsing a
1873   // structural element afterwards. Thus, in "class A {} n, m;",
1874   // "} n, m;" will end up in one unwrapped line.
1875 }
1876 
1877 void UnwrappedLineParser::parseObjCProtocolList() {
1878   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1879   do
1880     nextToken();
1881   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1882   nextToken(); // Skip '>'.
1883 }
1884 
1885 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1886   do {
1887     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1888       nextToken();
1889       addUnwrappedLine();
1890       break;
1891     }
1892     if (FormatTok->is(tok::l_brace)) {
1893       parseBlock(/*MustBeDeclaration=*/false);
1894       // In ObjC interfaces, nothing should be following the "}".
1895       addUnwrappedLine();
1896     } else if (FormatTok->is(tok::r_brace)) {
1897       // Ignore stray "}". parseStructuralElement doesn't consume them.
1898       nextToken();
1899       addUnwrappedLine();
1900     } else {
1901       parseStructuralElement();
1902     }
1903   } while (!eof());
1904 }
1905 
1906 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1907   nextToken();
1908   nextToken(); // interface name
1909 
1910   // @interface can be followed by either a base class, or a category.
1911   if (FormatTok->Tok.is(tok::colon)) {
1912     nextToken();
1913     nextToken(); // base class name
1914   } else if (FormatTok->Tok.is(tok::l_paren))
1915     // Skip category, if present.
1916     parseParens();
1917 
1918   if (FormatTok->Tok.is(tok::less))
1919     parseObjCProtocolList();
1920 
1921   if (FormatTok->Tok.is(tok::l_brace)) {
1922     if (Style.BraceWrapping.AfterObjCDeclaration)
1923       addUnwrappedLine();
1924     parseBlock(/*MustBeDeclaration=*/true);
1925   }
1926 
1927   // With instance variables, this puts '}' on its own line.  Without instance
1928   // variables, this ends the @interface line.
1929   addUnwrappedLine();
1930 
1931   parseObjCUntilAtEnd();
1932 }
1933 
1934 void UnwrappedLineParser::parseObjCProtocol() {
1935   nextToken();
1936   nextToken(); // protocol name
1937 
1938   if (FormatTok->Tok.is(tok::less))
1939     parseObjCProtocolList();
1940 
1941   // Check for protocol declaration.
1942   if (FormatTok->Tok.is(tok::semi)) {
1943     nextToken();
1944     return addUnwrappedLine();
1945   }
1946 
1947   addUnwrappedLine();
1948   parseObjCUntilAtEnd();
1949 }
1950 
1951 void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
1952   bool IsImport = FormatTok->is(Keywords.kw_import);
1953   assert(IsImport || FormatTok->is(tok::kw_export));
1954   nextToken();
1955 
1956   // Consume the "default" in "export default class/function".
1957   if (FormatTok->is(tok::kw_default))
1958     nextToken();
1959 
1960   // Consume "async function", "function" and "default function", so that these
1961   // get parsed as free-standing JS functions, i.e. do not require a trailing
1962   // semicolon.
1963   if (FormatTok->is(Keywords.kw_async))
1964     nextToken();
1965   if (FormatTok->is(Keywords.kw_function)) {
1966     nextToken();
1967     return;
1968   }
1969 
1970   // For imports, `export *`, `export {...}`, consume the rest of the line up
1971   // to the terminating `;`. For everything else, just return and continue
1972   // parsing the structural element, i.e. the declaration or expression for
1973   // `export default`.
1974   if (!IsImport && !FormatTok->isOneOf(tok::l_brace, tok::star) &&
1975       !FormatTok->isStringLiteral())
1976     return;
1977 
1978   while (!eof()) {
1979     if (FormatTok->is(tok::semi))
1980       return;
1981     if (Line->Tokens.size() == 0) {
1982       // Common issue: Automatic Semicolon Insertion wrapped the line, so the
1983       // import statement should terminate.
1984       return;
1985     }
1986     if (FormatTok->is(tok::l_brace)) {
1987       FormatTok->BlockKind = BK_Block;
1988       parseBracedList();
1989     } else {
1990       nextToken();
1991     }
1992   }
1993 }
1994 
1995 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1996                                                  StringRef Prefix = "") {
1997   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1998                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1999   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
2000                                                     E = Line.Tokens.end();
2001        I != E; ++I) {
2002     llvm::dbgs() << I->Tok->Tok.getName() << "["
2003                  << "T=" << I->Tok->Type
2004                  << ", OC=" << I->Tok->OriginalColumn << "] ";
2005   }
2006   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
2007                                                     E = Line.Tokens.end();
2008        I != E; ++I) {
2009     const UnwrappedLineNode &Node = *I;
2010     for (SmallVectorImpl<UnwrappedLine>::const_iterator
2011              I = Node.Children.begin(),
2012              E = Node.Children.end();
2013          I != E; ++I) {
2014       printDebugInfo(*I, "\nChild: ");
2015     }
2016   }
2017   llvm::dbgs() << "\n";
2018 }
2019 
2020 void UnwrappedLineParser::addUnwrappedLine() {
2021   if (Line->Tokens.empty())
2022     return;
2023   DEBUG({
2024     if (CurrentLines == &Lines)
2025       printDebugInfo(*Line);
2026   });
2027   CurrentLines->push_back(std::move(*Line));
2028   Line->Tokens.clear();
2029   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
2030     CurrentLines->append(
2031         std::make_move_iterator(PreprocessorDirectives.begin()),
2032         std::make_move_iterator(PreprocessorDirectives.end()));
2033     PreprocessorDirectives.clear();
2034   }
2035 }
2036 
2037 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
2038 
2039 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
2040   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
2041          FormatTok.NewlinesBefore > 0;
2042 }
2043 
2044 static bool isLineComment(const FormatToken &FormatTok) {
2045   return FormatTok.is(tok::comment) &&
2046          FormatTok.TokenText.startswith("//");
2047 }
2048 
2049 // Checks if \p FormatTok is a line comment that continues the line comment
2050 // section on \p Line.
2051 static bool continuesLineComment(const FormatToken &FormatTok,
2052                                  const UnwrappedLine &Line,
2053                                  llvm::Regex &CommentPragmasRegex) {
2054   if (Line.Tokens.empty())
2055     return false;
2056 
2057   StringRef IndentContent = FormatTok.TokenText;
2058   if (FormatTok.TokenText.startswith("//") ||
2059       FormatTok.TokenText.startswith("/*"))
2060     IndentContent = FormatTok.TokenText.substr(2);
2061   if (CommentPragmasRegex.match(IndentContent))
2062     return false;
2063 
2064   // If Line starts with a line comment, then FormatTok continues the comment
2065   // section if its original column is greater or equal to the original start
2066   // column of the line.
2067   //
2068   // Define the min column token of a line as follows: if a line ends in '{' or
2069   // contains a '{' followed by a line comment, then the min column token is
2070   // that '{'. Otherwise, the min column token of the line is the first token of
2071   // the line.
2072   //
2073   // If Line starts with a token other than a line comment, then FormatTok
2074   // continues the comment section if its original column is greater than the
2075   // original start column of the min column token of the line.
2076   //
2077   // For example, the second line comment continues the first in these cases:
2078   //
2079   // // first line
2080   // // second line
2081   //
2082   // and:
2083   //
2084   // // first line
2085   //  // second line
2086   //
2087   // and:
2088   //
2089   // int i; // first line
2090   //  // second line
2091   //
2092   // and:
2093   //
2094   // do { // first line
2095   //      // second line
2096   //   int i;
2097   // } while (true);
2098   //
2099   // and:
2100   //
2101   // enum {
2102   //   a, // first line
2103   //    // second line
2104   //   b
2105   // };
2106   //
2107   // The second line comment doesn't continue the first in these cases:
2108   //
2109   //   // first line
2110   //  // second line
2111   //
2112   // and:
2113   //
2114   // int i; // first line
2115   // // second line
2116   //
2117   // and:
2118   //
2119   // do { // first line
2120   //   // second line
2121   //   int i;
2122   // } while (true);
2123   //
2124   // and:
2125   //
2126   // enum {
2127   //   a, // first line
2128   //   // second line
2129   // };
2130   const FormatToken *MinColumnToken = Line.Tokens.front().Tok;
2131 
2132   // Scan for '{//'. If found, use the column of '{' as a min column for line
2133   // comment section continuation.
2134   const FormatToken *PreviousToken = nullptr;
2135   for (const UnwrappedLineNode Node : Line.Tokens) {
2136     if (PreviousToken && PreviousToken->is(tok::l_brace) &&
2137         isLineComment(*Node.Tok)) {
2138       MinColumnToken = PreviousToken;
2139       break;
2140     }
2141     PreviousToken = Node.Tok;
2142 
2143     // Grab the last newline preceding a token in this unwrapped line.
2144     if (Node.Tok->NewlinesBefore > 0) {
2145       MinColumnToken = Node.Tok;
2146     }
2147   }
2148   if (PreviousToken && PreviousToken->is(tok::l_brace)) {
2149     MinColumnToken = PreviousToken;
2150   }
2151 
2152   unsigned MinContinueColumn =
2153       MinColumnToken->OriginalColumn +
2154       (isLineComment(*MinColumnToken) ? 0 : 1);
2155   return isLineComment(FormatTok) && FormatTok.NewlinesBefore == 1 &&
2156          isLineComment(*(Line.Tokens.back().Tok)) &&
2157          FormatTok.OriginalColumn >= MinContinueColumn;
2158 }
2159 
2160 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
2161   bool JustComments = Line->Tokens.empty();
2162   for (SmallVectorImpl<FormatToken *>::const_iterator
2163            I = CommentsBeforeNextToken.begin(),
2164            E = CommentsBeforeNextToken.end();
2165        I != E; ++I) {
2166     // Line comments that belong to the same line comment section are put on the
2167     // same line since later we might want to reflow content between them.
2168     // Additional fine-grained breaking of line comment sections is controlled
2169     // by the class BreakableLineCommentSection in case it is desirable to keep
2170     // several line comment sections in the same unwrapped line.
2171     //
2172     // FIXME: Consider putting separate line comment sections as children to the
2173     // unwrapped line instead.
2174     (*I)->ContinuesLineCommentSection =
2175         continuesLineComment(**I, *Line, CommentPragmasRegex);
2176     if (isOnNewLine(**I) && JustComments && !(*I)->ContinuesLineCommentSection)
2177       addUnwrappedLine();
2178     pushToken(*I);
2179   }
2180   if (NewlineBeforeNext && JustComments)
2181     addUnwrappedLine();
2182   CommentsBeforeNextToken.clear();
2183 }
2184 
2185 void UnwrappedLineParser::nextToken() {
2186   if (eof())
2187     return;
2188   flushComments(isOnNewLine(*FormatTok));
2189   pushToken(FormatTok);
2190   if (Style.Language != FormatStyle::LK_JavaScript)
2191     readToken();
2192   else
2193     readTokenWithJavaScriptASI();
2194 }
2195 
2196 const FormatToken *UnwrappedLineParser::getPreviousToken() {
2197   // FIXME: This is a dirty way to access the previous token. Find a better
2198   // solution.
2199   if (!Line || Line->Tokens.empty())
2200     return nullptr;
2201   return Line->Tokens.back().Tok;
2202 }
2203 
2204 void UnwrappedLineParser::readToken() {
2205   bool CommentsInCurrentLine = true;
2206   do {
2207     FormatTok = Tokens->getNextToken();
2208     assert(FormatTok);
2209     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
2210            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
2211       // If there is an unfinished unwrapped line, we flush the preprocessor
2212       // directives only after that unwrapped line was finished later.
2213       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
2214       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
2215       // Comments stored before the preprocessor directive need to be output
2216       // before the preprocessor directive, at the same level as the
2217       // preprocessor directive, as we consider them to apply to the directive.
2218       flushComments(isOnNewLine(*FormatTok));
2219       parsePPDirective();
2220     }
2221     while (FormatTok->Type == TT_ConflictStart ||
2222            FormatTok->Type == TT_ConflictEnd ||
2223            FormatTok->Type == TT_ConflictAlternative) {
2224       if (FormatTok->Type == TT_ConflictStart) {
2225         conditionalCompilationStart(/*Unreachable=*/false);
2226       } else if (FormatTok->Type == TT_ConflictAlternative) {
2227         conditionalCompilationAlternative();
2228       } else if (FormatTok->Type == TT_ConflictEnd) {
2229         conditionalCompilationEnd();
2230       }
2231       FormatTok = Tokens->getNextToken();
2232       FormatTok->MustBreakBefore = true;
2233     }
2234 
2235     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
2236         !Line->InPPDirective) {
2237       continue;
2238     }
2239 
2240     if (!FormatTok->Tok.is(tok::comment))
2241       return;
2242     FormatTok->ContinuesLineCommentSection =
2243         continuesLineComment(*FormatTok, *Line, CommentPragmasRegex);
2244     if (!FormatTok->ContinuesLineCommentSection &&
2245         (isOnNewLine(*FormatTok) || FormatTok->IsFirst)) {
2246       CommentsInCurrentLine = false;
2247     }
2248     if (CommentsInCurrentLine) {
2249       pushToken(FormatTok);
2250     } else {
2251       CommentsBeforeNextToken.push_back(FormatTok);
2252     }
2253   } while (!eof());
2254 }
2255 
2256 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
2257   Line->Tokens.push_back(UnwrappedLineNode(Tok));
2258   if (MustBreakBeforeNextToken) {
2259     Line->Tokens.back().Tok->MustBreakBefore = true;
2260     MustBreakBeforeNextToken = false;
2261   }
2262 }
2263 
2264 } // end namespace format
2265 } // end namespace clang
2266