1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #define DEBUG_TYPE "format-parser"
22 
23 namespace clang {
24 namespace format {
25 
26 class FormatTokenSource {
27 public:
28   virtual ~FormatTokenSource() {}
29   virtual FormatToken *getNextToken() = 0;
30 
31   virtual unsigned getPosition() = 0;
32   virtual FormatToken *setPosition(unsigned Position) = 0;
33 };
34 
35 namespace {
36 
37 class ScopedDeclarationState {
38 public:
39   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
40                          bool MustBeDeclaration)
41       : Line(Line), Stack(Stack) {
42     Line.MustBeDeclaration = MustBeDeclaration;
43     Stack.push_back(MustBeDeclaration);
44   }
45   ~ScopedDeclarationState() {
46     Stack.pop_back();
47     if (!Stack.empty())
48       Line.MustBeDeclaration = Stack.back();
49     else
50       Line.MustBeDeclaration = true;
51   }
52 
53 private:
54   UnwrappedLine &Line;
55   std::vector<bool> &Stack;
56 };
57 
58 class ScopedMacroState : public FormatTokenSource {
59 public:
60   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
61                    FormatToken *&ResetToken, bool &StructuralError)
62       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
63         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
64         StructuralError(StructuralError),
65         PreviousStructuralError(StructuralError), Token(nullptr) {
66     TokenSource = this;
67     Line.Level = 0;
68     Line.InPPDirective = true;
69   }
70 
71   ~ScopedMacroState() override {
72     TokenSource = PreviousTokenSource;
73     ResetToken = Token;
74     Line.InPPDirective = false;
75     Line.Level = PreviousLineLevel;
76     StructuralError = PreviousStructuralError;
77   }
78 
79   FormatToken *getNextToken() override {
80     // The \c UnwrappedLineParser guards against this by never calling
81     // \c getNextToken() after it has encountered the first eof token.
82     assert(!eof());
83     Token = PreviousTokenSource->getNextToken();
84     if (eof())
85       return getFakeEOF();
86     return Token;
87   }
88 
89   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
90 
91   FormatToken *setPosition(unsigned Position) override {
92     Token = PreviousTokenSource->setPosition(Position);
93     return Token;
94   }
95 
96 private:
97   bool eof() { return Token && Token->HasUnescapedNewline; }
98 
99   FormatToken *getFakeEOF() {
100     static bool EOFInitialized = false;
101     static FormatToken FormatTok;
102     if (!EOFInitialized) {
103       FormatTok.Tok.startToken();
104       FormatTok.Tok.setKind(tok::eof);
105       EOFInitialized = true;
106     }
107     return &FormatTok;
108   }
109 
110   UnwrappedLine &Line;
111   FormatTokenSource *&TokenSource;
112   FormatToken *&ResetToken;
113   unsigned PreviousLineLevel;
114   FormatTokenSource *PreviousTokenSource;
115   bool &StructuralError;
116   bool PreviousStructuralError;
117 
118   FormatToken *Token;
119 };
120 
121 } // end anonymous namespace
122 
123 class ScopedLineState {
124 public:
125   ScopedLineState(UnwrappedLineParser &Parser,
126                   bool SwitchToPreprocessorLines = false)
127       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
128     if (SwitchToPreprocessorLines)
129       Parser.CurrentLines = &Parser.PreprocessorDirectives;
130     else if (!Parser.Line->Tokens.empty())
131       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
132     PreBlockLine = std::move(Parser.Line);
133     Parser.Line = llvm::make_unique<UnwrappedLine>();
134     Parser.Line->Level = PreBlockLine->Level;
135     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
136   }
137 
138   ~ScopedLineState() {
139     if (!Parser.Line->Tokens.empty()) {
140       Parser.addUnwrappedLine();
141     }
142     assert(Parser.Line->Tokens.empty());
143     Parser.Line = std::move(PreBlockLine);
144     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
145       Parser.MustBreakBeforeNextToken = true;
146     Parser.CurrentLines = OriginalLines;
147   }
148 
149 private:
150   UnwrappedLineParser &Parser;
151 
152   std::unique_ptr<UnwrappedLine> PreBlockLine;
153   SmallVectorImpl<UnwrappedLine> *OriginalLines;
154 };
155 
156 class CompoundStatementIndenter {
157 public:
158   CompoundStatementIndenter(UnwrappedLineParser *Parser,
159                             const FormatStyle &Style, unsigned &LineLevel)
160       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
161     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
162       Parser->addUnwrappedLine();
163     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
164       Parser->addUnwrappedLine();
165       ++LineLevel;
166     }
167   }
168   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
169 
170 private:
171   unsigned &LineLevel;
172   unsigned OldLineLevel;
173 };
174 
175 namespace {
176 
177 class IndexedTokenSource : public FormatTokenSource {
178 public:
179   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
180       : Tokens(Tokens), Position(-1) {}
181 
182   FormatToken *getNextToken() override {
183     ++Position;
184     return Tokens[Position];
185   }
186 
187   unsigned getPosition() override {
188     assert(Position >= 0);
189     return Position;
190   }
191 
192   FormatToken *setPosition(unsigned P) override {
193     Position = P;
194     return Tokens[Position];
195   }
196 
197   void reset() { Position = -1; }
198 
199 private:
200   ArrayRef<FormatToken *> Tokens;
201   int Position;
202 };
203 
204 } // end anonymous namespace
205 
206 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
207                                          const AdditionalKeywords &Keywords,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style),
212       Keywords(Keywords), Tokens(nullptr), Callback(Callback),
213       AllTokens(Tokens), PPBranchLevel(-1) {}
214 
215 void UnwrappedLineParser::reset() {
216   PPBranchLevel = -1;
217   Line.reset(new UnwrappedLine);
218   CommentsBeforeNextToken.clear();
219   FormatTok = nullptr;
220   MustBreakBeforeNextToken = false;
221   PreprocessorDirectives.clear();
222   CurrentLines = &Lines;
223   DeclarationScopeStack.clear();
224   StructuralError = false;
225   PPStack.clear();
226 }
227 
228 bool UnwrappedLineParser::parse() {
229   IndexedTokenSource TokenSource(AllTokens);
230   do {
231     DEBUG(llvm::dbgs() << "----\n");
232     reset();
233     Tokens = &TokenSource;
234     TokenSource.reset();
235 
236     readToken();
237     parseFile();
238     // Create line with eof token.
239     pushToken(FormatTok);
240     addUnwrappedLine();
241 
242     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
243                                                   E = Lines.end();
244          I != E; ++I) {
245       Callback.consumeUnwrappedLine(*I);
246     }
247     Callback.finishRun();
248     Lines.clear();
249     while (!PPLevelBranchIndex.empty() &&
250            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
251       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
252       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
253     }
254     if (!PPLevelBranchIndex.empty()) {
255       ++PPLevelBranchIndex.back();
256       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
257       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
258     }
259   } while (!PPLevelBranchIndex.empty());
260 
261   return StructuralError;
262 }
263 
264 void UnwrappedLineParser::parseFile() {
265   ScopedDeclarationState DeclarationState(
266       *Line, DeclarationScopeStack,
267       /*MustBeDeclaration=*/!Line->InPPDirective);
268   parseLevel(/*HasOpeningBrace=*/false);
269   // Make sure to format the remaining tokens.
270   flushComments(true);
271   addUnwrappedLine();
272 }
273 
274 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
275   bool SwitchLabelEncountered = false;
276   do {
277     switch (FormatTok->Tok.getKind()) {
278     case tok::comment:
279       nextToken();
280       addUnwrappedLine();
281       break;
282     case tok::l_brace:
283       // FIXME: Add parameter whether this can happen - if this happens, we must
284       // be in a non-declaration context.
285       parseBlock(/*MustBeDeclaration=*/false);
286       addUnwrappedLine();
287       break;
288     case tok::r_brace:
289       if (HasOpeningBrace)
290         return;
291       StructuralError = true;
292       nextToken();
293       addUnwrappedLine();
294       break;
295     case tok::kw_default:
296     case tok::kw_case:
297       if (!SwitchLabelEncountered &&
298           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
299         ++Line->Level;
300       SwitchLabelEncountered = true;
301       parseStructuralElement();
302       break;
303     default:
304       parseStructuralElement();
305       break;
306     }
307   } while (!eof());
308 }
309 
310 void UnwrappedLineParser::calculateBraceTypes() {
311   // We'll parse forward through the tokens until we hit
312   // a closing brace or eof - note that getNextToken() will
313   // parse macros, so this will magically work inside macro
314   // definitions, too.
315   unsigned StoredPosition = Tokens->getPosition();
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           bool ProbablyBracedList = false;
339           if (Style.Language == FormatStyle::LK_Proto) {
340             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
341           } else {
342             // Using OriginalColumn to distinguish between ObjC methods and
343             // binary operators is a bit hacky.
344             bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
345                                     NextTok->OriginalColumn == 0;
346 
347             // If there is a comma, semicolon or right paren after the closing
348             // brace, we assume this is a braced initializer list.  Note that
349             // regardless how we mark inner braces here, we will overwrite the
350             // BlockKind later if we parse a braced list (where all blocks
351             // inside are by default braced lists), or when we explicitly detect
352             // blocks (for example while parsing lambdas).
353             //
354             // We exclude + and - as they can be ObjC visibility modifiers.
355             ProbablyBracedList =
356                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
357                                  tok::r_paren, tok::r_square, tok::l_brace,
358                                  tok::l_paren, tok::ellipsis) ||
359                 (NextTok->isBinaryOperator() && !NextIsObjCMethod);
360           }
361           if (ProbablyBracedList) {
362             Tok->BlockKind = BK_BracedInit;
363             LBraceStack.back()->BlockKind = BK_BracedInit;
364           } else {
365             Tok->BlockKind = BK_Block;
366             LBraceStack.back()->BlockKind = BK_Block;
367           }
368         }
369         LBraceStack.pop_back();
370       }
371       break;
372     case tok::at:
373     case tok::semi:
374     case tok::kw_if:
375     case tok::kw_while:
376     case tok::kw_for:
377     case tok::kw_switch:
378     case tok::kw_try:
379     case tok::kw___try:
380       if (!LBraceStack.empty())
381         LBraceStack.back()->BlockKind = BK_Block;
382       break;
383     default:
384       break;
385     }
386     Tok = NextTok;
387   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
388   // Assume other blocks for all unclosed opening braces.
389   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
390     if (LBraceStack[i]->BlockKind == BK_Unknown)
391       LBraceStack[i]->BlockKind = BK_Block;
392   }
393 
394   FormatTok = Tokens->setPosition(StoredPosition);
395 }
396 
397 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
398                                      bool MunchSemi) {
399   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
400   unsigned InitialLevel = Line->Level;
401   nextToken();
402 
403   addUnwrappedLine();
404 
405   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
406                                           MustBeDeclaration);
407   if (AddLevel)
408     ++Line->Level;
409   parseLevel(/*HasOpeningBrace=*/true);
410 
411   if (!FormatTok->Tok.is(tok::r_brace)) {
412     Line->Level = InitialLevel;
413     StructuralError = true;
414     return;
415   }
416 
417   nextToken(); // Munch the closing brace.
418   if (MunchSemi && FormatTok->Tok.is(tok::semi))
419     nextToken();
420   Line->Level = InitialLevel;
421 }
422 
423 static bool isGoogScope(const UnwrappedLine &Line) {
424   // FIXME: Closure-library specific stuff should not be hard-coded but be
425   // configurable.
426   if (Line.Tokens.size() < 4)
427     return false;
428   auto I = Line.Tokens.begin();
429   if (I->Tok->TokenText != "goog")
430     return false;
431   ++I;
432   if (I->Tok->isNot(tok::period))
433     return false;
434   ++I;
435   if (I->Tok->TokenText != "scope")
436     return false;
437   ++I;
438   return I->Tok->is(tok::l_paren);
439 }
440 
441 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
442                                    const FormatToken &InitialToken) {
443   switch (Style.BreakBeforeBraces) {
444   case FormatStyle::BS_Linux:
445     return InitialToken.isOneOf(tok::kw_namespace, tok::kw_class);
446   case FormatStyle::BS_Allman:
447   case FormatStyle::BS_GNU:
448     return true;
449   default:
450     return false;
451   }
452 }
453 
454 void UnwrappedLineParser::parseChildBlock() {
455   FormatTok->BlockKind = BK_Block;
456   nextToken();
457   {
458     bool GoogScope =
459         Style.Language == FormatStyle::LK_JavaScript && isGoogScope(*Line);
460     ScopedLineState LineState(*this);
461     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
462                                             /*MustBeDeclaration=*/false);
463     Line->Level += GoogScope ? 0 : 1;
464     parseLevel(/*HasOpeningBrace=*/true);
465     flushComments(isOnNewLine(*FormatTok));
466     Line->Level -= GoogScope ? 0 : 1;
467   }
468   nextToken();
469 }
470 
471 void UnwrappedLineParser::parsePPDirective() {
472   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
473   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
474   nextToken();
475 
476   if (!FormatTok->Tok.getIdentifierInfo()) {
477     parsePPUnknown();
478     return;
479   }
480 
481   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
482   case tok::pp_define:
483     parsePPDefine();
484     return;
485   case tok::pp_if:
486     parsePPIf(/*IfDef=*/false);
487     break;
488   case tok::pp_ifdef:
489   case tok::pp_ifndef:
490     parsePPIf(/*IfDef=*/true);
491     break;
492   case tok::pp_else:
493     parsePPElse();
494     break;
495   case tok::pp_elif:
496     parsePPElIf();
497     break;
498   case tok::pp_endif:
499     parsePPEndIf();
500     break;
501   default:
502     parsePPUnknown();
503     break;
504   }
505 }
506 
507 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
508   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
509     PPStack.push_back(PP_Unreachable);
510   else
511     PPStack.push_back(PP_Conditional);
512 }
513 
514 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
515   ++PPBranchLevel;
516   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
517   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
518     PPLevelBranchIndex.push_back(0);
519     PPLevelBranchCount.push_back(0);
520   }
521   PPChainBranchIndex.push(0);
522   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
523   conditionalCompilationCondition(Unreachable || Skip);
524 }
525 
526 void UnwrappedLineParser::conditionalCompilationAlternative() {
527   if (!PPStack.empty())
528     PPStack.pop_back();
529   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
530   if (!PPChainBranchIndex.empty())
531     ++PPChainBranchIndex.top();
532   conditionalCompilationCondition(
533       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
534       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
535 }
536 
537 void UnwrappedLineParser::conditionalCompilationEnd() {
538   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
539   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
540     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
541       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
542     }
543   }
544   // Guard against #endif's without #if.
545   if (PPBranchLevel > 0)
546     --PPBranchLevel;
547   if (!PPChainBranchIndex.empty())
548     PPChainBranchIndex.pop();
549   if (!PPStack.empty())
550     PPStack.pop_back();
551 }
552 
553 void UnwrappedLineParser::parsePPIf(bool IfDef) {
554   nextToken();
555   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
556                          FormatTok->Tok.getLiteralData() != nullptr &&
557                          StringRef(FormatTok->Tok.getLiteralData(),
558                                    FormatTok->Tok.getLength()) == "0") ||
559                         FormatTok->Tok.is(tok::kw_false);
560   conditionalCompilationStart(!IfDef && IsLiteralFalse);
561   parsePPUnknown();
562 }
563 
564 void UnwrappedLineParser::parsePPElse() {
565   conditionalCompilationAlternative();
566   parsePPUnknown();
567 }
568 
569 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
570 
571 void UnwrappedLineParser::parsePPEndIf() {
572   conditionalCompilationEnd();
573   parsePPUnknown();
574 }
575 
576 void UnwrappedLineParser::parsePPDefine() {
577   nextToken();
578 
579   if (FormatTok->Tok.getKind() != tok::identifier) {
580     parsePPUnknown();
581     return;
582   }
583   nextToken();
584   if (FormatTok->Tok.getKind() == tok::l_paren &&
585       FormatTok->WhitespaceRange.getBegin() ==
586           FormatTok->WhitespaceRange.getEnd()) {
587     parseParens();
588   }
589   addUnwrappedLine();
590   Line->Level = 1;
591 
592   // Errors during a preprocessor directive can only affect the layout of the
593   // preprocessor directive, and thus we ignore them. An alternative approach
594   // would be to use the same approach we use on the file level (no
595   // re-indentation if there was a structural error) within the macro
596   // definition.
597   parseFile();
598 }
599 
600 void UnwrappedLineParser::parsePPUnknown() {
601   do {
602     nextToken();
603   } while (!eof());
604   addUnwrappedLine();
605 }
606 
607 // Here we blacklist certain tokens that are not usually the first token in an
608 // unwrapped line. This is used in attempt to distinguish macro calls without
609 // trailing semicolons from other constructs split to several lines.
610 static bool tokenCanStartNewLine(const clang::Token &Tok) {
611   // Semicolon can be a null-statement, l_square can be a start of a macro or
612   // a C++11 attribute, but this doesn't seem to be common.
613   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
614          Tok.isNot(tok::l_square) &&
615          // Tokens that can only be used as binary operators and a part of
616          // overloaded operator names.
617          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
618          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
619          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
620          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
621          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
622          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
623          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
624          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
625          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
626          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
627          Tok.isNot(tok::lesslessequal) &&
628          // Colon is used in labels, base class lists, initializer lists,
629          // range-based for loops, ternary operator, but should never be the
630          // first token in an unwrapped line.
631          Tok.isNot(tok::colon) &&
632          // 'noexcept' is a trailing annotation.
633          Tok.isNot(tok::kw_noexcept);
634 }
635 
636 void UnwrappedLineParser::parseStructuralElement() {
637   assert(!FormatTok->Tok.is(tok::l_brace));
638   switch (FormatTok->Tok.getKind()) {
639   case tok::at:
640     nextToken();
641     if (FormatTok->Tok.is(tok::l_brace)) {
642       parseBracedList();
643       break;
644     }
645     switch (FormatTok->Tok.getObjCKeywordID()) {
646     case tok::objc_public:
647     case tok::objc_protected:
648     case tok::objc_package:
649     case tok::objc_private:
650       return parseAccessSpecifier();
651     case tok::objc_interface:
652     case tok::objc_implementation:
653       return parseObjCInterfaceOrImplementation();
654     case tok::objc_protocol:
655       return parseObjCProtocol();
656     case tok::objc_end:
657       return; // Handled by the caller.
658     case tok::objc_optional:
659     case tok::objc_required:
660       nextToken();
661       addUnwrappedLine();
662       return;
663     case tok::objc_try:
664       // This branch isn't strictly necessary (the kw_try case below would
665       // do this too after the tok::at is parsed above).  But be explicit.
666       parseTryCatch();
667       return;
668     default:
669       break;
670     }
671     break;
672   case tok::kw_asm:
673     nextToken();
674     if (FormatTok->is(tok::l_brace)) {
675       nextToken();
676       while (FormatTok && FormatTok->isNot(tok::eof)) {
677         if (FormatTok->is(tok::r_brace)) {
678           nextToken();
679           break;
680         }
681         FormatTok->Finalized = true;
682         nextToken();
683       }
684     }
685     break;
686   case tok::kw_namespace:
687     parseNamespace();
688     return;
689   case tok::kw_inline:
690     nextToken();
691     if (FormatTok->Tok.is(tok::kw_namespace)) {
692       parseNamespace();
693       return;
694     }
695     break;
696   case tok::kw_public:
697   case tok::kw_protected:
698   case tok::kw_private:
699     if (Style.Language == FormatStyle::LK_Java ||
700         Style.Language == FormatStyle::LK_JavaScript)
701       nextToken();
702     else
703       parseAccessSpecifier();
704     return;
705   case tok::kw_if:
706     parseIfThenElse();
707     return;
708   case tok::kw_for:
709   case tok::kw_while:
710     parseForOrWhileLoop();
711     return;
712   case tok::kw_do:
713     parseDoWhile();
714     return;
715   case tok::kw_switch:
716     parseSwitch();
717     return;
718   case tok::kw_default:
719     nextToken();
720     parseLabel();
721     return;
722   case tok::kw_case:
723     parseCaseLabel();
724     return;
725   case tok::kw_try:
726   case tok::kw___try:
727     parseTryCatch();
728     return;
729   case tok::kw_extern:
730     nextToken();
731     if (FormatTok->Tok.is(tok::string_literal)) {
732       nextToken();
733       if (FormatTok->Tok.is(tok::l_brace)) {
734         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
735         addUnwrappedLine();
736         return;
737       }
738     }
739     break;
740   case tok::kw_export:
741     if (Style.Language == FormatStyle::LK_JavaScript) {
742       parseJavaScriptEs6ImportExport();
743       return;
744     }
745     break;
746   case tok::identifier:
747     if (FormatTok->is(TT_ForEachMacro)) {
748       parseForOrWhileLoop();
749       return;
750     }
751     if (Style.Language == FormatStyle::LK_JavaScript &&
752         FormatTok->is(Keywords.kw_import)) {
753       parseJavaScriptEs6ImportExport();
754       return;
755     }
756     if (FormatTok->is(Keywords.kw_signals)) {
757       nextToken();
758       if (FormatTok->is(tok::colon)) {
759         nextToken();
760         addUnwrappedLine();
761       }
762       return;
763     }
764     // In all other cases, parse the declaration.
765     break;
766   default:
767     break;
768   }
769   do {
770     switch (FormatTok->Tok.getKind()) {
771     case tok::at:
772       nextToken();
773       if (FormatTok->Tok.is(tok::l_brace))
774         parseBracedList();
775       break;
776     case tok::kw_enum:
777       parseEnum();
778       break;
779     case tok::kw_typedef:
780       nextToken();
781       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
782                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
783         parseEnum();
784       break;
785     case tok::kw_struct:
786     case tok::kw_union:
787     case tok::kw_class:
788       parseRecord();
789       // A record declaration or definition is always the start of a structural
790       // element.
791       break;
792     case tok::period:
793       nextToken();
794       // In Java, classes have an implicit static member "class".
795       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
796           FormatTok->is(tok::kw_class))
797         nextToken();
798       break;
799     case tok::semi:
800       nextToken();
801       addUnwrappedLine();
802       return;
803     case tok::r_brace:
804       addUnwrappedLine();
805       return;
806     case tok::l_paren:
807       parseParens();
808       break;
809     case tok::caret:
810       nextToken();
811       if (FormatTok->Tok.isAnyIdentifier() ||
812           FormatTok->isSimpleTypeSpecifier())
813         nextToken();
814       if (FormatTok->is(tok::l_paren))
815         parseParens();
816       if (FormatTok->is(tok::l_brace))
817         parseChildBlock();
818       break;
819     case tok::l_brace:
820       if (!tryToParseBracedList()) {
821         // A block outside of parentheses must be the last part of a
822         // structural element.
823         // FIXME: Figure out cases where this is not true, and add projections
824         // for them (the one we know is missing are lambdas).
825         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
826           addUnwrappedLine();
827         FormatTok->Type = TT_FunctionLBrace;
828         parseBlock(/*MustBeDeclaration=*/false);
829         addUnwrappedLine();
830         return;
831       }
832       // Otherwise this was a braced init list, and the structural
833       // element continues.
834       break;
835     case tok::kw_try:
836       // We arrive here when parsing function-try blocks.
837       parseTryCatch();
838       return;
839     case tok::identifier: {
840       StringRef Text = FormatTok->TokenText;
841       // Parse function literal unless 'function' is the first token in a line
842       // in which case this should be treated as a free-standing function.
843       if (Style.Language == FormatStyle::LK_JavaScript && Text == "function" &&
844           Line->Tokens.size() > 0) {
845         tryToParseJSFunction();
846         break;
847       }
848       nextToken();
849       if (Line->Tokens.size() == 1 &&
850           // JS doesn't have macros, and within classes colons indicate fields,
851           // not labels.
852           Style.Language != FormatStyle::LK_JavaScript) {
853         if (FormatTok->Tok.is(tok::colon) && !Line->MustBeDeclaration) {
854           parseLabel();
855           return;
856         }
857         // Recognize function-like macro usages without trailing semicolon as
858         // well as free-standing macros like Q_OBJECT.
859         bool FunctionLike = FormatTok->is(tok::l_paren);
860         if (FunctionLike)
861           parseParens();
862         if (FormatTok->NewlinesBefore > 0 &&
863             (Text.size() >= 5 || FunctionLike) &&
864             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
865           addUnwrappedLine();
866           return;
867         }
868       }
869       break;
870     }
871     case tok::equal:
872       nextToken();
873       if (FormatTok->Tok.is(tok::l_brace)) {
874         parseBracedList();
875       }
876       break;
877     case tok::l_square:
878       parseSquare();
879       break;
880     case tok::kw_new:
881       parseNew();
882       break;
883     default:
884       nextToken();
885       break;
886     }
887   } while (!eof());
888 }
889 
890 bool UnwrappedLineParser::tryToParseLambda() {
891   // FIXME: This is a dirty way to access the previous token. Find a better
892   // solution.
893   if (!Line->Tokens.empty() &&
894       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator,
895                                         tok::kw_new, tok::kw_delete) ||
896        Line->Tokens.back().Tok->closesScope() ||
897        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
898     nextToken();
899     return false;
900   }
901   assert(FormatTok->is(tok::l_square));
902   FormatToken &LSquare = *FormatTok;
903   if (!tryToParseLambdaIntroducer())
904     return false;
905 
906   while (FormatTok->isNot(tok::l_brace)) {
907     if (FormatTok->isSimpleTypeSpecifier()) {
908       nextToken();
909       continue;
910     }
911     switch (FormatTok->Tok.getKind()) {
912     case tok::l_brace:
913       break;
914     case tok::l_paren:
915       parseParens();
916       break;
917     case tok::amp:
918     case tok::star:
919     case tok::kw_const:
920     case tok::comma:
921     case tok::less:
922     case tok::greater:
923     case tok::identifier:
924     case tok::coloncolon:
925     case tok::kw_mutable:
926       nextToken();
927       break;
928     case tok::arrow:
929       FormatTok->Type = TT_TrailingReturnArrow;
930       nextToken();
931       break;
932     default:
933       return true;
934     }
935   }
936   LSquare.Type = TT_LambdaLSquare;
937   parseChildBlock();
938   return true;
939 }
940 
941 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
942   nextToken();
943   if (FormatTok->is(tok::equal)) {
944     nextToken();
945     if (FormatTok->is(tok::r_square)) {
946       nextToken();
947       return true;
948     }
949     if (FormatTok->isNot(tok::comma))
950       return false;
951     nextToken();
952   } else if (FormatTok->is(tok::amp)) {
953     nextToken();
954     if (FormatTok->is(tok::r_square)) {
955       nextToken();
956       return true;
957     }
958     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
959       return false;
960     }
961     if (FormatTok->is(tok::comma))
962       nextToken();
963   } else if (FormatTok->is(tok::r_square)) {
964     nextToken();
965     return true;
966   }
967   do {
968     if (FormatTok->is(tok::amp))
969       nextToken();
970     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
971       return false;
972     nextToken();
973     if (FormatTok->is(tok::ellipsis))
974       nextToken();
975     if (FormatTok->is(tok::comma)) {
976       nextToken();
977     } else if (FormatTok->is(tok::r_square)) {
978       nextToken();
979       return true;
980     } else {
981       return false;
982     }
983   } while (!eof());
984   return false;
985 }
986 
987 void UnwrappedLineParser::tryToParseJSFunction() {
988   nextToken();
989 
990   // Consume function name.
991   if (FormatTok->is(tok::identifier))
992     nextToken();
993 
994   if (FormatTok->isNot(tok::l_paren))
995     return;
996   nextToken();
997   while (FormatTok->isNot(tok::l_brace)) {
998     // Err on the side of caution in order to avoid consuming the full file in
999     // case of incomplete code.
1000     if (!FormatTok->isOneOf(tok::identifier, tok::comma, tok::r_paren,
1001                             tok::comment))
1002       return;
1003     nextToken();
1004   }
1005   parseChildBlock();
1006 }
1007 
1008 bool UnwrappedLineParser::tryToParseBracedList() {
1009   if (FormatTok->BlockKind == BK_Unknown)
1010     calculateBraceTypes();
1011   assert(FormatTok->BlockKind != BK_Unknown);
1012   if (FormatTok->BlockKind == BK_Block)
1013     return false;
1014   parseBracedList();
1015   return true;
1016 }
1017 
1018 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
1019   bool HasError = false;
1020   nextToken();
1021 
1022   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
1023   // replace this by using parseAssigmentExpression() inside.
1024   do {
1025     if (Style.Language == FormatStyle::LK_JavaScript &&
1026         FormatTok->is(Keywords.kw_function)) {
1027       tryToParseJSFunction();
1028       continue;
1029     }
1030     switch (FormatTok->Tok.getKind()) {
1031     case tok::caret:
1032       nextToken();
1033       if (FormatTok->is(tok::l_brace)) {
1034         parseChildBlock();
1035       }
1036       break;
1037     case tok::l_square:
1038       tryToParseLambda();
1039       break;
1040     case tok::l_brace:
1041       // Assume there are no blocks inside a braced init list apart
1042       // from the ones we explicitly parse out (like lambdas).
1043       FormatTok->BlockKind = BK_BracedInit;
1044       parseBracedList();
1045       break;
1046     case tok::r_paren:
1047       // JavaScript can just have free standing methods and getters/setters in
1048       // object literals. Detect them by a "{" following ")".
1049       if (Style.Language == FormatStyle::LK_JavaScript) {
1050         nextToken();
1051         if (FormatTok->is(tok::l_brace))
1052           parseChildBlock();
1053         break;
1054       }
1055       nextToken();
1056       break;
1057     case tok::r_brace:
1058       nextToken();
1059       return !HasError;
1060     case tok::semi:
1061       HasError = true;
1062       if (!ContinueOnSemicolons)
1063         return !HasError;
1064       nextToken();
1065       break;
1066     case tok::comma:
1067       nextToken();
1068       break;
1069     default:
1070       nextToken();
1071       break;
1072     }
1073   } while (!eof());
1074   return false;
1075 }
1076 
1077 void UnwrappedLineParser::parseParens() {
1078   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1079   nextToken();
1080   do {
1081     switch (FormatTok->Tok.getKind()) {
1082     case tok::l_paren:
1083       parseParens();
1084       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1085         parseChildBlock();
1086       break;
1087     case tok::r_paren:
1088       nextToken();
1089       return;
1090     case tok::r_brace:
1091       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1092       return;
1093     case tok::l_square:
1094       tryToParseLambda();
1095       break;
1096     case tok::l_brace:
1097       if (!tryToParseBracedList()) {
1098         parseChildBlock();
1099       }
1100       break;
1101     case tok::at:
1102       nextToken();
1103       if (FormatTok->Tok.is(tok::l_brace))
1104         parseBracedList();
1105       break;
1106     case tok::identifier:
1107       if (Style.Language == FormatStyle::LK_JavaScript &&
1108           FormatTok->is(Keywords.kw_function))
1109         tryToParseJSFunction();
1110       else
1111         nextToken();
1112       break;
1113     default:
1114       nextToken();
1115       break;
1116     }
1117   } while (!eof());
1118 }
1119 
1120 void UnwrappedLineParser::parseSquare() {
1121   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1122   if (tryToParseLambda())
1123     return;
1124   do {
1125     switch (FormatTok->Tok.getKind()) {
1126     case tok::l_paren:
1127       parseParens();
1128       break;
1129     case tok::r_square:
1130       nextToken();
1131       return;
1132     case tok::r_brace:
1133       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1134       return;
1135     case tok::l_square:
1136       parseSquare();
1137       break;
1138     case tok::l_brace: {
1139       if (!tryToParseBracedList()) {
1140         parseChildBlock();
1141       }
1142       break;
1143     }
1144     case tok::at:
1145       nextToken();
1146       if (FormatTok->Tok.is(tok::l_brace))
1147         parseBracedList();
1148       break;
1149     default:
1150       nextToken();
1151       break;
1152     }
1153   } while (!eof());
1154 }
1155 
1156 void UnwrappedLineParser::parseIfThenElse() {
1157   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1158   nextToken();
1159   if (FormatTok->Tok.is(tok::l_paren))
1160     parseParens();
1161   bool NeedsUnwrappedLine = false;
1162   if (FormatTok->Tok.is(tok::l_brace)) {
1163     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1164     parseBlock(/*MustBeDeclaration=*/false);
1165     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1166         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1167       addUnwrappedLine();
1168     } else {
1169       NeedsUnwrappedLine = true;
1170     }
1171   } else {
1172     addUnwrappedLine();
1173     ++Line->Level;
1174     parseStructuralElement();
1175     --Line->Level;
1176   }
1177   if (FormatTok->Tok.is(tok::kw_else)) {
1178     if (Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup)
1179       addUnwrappedLine();
1180     nextToken();
1181     if (FormatTok->Tok.is(tok::l_brace)) {
1182       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1183       parseBlock(/*MustBeDeclaration=*/false);
1184       addUnwrappedLine();
1185     } else if (FormatTok->Tok.is(tok::kw_if)) {
1186       parseIfThenElse();
1187     } else {
1188       addUnwrappedLine();
1189       ++Line->Level;
1190       parseStructuralElement();
1191       --Line->Level;
1192     }
1193   } else if (NeedsUnwrappedLine) {
1194     addUnwrappedLine();
1195   }
1196 }
1197 
1198 void UnwrappedLineParser::parseTryCatch() {
1199   assert(FormatTok->isOneOf(tok::kw_try, tok::kw___try) && "'try' expected");
1200   nextToken();
1201   bool NeedsUnwrappedLine = false;
1202   if (FormatTok->is(tok::colon)) {
1203     // We are in a function try block, what comes is an initializer list.
1204     nextToken();
1205     while (FormatTok->is(tok::identifier)) {
1206       nextToken();
1207       if (FormatTok->is(tok::l_paren))
1208         parseParens();
1209       else
1210         StructuralError = true;
1211       if (FormatTok->is(tok::comma))
1212         nextToken();
1213     }
1214   }
1215   // Parse try with resource.
1216   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1217     parseParens();
1218   }
1219   if (FormatTok->is(tok::l_brace)) {
1220     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1221     parseBlock(/*MustBeDeclaration=*/false);
1222     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1223         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1224         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1225       addUnwrappedLine();
1226     } else {
1227       NeedsUnwrappedLine = true;
1228     }
1229   } else if (!FormatTok->is(tok::kw_catch)) {
1230     // The C++ standard requires a compound-statement after a try.
1231     // If there's none, we try to assume there's a structuralElement
1232     // and try to continue.
1233     StructuralError = true;
1234     addUnwrappedLine();
1235     ++Line->Level;
1236     parseStructuralElement();
1237     --Line->Level;
1238   }
1239   while (1) {
1240     if (FormatTok->is(tok::at))
1241       nextToken();
1242     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
1243                              tok::kw___finally) ||
1244           ((Style.Language == FormatStyle::LK_Java ||
1245             Style.Language == FormatStyle::LK_JavaScript) &&
1246            FormatTok->is(Keywords.kw_finally)) ||
1247           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
1248            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
1249       break;
1250     nextToken();
1251     while (FormatTok->isNot(tok::l_brace)) {
1252       if (FormatTok->is(tok::l_paren)) {
1253         parseParens();
1254         continue;
1255       }
1256       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1257         return;
1258       nextToken();
1259     }
1260     NeedsUnwrappedLine = false;
1261     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1262     parseBlock(/*MustBeDeclaration=*/false);
1263     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1264         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1265         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1266       addUnwrappedLine();
1267     } else {
1268       NeedsUnwrappedLine = true;
1269     }
1270   }
1271   if (NeedsUnwrappedLine) {
1272     addUnwrappedLine();
1273   }
1274 }
1275 
1276 void UnwrappedLineParser::parseNamespace() {
1277   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1278 
1279   const FormatToken &InitialToken = *FormatTok;
1280   nextToken();
1281   if (FormatTok->Tok.is(tok::identifier))
1282     nextToken();
1283   if (FormatTok->Tok.is(tok::l_brace)) {
1284     if (ShouldBreakBeforeBrace(Style, InitialToken))
1285       addUnwrappedLine();
1286 
1287     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1288                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1289                      DeclarationScopeStack.size() > 1);
1290     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1291     // Munch the semicolon after a namespace. This is more common than one would
1292     // think. Puttin the semicolon into its own line is very ugly.
1293     if (FormatTok->Tok.is(tok::semi))
1294       nextToken();
1295     addUnwrappedLine();
1296   }
1297   // FIXME: Add error handling.
1298 }
1299 
1300 void UnwrappedLineParser::parseNew() {
1301   assert(FormatTok->is(tok::kw_new) && "'new' expected");
1302   nextToken();
1303   if (Style.Language != FormatStyle::LK_Java)
1304     return;
1305 
1306   // In Java, we can parse everything up to the parens, which aren't optional.
1307   do {
1308     // There should not be a ;, { or } before the new's open paren.
1309     if (FormatTok->isOneOf(tok::semi, tok::l_brace, tok::r_brace))
1310       return;
1311 
1312     // Consume the parens.
1313     if (FormatTok->is(tok::l_paren)) {
1314       parseParens();
1315 
1316       // If there is a class body of an anonymous class, consume that as child.
1317       if (FormatTok->is(tok::l_brace))
1318         parseChildBlock();
1319       return;
1320     }
1321     nextToken();
1322   } while (!eof());
1323 }
1324 
1325 void UnwrappedLineParser::parseForOrWhileLoop() {
1326   assert(FormatTok->isOneOf(tok::kw_for, tok::kw_while, TT_ForEachMacro) &&
1327          "'for', 'while' or foreach macro expected");
1328   nextToken();
1329   if (FormatTok->Tok.is(tok::l_paren))
1330     parseParens();
1331   if (FormatTok->Tok.is(tok::l_brace)) {
1332     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1333     parseBlock(/*MustBeDeclaration=*/false);
1334     addUnwrappedLine();
1335   } else {
1336     addUnwrappedLine();
1337     ++Line->Level;
1338     parseStructuralElement();
1339     --Line->Level;
1340   }
1341 }
1342 
1343 void UnwrappedLineParser::parseDoWhile() {
1344   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1345   nextToken();
1346   if (FormatTok->Tok.is(tok::l_brace)) {
1347     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1348     parseBlock(/*MustBeDeclaration=*/false);
1349     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1350       addUnwrappedLine();
1351   } else {
1352     addUnwrappedLine();
1353     ++Line->Level;
1354     parseStructuralElement();
1355     --Line->Level;
1356   }
1357 
1358   // FIXME: Add error handling.
1359   if (!FormatTok->Tok.is(tok::kw_while)) {
1360     addUnwrappedLine();
1361     return;
1362   }
1363 
1364   nextToken();
1365   parseStructuralElement();
1366 }
1367 
1368 void UnwrappedLineParser::parseLabel() {
1369   nextToken();
1370   unsigned OldLineLevel = Line->Level;
1371   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1372     --Line->Level;
1373   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1374     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1375     parseBlock(/*MustBeDeclaration=*/false);
1376     if (FormatTok->Tok.is(tok::kw_break)) {
1377       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1378       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1379           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1380         addUnwrappedLine();
1381       }
1382       parseStructuralElement();
1383     }
1384     addUnwrappedLine();
1385   } else {
1386     addUnwrappedLine();
1387   }
1388   Line->Level = OldLineLevel;
1389 }
1390 
1391 void UnwrappedLineParser::parseCaseLabel() {
1392   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1393   // FIXME: fix handling of complex expressions here.
1394   do {
1395     nextToken();
1396   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1397   parseLabel();
1398 }
1399 
1400 void UnwrappedLineParser::parseSwitch() {
1401   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1402   nextToken();
1403   if (FormatTok->Tok.is(tok::l_paren))
1404     parseParens();
1405   if (FormatTok->Tok.is(tok::l_brace)) {
1406     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1407     parseBlock(/*MustBeDeclaration=*/false);
1408     addUnwrappedLine();
1409   } else {
1410     addUnwrappedLine();
1411     ++Line->Level;
1412     parseStructuralElement();
1413     --Line->Level;
1414   }
1415 }
1416 
1417 void UnwrappedLineParser::parseAccessSpecifier() {
1418   nextToken();
1419   // Understand Qt's slots.
1420   if (FormatTok->isOneOf(Keywords.kw_slots, Keywords.kw_qslots))
1421     nextToken();
1422   // Otherwise, we don't know what it is, and we'd better keep the next token.
1423   if (FormatTok->Tok.is(tok::colon))
1424     nextToken();
1425   addUnwrappedLine();
1426 }
1427 
1428 void UnwrappedLineParser::parseEnum() {
1429   // Won't be 'enum' for NS_ENUMs.
1430   if (FormatTok->Tok.is(tok::kw_enum))
1431     nextToken();
1432 
1433   // Eat up enum class ...
1434   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1435     nextToken();
1436   while (FormatTok->Tok.getIdentifierInfo() ||
1437          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1438                             tok::greater, tok::comma, tok::question)) {
1439     nextToken();
1440     // We can have macros or attributes in between 'enum' and the enum name.
1441     if (FormatTok->is(tok::l_paren))
1442       parseParens();
1443     if (FormatTok->is(tok::identifier))
1444       nextToken();
1445   }
1446 
1447   // Just a declaration or something is wrong.
1448   if (FormatTok->isNot(tok::l_brace))
1449     return;
1450   FormatTok->BlockKind = BK_Block;
1451 
1452   if (Style.Language == FormatStyle::LK_Java) {
1453     // Java enums are different.
1454     parseJavaEnumBody();
1455     return;
1456   }
1457 
1458   // Parse enum body.
1459   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1460   if (HasError) {
1461     if (FormatTok->is(tok::semi))
1462       nextToken();
1463     addUnwrappedLine();
1464   }
1465 
1466   // We fall through to parsing a structural element afterwards, so that in
1467   // enum A {} n, m;
1468   // "} n, m;" will end up in one unwrapped line.
1469 }
1470 
1471 void UnwrappedLineParser::parseJavaEnumBody() {
1472   // Determine whether the enum is simple, i.e. does not have a semicolon or
1473   // constants with class bodies. Simple enums can be formatted like braced
1474   // lists, contracted to a single line, etc.
1475   unsigned StoredPosition = Tokens->getPosition();
1476   bool IsSimple = true;
1477   FormatToken *Tok = Tokens->getNextToken();
1478   while (Tok) {
1479     if (Tok->is(tok::r_brace))
1480       break;
1481     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1482       IsSimple = false;
1483       break;
1484     }
1485     // FIXME: This will also mark enums with braces in the arguments to enum
1486     // constants as "not simple". This is probably fine in practice, though.
1487     Tok = Tokens->getNextToken();
1488   }
1489   FormatTok = Tokens->setPosition(StoredPosition);
1490 
1491   if (IsSimple) {
1492     parseBracedList();
1493     addUnwrappedLine();
1494     return;
1495   }
1496 
1497   // Parse the body of a more complex enum.
1498   // First add a line for everything up to the "{".
1499   nextToken();
1500   addUnwrappedLine();
1501   ++Line->Level;
1502 
1503   // Parse the enum constants.
1504   while (FormatTok) {
1505     if (FormatTok->is(tok::l_brace)) {
1506       // Parse the constant's class body.
1507       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1508                  /*MunchSemi=*/false);
1509     } else if (FormatTok->is(tok::l_paren)) {
1510       parseParens();
1511     } else if (FormatTok->is(tok::comma)) {
1512       nextToken();
1513       addUnwrappedLine();
1514     } else if (FormatTok->is(tok::semi)) {
1515       nextToken();
1516       addUnwrappedLine();
1517       break;
1518     } else if (FormatTok->is(tok::r_brace)) {
1519       addUnwrappedLine();
1520       break;
1521     } else {
1522       nextToken();
1523     }
1524   }
1525 
1526   // Parse the class body after the enum's ";" if any.
1527   parseLevel(/*HasOpeningBrace=*/true);
1528   nextToken();
1529   --Line->Level;
1530   addUnwrappedLine();
1531 }
1532 
1533 void UnwrappedLineParser::parseRecord() {
1534   const FormatToken &InitialToken = *FormatTok;
1535   nextToken();
1536   if (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::kw___attribute,
1537                          tok::kw___declspec, tok::kw_alignas)) {
1538     nextToken();
1539     // We can have macros or attributes in between 'class' and the class name.
1540     if (FormatTok->Tok.is(tok::l_paren)) {
1541       parseParens();
1542     }
1543     // The actual identifier can be a nested name specifier, and in macros
1544     // it is often token-pasted.
1545     while (FormatTok->is(tok::identifier) || FormatTok->is(tok::coloncolon) ||
1546            FormatTok->is(tok::hashhash) ||
1547            ((Style.Language == FormatStyle::LK_Java ||
1548              Style.Language == FormatStyle::LK_JavaScript) &&
1549             FormatTok->isOneOf(tok::period, tok::comma)))
1550       nextToken();
1551 
1552     // Note that parsing away template declarations here leads to incorrectly
1553     // accepting function declarations as record declarations.
1554     // In general, we cannot solve this problem. Consider:
1555     // class A<int> B() {}
1556     // which can be a function definition or a class definition when B() is a
1557     // macro. If we find enough real-world cases where this is a problem, we
1558     // can parse for the 'template' keyword in the beginning of the statement,
1559     // and thus rule out the record production in case there is no template
1560     // (this would still leave us with an ambiguity between template function
1561     // and class declarations).
1562     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1563       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1564         if (FormatTok->Tok.is(tok::semi))
1565           return;
1566         nextToken();
1567       }
1568     }
1569   }
1570   if (FormatTok->Tok.is(tok::l_brace)) {
1571     if (ShouldBreakBeforeBrace(Style, InitialToken))
1572       addUnwrappedLine();
1573 
1574     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1575                /*MunchSemi=*/false);
1576   }
1577   // We fall through to parsing a structural element afterwards, so
1578   // class A {} n, m;
1579   // will end up in one unwrapped line.
1580   // This does not apply for Java.
1581   if (Style.Language == FormatStyle::LK_Java ||
1582       Style.Language == FormatStyle::LK_JavaScript)
1583     addUnwrappedLine();
1584 }
1585 
1586 void UnwrappedLineParser::parseObjCProtocolList() {
1587   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1588   do
1589     nextToken();
1590   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1591   nextToken(); // Skip '>'.
1592 }
1593 
1594 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1595   do {
1596     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1597       nextToken();
1598       addUnwrappedLine();
1599       break;
1600     }
1601     if (FormatTok->is(tok::l_brace)) {
1602       parseBlock(/*MustBeDeclaration=*/false);
1603       // In ObjC interfaces, nothing should be following the "}".
1604       addUnwrappedLine();
1605     } else if (FormatTok->is(tok::r_brace)) {
1606       // Ignore stray "}". parseStructuralElement doesn't consume them.
1607       nextToken();
1608       addUnwrappedLine();
1609     } else {
1610       parseStructuralElement();
1611     }
1612   } while (!eof());
1613 }
1614 
1615 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1616   nextToken();
1617   nextToken(); // interface name
1618 
1619   // @interface can be followed by either a base class, or a category.
1620   if (FormatTok->Tok.is(tok::colon)) {
1621     nextToken();
1622     nextToken(); // base class name
1623   } else if (FormatTok->Tok.is(tok::l_paren))
1624     // Skip category, if present.
1625     parseParens();
1626 
1627   if (FormatTok->Tok.is(tok::less))
1628     parseObjCProtocolList();
1629 
1630   if (FormatTok->Tok.is(tok::l_brace)) {
1631     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1632         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1633       addUnwrappedLine();
1634     parseBlock(/*MustBeDeclaration=*/true);
1635   }
1636 
1637   // With instance variables, this puts '}' on its own line.  Without instance
1638   // variables, this ends the @interface line.
1639   addUnwrappedLine();
1640 
1641   parseObjCUntilAtEnd();
1642 }
1643 
1644 void UnwrappedLineParser::parseObjCProtocol() {
1645   nextToken();
1646   nextToken(); // protocol name
1647 
1648   if (FormatTok->Tok.is(tok::less))
1649     parseObjCProtocolList();
1650 
1651   // Check for protocol declaration.
1652   if (FormatTok->Tok.is(tok::semi)) {
1653     nextToken();
1654     return addUnwrappedLine();
1655   }
1656 
1657   addUnwrappedLine();
1658   parseObjCUntilAtEnd();
1659 }
1660 
1661 void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
1662   assert(FormatTok->isOneOf(Keywords.kw_import, tok::kw_export));
1663   nextToken();
1664 
1665   if (FormatTok->isOneOf(tok::kw_const, tok::kw_class, Keywords.kw_function,
1666                          Keywords.kw_var))
1667     return; // Fall through to parsing the corresponding structure.
1668 
1669   if (FormatTok->is(tok::kw_default)) {
1670     nextToken(); // export default ..., fall through after eating 'default'.
1671     return;
1672   }
1673 
1674   if (FormatTok->is(tok::l_brace)) {
1675     FormatTok->BlockKind = BK_Block;
1676     parseBracedList();
1677   }
1678 
1679   while (!eof() && FormatTok->isNot(tok::semi) &&
1680          FormatTok->isNot(tok::l_brace)) {
1681     nextToken();
1682   }
1683 }
1684 
1685 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1686                                                  StringRef Prefix = "") {
1687   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1688                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1689   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1690                                                     E = Line.Tokens.end();
1691        I != E; ++I) {
1692     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1693   }
1694   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1695                                                     E = Line.Tokens.end();
1696        I != E; ++I) {
1697     const UnwrappedLineNode &Node = *I;
1698     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1699              I = Node.Children.begin(),
1700              E = Node.Children.end();
1701          I != E; ++I) {
1702       printDebugInfo(*I, "\nChild: ");
1703     }
1704   }
1705   llvm::dbgs() << "\n";
1706 }
1707 
1708 void UnwrappedLineParser::addUnwrappedLine() {
1709   if (Line->Tokens.empty())
1710     return;
1711   DEBUG({
1712     if (CurrentLines == &Lines)
1713       printDebugInfo(*Line);
1714   });
1715   CurrentLines->push_back(*Line);
1716   Line->Tokens.clear();
1717   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1718     for (SmallVectorImpl<UnwrappedLine>::iterator
1719              I = PreprocessorDirectives.begin(),
1720              E = PreprocessorDirectives.end();
1721          I != E; ++I) {
1722       CurrentLines->push_back(*I);
1723     }
1724     PreprocessorDirectives.clear();
1725   }
1726 }
1727 
1728 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1729 
1730 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
1731   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1732          FormatTok.NewlinesBefore > 0;
1733 }
1734 
1735 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1736   bool JustComments = Line->Tokens.empty();
1737   for (SmallVectorImpl<FormatToken *>::const_iterator
1738            I = CommentsBeforeNextToken.begin(),
1739            E = CommentsBeforeNextToken.end();
1740        I != E; ++I) {
1741     if (isOnNewLine(**I) && JustComments) {
1742       addUnwrappedLine();
1743     }
1744     pushToken(*I);
1745   }
1746   if (NewlineBeforeNext && JustComments) {
1747     addUnwrappedLine();
1748   }
1749   CommentsBeforeNextToken.clear();
1750 }
1751 
1752 void UnwrappedLineParser::nextToken() {
1753   if (eof())
1754     return;
1755   flushComments(isOnNewLine(*FormatTok));
1756   pushToken(FormatTok);
1757   readToken();
1758 }
1759 
1760 void UnwrappedLineParser::readToken() {
1761   bool CommentsInCurrentLine = true;
1762   do {
1763     FormatTok = Tokens->getNextToken();
1764     assert(FormatTok);
1765     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1766            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1767       // If there is an unfinished unwrapped line, we flush the preprocessor
1768       // directives only after that unwrapped line was finished later.
1769       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
1770       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1771       // Comments stored before the preprocessor directive need to be output
1772       // before the preprocessor directive, at the same level as the
1773       // preprocessor directive, as we consider them to apply to the directive.
1774       flushComments(isOnNewLine(*FormatTok));
1775       parsePPDirective();
1776     }
1777     while (FormatTok->Type == TT_ConflictStart ||
1778            FormatTok->Type == TT_ConflictEnd ||
1779            FormatTok->Type == TT_ConflictAlternative) {
1780       if (FormatTok->Type == TT_ConflictStart) {
1781         conditionalCompilationStart(/*Unreachable=*/false);
1782       } else if (FormatTok->Type == TT_ConflictAlternative) {
1783         conditionalCompilationAlternative();
1784       } else if (FormatTok->Type == TT_ConflictEnd) {
1785         conditionalCompilationEnd();
1786       }
1787       FormatTok = Tokens->getNextToken();
1788       FormatTok->MustBreakBefore = true;
1789     }
1790 
1791     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1792         !Line->InPPDirective) {
1793       continue;
1794     }
1795 
1796     if (!FormatTok->Tok.is(tok::comment))
1797       return;
1798     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
1799       CommentsInCurrentLine = false;
1800     }
1801     if (CommentsInCurrentLine) {
1802       pushToken(FormatTok);
1803     } else {
1804       CommentsBeforeNextToken.push_back(FormatTok);
1805     }
1806   } while (!eof());
1807 }
1808 
1809 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1810   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1811   if (MustBreakBeforeNextToken) {
1812     Line->Tokens.back().Tok->MustBreakBefore = true;
1813     MustBreakBeforeNextToken = false;
1814   }
1815 }
1816 
1817 } // end namespace format
1818 } // end namespace clang
1819