1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/Support/Debug.h"
19 #include "llvm/Support/raw_ostream.h"
20 
21 #define DEBUG_TYPE "format-parser"
22 
23 namespace clang {
24 namespace format {
25 
26 class FormatTokenSource {
27 public:
28   virtual ~FormatTokenSource() {}
29   virtual FormatToken *getNextToken() = 0;
30 
31   virtual unsigned getPosition() = 0;
32   virtual FormatToken *setPosition(unsigned Position) = 0;
33 };
34 
35 namespace {
36 
37 class ScopedDeclarationState {
38 public:
39   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
40                          bool MustBeDeclaration)
41       : Line(Line), Stack(Stack) {
42     Line.MustBeDeclaration = MustBeDeclaration;
43     Stack.push_back(MustBeDeclaration);
44   }
45   ~ScopedDeclarationState() {
46     Stack.pop_back();
47     if (!Stack.empty())
48       Line.MustBeDeclaration = Stack.back();
49     else
50       Line.MustBeDeclaration = true;
51   }
52 
53 private:
54   UnwrappedLine &Line;
55   std::vector<bool> &Stack;
56 };
57 
58 class ScopedMacroState : public FormatTokenSource {
59 public:
60   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
61                    FormatToken *&ResetToken, bool &StructuralError)
62       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
63         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
64         StructuralError(StructuralError),
65         PreviousStructuralError(StructuralError), Token(nullptr) {
66     TokenSource = this;
67     Line.Level = 0;
68     Line.InPPDirective = true;
69   }
70 
71   ~ScopedMacroState() {
72     TokenSource = PreviousTokenSource;
73     ResetToken = Token;
74     Line.InPPDirective = false;
75     Line.Level = PreviousLineLevel;
76     StructuralError = PreviousStructuralError;
77   }
78 
79   FormatToken *getNextToken() override {
80     // The \c UnwrappedLineParser guards against this by never calling
81     // \c getNextToken() after it has encountered the first eof token.
82     assert(!eof());
83     Token = PreviousTokenSource->getNextToken();
84     if (eof())
85       return getFakeEOF();
86     return Token;
87   }
88 
89   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
90 
91   FormatToken *setPosition(unsigned Position) override {
92     Token = PreviousTokenSource->setPosition(Position);
93     return Token;
94   }
95 
96 private:
97   bool eof() { return Token && Token->HasUnescapedNewline; }
98 
99   FormatToken *getFakeEOF() {
100     static bool EOFInitialized = false;
101     static FormatToken FormatTok;
102     if (!EOFInitialized) {
103       FormatTok.Tok.startToken();
104       FormatTok.Tok.setKind(tok::eof);
105       EOFInitialized = true;
106     }
107     return &FormatTok;
108   }
109 
110   UnwrappedLine &Line;
111   FormatTokenSource *&TokenSource;
112   FormatToken *&ResetToken;
113   unsigned PreviousLineLevel;
114   FormatTokenSource *PreviousTokenSource;
115   bool &StructuralError;
116   bool PreviousStructuralError;
117 
118   FormatToken *Token;
119 };
120 
121 } // end anonymous namespace
122 
123 class ScopedLineState {
124 public:
125   ScopedLineState(UnwrappedLineParser &Parser,
126                   bool SwitchToPreprocessorLines = false)
127       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
128     if (SwitchToPreprocessorLines)
129       Parser.CurrentLines = &Parser.PreprocessorDirectives;
130     else if (!Parser.Line->Tokens.empty())
131       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
132     PreBlockLine = std::move(Parser.Line);
133     Parser.Line = llvm::make_unique<UnwrappedLine>();
134     Parser.Line->Level = PreBlockLine->Level;
135     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
136   }
137 
138   ~ScopedLineState() {
139     if (!Parser.Line->Tokens.empty()) {
140       Parser.addUnwrappedLine();
141     }
142     assert(Parser.Line->Tokens.empty());
143     Parser.Line = std::move(PreBlockLine);
144     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
145       Parser.MustBreakBeforeNextToken = true;
146     Parser.CurrentLines = OriginalLines;
147   }
148 
149 private:
150   UnwrappedLineParser &Parser;
151 
152   std::unique_ptr<UnwrappedLine> PreBlockLine;
153   SmallVectorImpl<UnwrappedLine> *OriginalLines;
154 };
155 
156 class CompoundStatementIndenter {
157 public:
158   CompoundStatementIndenter(UnwrappedLineParser *Parser,
159                             const FormatStyle &Style, unsigned &LineLevel)
160       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
161     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
162       Parser->addUnwrappedLine();
163     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
164       Parser->addUnwrappedLine();
165       ++LineLevel;
166     }
167   }
168   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
169 
170 private:
171   unsigned &LineLevel;
172   unsigned OldLineLevel;
173 };
174 
175 namespace {
176 
177 class IndexedTokenSource : public FormatTokenSource {
178 public:
179   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
180       : Tokens(Tokens), Position(-1) {}
181 
182   FormatToken *getNextToken() override {
183     ++Position;
184     return Tokens[Position];
185   }
186 
187   unsigned getPosition() override {
188     assert(Position >= 0);
189     return Position;
190   }
191 
192   FormatToken *setPosition(unsigned P) override {
193     Position = P;
194     return Tokens[Position];
195   }
196 
197   void reset() { Position = -1; }
198 
199 private:
200   ArrayRef<FormatToken *> Tokens;
201   int Position;
202 };
203 
204 } // end anonymous namespace
205 
206 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
207                                          const AdditionalKeywords &Keywords,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style),
212       Keywords(Keywords), Tokens(nullptr), Callback(Callback),
213       AllTokens(Tokens), PPBranchLevel(-1) {}
214 
215 void UnwrappedLineParser::reset() {
216   PPBranchLevel = -1;
217   Line.reset(new UnwrappedLine);
218   CommentsBeforeNextToken.clear();
219   FormatTok = nullptr;
220   MustBreakBeforeNextToken = false;
221   PreprocessorDirectives.clear();
222   CurrentLines = &Lines;
223   DeclarationScopeStack.clear();
224   StructuralError = false;
225   PPStack.clear();
226 }
227 
228 bool UnwrappedLineParser::parse() {
229   IndexedTokenSource TokenSource(AllTokens);
230   do {
231     DEBUG(llvm::dbgs() << "----\n");
232     reset();
233     Tokens = &TokenSource;
234     TokenSource.reset();
235 
236     readToken();
237     parseFile();
238     // Create line with eof token.
239     pushToken(FormatTok);
240     addUnwrappedLine();
241 
242     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
243                                                   E = Lines.end();
244          I != E; ++I) {
245       Callback.consumeUnwrappedLine(*I);
246     }
247     Callback.finishRun();
248     Lines.clear();
249     while (!PPLevelBranchIndex.empty() &&
250            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
251       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
252       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
253     }
254     if (!PPLevelBranchIndex.empty()) {
255       ++PPLevelBranchIndex.back();
256       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
257       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
258     }
259   } while (!PPLevelBranchIndex.empty());
260 
261   return StructuralError;
262 }
263 
264 void UnwrappedLineParser::parseFile() {
265   ScopedDeclarationState DeclarationState(
266       *Line, DeclarationScopeStack,
267       /*MustBeDeclaration=*/!Line->InPPDirective);
268   parseLevel(/*HasOpeningBrace=*/false);
269   // Make sure to format the remaining tokens.
270   flushComments(true);
271   addUnwrappedLine();
272 }
273 
274 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
275   bool SwitchLabelEncountered = false;
276   do {
277     switch (FormatTok->Tok.getKind()) {
278     case tok::comment:
279       nextToken();
280       addUnwrappedLine();
281       break;
282     case tok::l_brace:
283       // FIXME: Add parameter whether this can happen - if this happens, we must
284       // be in a non-declaration context.
285       parseBlock(/*MustBeDeclaration=*/false);
286       addUnwrappedLine();
287       break;
288     case tok::r_brace:
289       if (HasOpeningBrace)
290         return;
291       StructuralError = true;
292       nextToken();
293       addUnwrappedLine();
294       break;
295     case tok::kw_default:
296     case tok::kw_case:
297       if (!SwitchLabelEncountered &&
298           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
299         ++Line->Level;
300       SwitchLabelEncountered = true;
301       parseStructuralElement();
302       break;
303     default:
304       parseStructuralElement();
305       break;
306     }
307   } while (!eof());
308 }
309 
310 void UnwrappedLineParser::calculateBraceTypes() {
311   // We'll parse forward through the tokens until we hit
312   // a closing brace or eof - note that getNextToken() will
313   // parse macros, so this will magically work inside macro
314   // definitions, too.
315   unsigned StoredPosition = Tokens->getPosition();
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           bool ProbablyBracedList = false;
339           if (Style.Language == FormatStyle::LK_Proto) {
340             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
341           } else {
342             // Using OriginalColumn to distinguish between ObjC methods and
343             // binary operators is a bit hacky.
344             bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
345                                     NextTok->OriginalColumn == 0;
346 
347             // If there is a comma, semicolon or right paren after the closing
348             // brace, we assume this is a braced initializer list.  Note that
349             // regardless how we mark inner braces here, we will overwrite the
350             // BlockKind later if we parse a braced list (where all blocks
351             // inside are by default braced lists), or when we explicitly detect
352             // blocks (for example while parsing lambdas).
353             //
354             // We exclude + and - as they can be ObjC visibility modifiers.
355             ProbablyBracedList =
356                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
357                                  tok::r_paren, tok::r_square, tok::l_brace,
358                                  tok::l_paren, tok::ellipsis) ||
359                 (NextTok->isBinaryOperator() && !NextIsObjCMethod);
360           }
361           if (ProbablyBracedList) {
362             Tok->BlockKind = BK_BracedInit;
363             LBraceStack.back()->BlockKind = BK_BracedInit;
364           } else {
365             Tok->BlockKind = BK_Block;
366             LBraceStack.back()->BlockKind = BK_Block;
367           }
368         }
369         LBraceStack.pop_back();
370       }
371       break;
372     case tok::at:
373     case tok::semi:
374     case tok::kw_if:
375     case tok::kw_while:
376     case tok::kw_for:
377     case tok::kw_switch:
378     case tok::kw_try:
379     case tok::kw___try:
380       if (!LBraceStack.empty())
381         LBraceStack.back()->BlockKind = BK_Block;
382       break;
383     default:
384       break;
385     }
386     Tok = NextTok;
387   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
388   // Assume other blocks for all unclosed opening braces.
389   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
390     if (LBraceStack[i]->BlockKind == BK_Unknown)
391       LBraceStack[i]->BlockKind = BK_Block;
392   }
393 
394   FormatTok = Tokens->setPosition(StoredPosition);
395 }
396 
397 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
398                                      bool MunchSemi) {
399   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
400   unsigned InitialLevel = Line->Level;
401   nextToken();
402 
403   addUnwrappedLine();
404 
405   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
406                                           MustBeDeclaration);
407   if (AddLevel)
408     ++Line->Level;
409   parseLevel(/*HasOpeningBrace=*/true);
410 
411   if (!FormatTok->Tok.is(tok::r_brace)) {
412     Line->Level = InitialLevel;
413     StructuralError = true;
414     return;
415   }
416 
417   nextToken(); // Munch the closing brace.
418   if (MunchSemi && FormatTok->Tok.is(tok::semi))
419     nextToken();
420   Line->Level = InitialLevel;
421 }
422 
423 static bool isGoogScope(const UnwrappedLine &Line) {
424   // FIXME: Closure-library specific stuff should not be hard-coded but be
425   // configurable.
426   if (Line.Tokens.size() < 4)
427     return false;
428   auto I = Line.Tokens.begin();
429   if (I->Tok->TokenText != "goog")
430     return false;
431   ++I;
432   if (I->Tok->isNot(tok::period))
433     return false;
434   ++I;
435   if (I->Tok->TokenText != "scope")
436     return false;
437   ++I;
438   return I->Tok->is(tok::l_paren);
439 }
440 
441 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
442                                    const FormatToken &InitialToken) {
443   switch (Style.BreakBeforeBraces) {
444   case FormatStyle::BS_Linux:
445     return InitialToken.isOneOf(tok::kw_namespace, tok::kw_class);
446   case FormatStyle::BS_Allman:
447   case FormatStyle::BS_GNU:
448     return true;
449   default:
450     return false;
451   }
452 }
453 
454 void UnwrappedLineParser::parseChildBlock() {
455   FormatTok->BlockKind = BK_Block;
456   nextToken();
457   {
458     bool GoogScope =
459         Style.Language == FormatStyle::LK_JavaScript && isGoogScope(*Line);
460     ScopedLineState LineState(*this);
461     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
462                                             /*MustBeDeclaration=*/false);
463     Line->Level += GoogScope ? 0 : 1;
464     parseLevel(/*HasOpeningBrace=*/true);
465     flushComments(isOnNewLine(*FormatTok));
466     Line->Level -= GoogScope ? 0 : 1;
467   }
468   nextToken();
469 }
470 
471 void UnwrappedLineParser::parsePPDirective() {
472   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
473   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
474   nextToken();
475 
476   if (!FormatTok->Tok.getIdentifierInfo()) {
477     parsePPUnknown();
478     return;
479   }
480 
481   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
482   case tok::pp_define:
483     parsePPDefine();
484     return;
485   case tok::pp_if:
486     parsePPIf(/*IfDef=*/false);
487     break;
488   case tok::pp_ifdef:
489   case tok::pp_ifndef:
490     parsePPIf(/*IfDef=*/true);
491     break;
492   case tok::pp_else:
493     parsePPElse();
494     break;
495   case tok::pp_elif:
496     parsePPElIf();
497     break;
498   case tok::pp_endif:
499     parsePPEndIf();
500     break;
501   default:
502     parsePPUnknown();
503     break;
504   }
505 }
506 
507 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
508   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
509     PPStack.push_back(PP_Unreachable);
510   else
511     PPStack.push_back(PP_Conditional);
512 }
513 
514 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
515   ++PPBranchLevel;
516   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
517   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
518     PPLevelBranchIndex.push_back(0);
519     PPLevelBranchCount.push_back(0);
520   }
521   PPChainBranchIndex.push(0);
522   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
523   conditionalCompilationCondition(Unreachable || Skip);
524 }
525 
526 void UnwrappedLineParser::conditionalCompilationAlternative() {
527   if (!PPStack.empty())
528     PPStack.pop_back();
529   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
530   if (!PPChainBranchIndex.empty())
531     ++PPChainBranchIndex.top();
532   conditionalCompilationCondition(
533       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
534       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
535 }
536 
537 void UnwrappedLineParser::conditionalCompilationEnd() {
538   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
539   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
540     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
541       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
542     }
543   }
544   // Guard against #endif's without #if.
545   if (PPBranchLevel > 0)
546     --PPBranchLevel;
547   if (!PPChainBranchIndex.empty())
548     PPChainBranchIndex.pop();
549   if (!PPStack.empty())
550     PPStack.pop_back();
551 }
552 
553 void UnwrappedLineParser::parsePPIf(bool IfDef) {
554   nextToken();
555   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
556                          FormatTok->Tok.getLiteralData() != nullptr &&
557                          StringRef(FormatTok->Tok.getLiteralData(),
558                                    FormatTok->Tok.getLength()) == "0") ||
559                         FormatTok->Tok.is(tok::kw_false);
560   conditionalCompilationStart(!IfDef && IsLiteralFalse);
561   parsePPUnknown();
562 }
563 
564 void UnwrappedLineParser::parsePPElse() {
565   conditionalCompilationAlternative();
566   parsePPUnknown();
567 }
568 
569 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
570 
571 void UnwrappedLineParser::parsePPEndIf() {
572   conditionalCompilationEnd();
573   parsePPUnknown();
574 }
575 
576 void UnwrappedLineParser::parsePPDefine() {
577   nextToken();
578 
579   if (FormatTok->Tok.getKind() != tok::identifier) {
580     parsePPUnknown();
581     return;
582   }
583   nextToken();
584   if (FormatTok->Tok.getKind() == tok::l_paren &&
585       FormatTok->WhitespaceRange.getBegin() ==
586           FormatTok->WhitespaceRange.getEnd()) {
587     parseParens();
588   }
589   addUnwrappedLine();
590   Line->Level = 1;
591 
592   // Errors during a preprocessor directive can only affect the layout of the
593   // preprocessor directive, and thus we ignore them. An alternative approach
594   // would be to use the same approach we use on the file level (no
595   // re-indentation if there was a structural error) within the macro
596   // definition.
597   parseFile();
598 }
599 
600 void UnwrappedLineParser::parsePPUnknown() {
601   do {
602     nextToken();
603   } while (!eof());
604   addUnwrappedLine();
605 }
606 
607 // Here we blacklist certain tokens that are not usually the first token in an
608 // unwrapped line. This is used in attempt to distinguish macro calls without
609 // trailing semicolons from other constructs split to several lines.
610 static bool tokenCanStartNewLine(const clang::Token &Tok) {
611   // Semicolon can be a null-statement, l_square can be a start of a macro or
612   // a C++11 attribute, but this doesn't seem to be common.
613   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
614          Tok.isNot(tok::l_square) &&
615          // Tokens that can only be used as binary operators and a part of
616          // overloaded operator names.
617          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
618          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
619          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
620          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
621          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
622          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
623          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
624          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
625          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
626          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
627          Tok.isNot(tok::lesslessequal) &&
628          // Colon is used in labels, base class lists, initializer lists,
629          // range-based for loops, ternary operator, but should never be the
630          // first token in an unwrapped line.
631          Tok.isNot(tok::colon) &&
632          // 'noexcept' is a trailing annotation.
633          Tok.isNot(tok::kw_noexcept);
634 }
635 
636 void UnwrappedLineParser::parseStructuralElement() {
637   assert(!FormatTok->Tok.is(tok::l_brace));
638   switch (FormatTok->Tok.getKind()) {
639   case tok::at:
640     nextToken();
641     if (FormatTok->Tok.is(tok::l_brace)) {
642       parseBracedList();
643       break;
644     }
645     switch (FormatTok->Tok.getObjCKeywordID()) {
646     case tok::objc_public:
647     case tok::objc_protected:
648     case tok::objc_package:
649     case tok::objc_private:
650       return parseAccessSpecifier();
651     case tok::objc_interface:
652     case tok::objc_implementation:
653       return parseObjCInterfaceOrImplementation();
654     case tok::objc_protocol:
655       return parseObjCProtocol();
656     case tok::objc_end:
657       return; // Handled by the caller.
658     case tok::objc_optional:
659     case tok::objc_required:
660       nextToken();
661       addUnwrappedLine();
662       return;
663     case tok::objc_try:
664       // This branch isn't strictly necessary (the kw_try case below would
665       // do this too after the tok::at is parsed above).  But be explicit.
666       parseTryCatch();
667       return;
668     default:
669       break;
670     }
671     break;
672   case tok::kw_asm:
673     nextToken();
674     if (FormatTok->is(tok::l_brace)) {
675       nextToken();
676       while (FormatTok && FormatTok->isNot(tok::eof)) {
677         if (FormatTok->is(tok::r_brace)) {
678           nextToken();
679           break;
680         }
681         FormatTok->Finalized = true;
682         nextToken();
683       }
684     }
685     break;
686   case tok::kw_namespace:
687     parseNamespace();
688     return;
689   case tok::kw_inline:
690     nextToken();
691     if (FormatTok->Tok.is(tok::kw_namespace)) {
692       parseNamespace();
693       return;
694     }
695     break;
696   case tok::kw_public:
697   case tok::kw_protected:
698   case tok::kw_private:
699     if (Style.Language == FormatStyle::LK_Java ||
700         Style.Language == FormatStyle::LK_JavaScript)
701       nextToken();
702     else
703       parseAccessSpecifier();
704     return;
705   case tok::kw_if:
706     parseIfThenElse();
707     return;
708   case tok::kw_for:
709   case tok::kw_while:
710     parseForOrWhileLoop();
711     return;
712   case tok::kw_do:
713     parseDoWhile();
714     return;
715   case tok::kw_switch:
716     parseSwitch();
717     return;
718   case tok::kw_default:
719     nextToken();
720     parseLabel();
721     return;
722   case tok::kw_case:
723     parseCaseLabel();
724     return;
725   case tok::kw_try:
726   case tok::kw___try:
727     parseTryCatch();
728     return;
729   case tok::kw_extern:
730     nextToken();
731     if (FormatTok->Tok.is(tok::string_literal)) {
732       nextToken();
733       if (FormatTok->Tok.is(tok::l_brace)) {
734         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
735         addUnwrappedLine();
736         return;
737       }
738     }
739     break;
740   case tok::kw_export:
741     if (Style.Language == FormatStyle::LK_JavaScript) {
742       parseJavaScriptEs6ImportExport();
743       return;
744     }
745     break;
746   case tok::identifier:
747     if (FormatTok->IsForEachMacro) {
748       parseForOrWhileLoop();
749       return;
750     }
751     if (Style.Language == FormatStyle::LK_JavaScript &&
752         FormatTok->is(Keywords.kw_import)) {
753       parseJavaScriptEs6ImportExport();
754       return;
755     }
756     // In all other cases, parse the declaration.
757     break;
758   default:
759     break;
760   }
761   do {
762     switch (FormatTok->Tok.getKind()) {
763     case tok::at:
764       nextToken();
765       if (FormatTok->Tok.is(tok::l_brace))
766         parseBracedList();
767       break;
768     case tok::kw_enum:
769       parseEnum();
770       break;
771     case tok::kw_typedef:
772       nextToken();
773       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
774                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
775         parseEnum();
776       break;
777     case tok::kw_struct:
778     case tok::kw_union:
779     case tok::kw_class:
780       parseRecord();
781       // A record declaration or definition is always the start of a structural
782       // element.
783       break;
784     case tok::period:
785       nextToken();
786       // In Java, classes have an implicit static member "class".
787       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
788           FormatTok->is(tok::kw_class))
789         nextToken();
790       break;
791     case tok::semi:
792       nextToken();
793       addUnwrappedLine();
794       return;
795     case tok::r_brace:
796       addUnwrappedLine();
797       return;
798     case tok::l_paren:
799       parseParens();
800       break;
801     case tok::caret:
802       nextToken();
803       if (FormatTok->Tok.isAnyIdentifier() ||
804           FormatTok->isSimpleTypeSpecifier())
805         nextToken();
806       if (FormatTok->is(tok::l_paren))
807         parseParens();
808       if (FormatTok->is(tok::l_brace))
809         parseChildBlock();
810       break;
811     case tok::l_brace:
812       if (!tryToParseBracedList()) {
813         // A block outside of parentheses must be the last part of a
814         // structural element.
815         // FIXME: Figure out cases where this is not true, and add projections
816         // for them (the one we know is missing are lambdas).
817         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
818           addUnwrappedLine();
819         FormatTok->Type = TT_FunctionLBrace;
820         parseBlock(/*MustBeDeclaration=*/false);
821         addUnwrappedLine();
822         return;
823       }
824       // Otherwise this was a braced init list, and the structural
825       // element continues.
826       break;
827     case tok::kw_try:
828       // We arrive here when parsing function-try blocks.
829       parseTryCatch();
830       return;
831     case tok::identifier: {
832       StringRef Text = FormatTok->TokenText;
833       // Parse function literal unless 'function' is the first token in a line
834       // in which case this should be treated as a free-standing function.
835       if (Style.Language == FormatStyle::LK_JavaScript && Text == "function" &&
836           Line->Tokens.size() > 0) {
837         tryToParseJSFunction();
838         break;
839       }
840       nextToken();
841       if (Line->Tokens.size() == 1 &&
842           // JS doesn't have macros, and within classes colons indicate fields,
843           // not labels.
844           (Style.Language != FormatStyle::LK_JavaScript ||
845            !Line->MustBeDeclaration)) {
846         if (FormatTok->Tok.is(tok::colon)) {
847           parseLabel();
848           return;
849         }
850         // Recognize function-like macro usages without trailing semicolon as
851         // well as free-standing macros like Q_OBJECT.
852         bool FunctionLike = FormatTok->is(tok::l_paren);
853         if (FunctionLike)
854           parseParens();
855         if (FormatTok->NewlinesBefore > 0 &&
856             (Text.size() >= 5 || FunctionLike) &&
857             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
858           addUnwrappedLine();
859           return;
860         }
861       }
862       break;
863     }
864     case tok::equal:
865       nextToken();
866       if (FormatTok->Tok.is(tok::l_brace)) {
867         parseBracedList();
868       }
869       break;
870     case tok::l_square:
871       parseSquare();
872       break;
873     case tok::kw_new:
874       parseNew();
875       break;
876     default:
877       nextToken();
878       break;
879     }
880   } while (!eof());
881 }
882 
883 bool UnwrappedLineParser::tryToParseLambda() {
884   // FIXME: This is a dirty way to access the previous token. Find a better
885   // solution.
886   if (!Line->Tokens.empty() &&
887       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator,
888                                         tok::kw_new, tok::kw_delete) ||
889        Line->Tokens.back().Tok->closesScope() ||
890        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
891     nextToken();
892     return false;
893   }
894   assert(FormatTok->is(tok::l_square));
895   FormatToken &LSquare = *FormatTok;
896   if (!tryToParseLambdaIntroducer())
897     return false;
898 
899   while (FormatTok->isNot(tok::l_brace)) {
900     if (FormatTok->isSimpleTypeSpecifier()) {
901       nextToken();
902       continue;
903     }
904     switch (FormatTok->Tok.getKind()) {
905     case tok::l_brace:
906       break;
907     case tok::l_paren:
908       parseParens();
909       break;
910     case tok::amp:
911     case tok::star:
912     case tok::kw_const:
913     case tok::comma:
914     case tok::less:
915     case tok::greater:
916     case tok::identifier:
917     case tok::coloncolon:
918     case tok::kw_mutable:
919       nextToken();
920       break;
921     case tok::arrow:
922       FormatTok->Type = TT_TrailingReturnArrow;
923       nextToken();
924       break;
925     default:
926       return true;
927     }
928   }
929   LSquare.Type = TT_LambdaLSquare;
930   parseChildBlock();
931   return true;
932 }
933 
934 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
935   nextToken();
936   if (FormatTok->is(tok::equal)) {
937     nextToken();
938     if (FormatTok->is(tok::r_square)) {
939       nextToken();
940       return true;
941     }
942     if (FormatTok->isNot(tok::comma))
943       return false;
944     nextToken();
945   } else if (FormatTok->is(tok::amp)) {
946     nextToken();
947     if (FormatTok->is(tok::r_square)) {
948       nextToken();
949       return true;
950     }
951     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
952       return false;
953     }
954     if (FormatTok->is(tok::comma))
955       nextToken();
956   } else if (FormatTok->is(tok::r_square)) {
957     nextToken();
958     return true;
959   }
960   do {
961     if (FormatTok->is(tok::amp))
962       nextToken();
963     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
964       return false;
965     nextToken();
966     if (FormatTok->is(tok::ellipsis))
967       nextToken();
968     if (FormatTok->is(tok::comma)) {
969       nextToken();
970     } else if (FormatTok->is(tok::r_square)) {
971       nextToken();
972       return true;
973     } else {
974       return false;
975     }
976   } while (!eof());
977   return false;
978 }
979 
980 void UnwrappedLineParser::tryToParseJSFunction() {
981   nextToken();
982 
983   // Consume function name.
984   if (FormatTok->is(tok::identifier))
985     nextToken();
986 
987   if (FormatTok->isNot(tok::l_paren))
988     return;
989   nextToken();
990   while (FormatTok->isNot(tok::l_brace)) {
991     // Err on the side of caution in order to avoid consuming the full file in
992     // case of incomplete code.
993     if (!FormatTok->isOneOf(tok::identifier, tok::comma, tok::r_paren,
994                             tok::comment))
995       return;
996     nextToken();
997   }
998   parseChildBlock();
999 }
1000 
1001 bool UnwrappedLineParser::tryToParseBracedList() {
1002   if (FormatTok->BlockKind == BK_Unknown)
1003     calculateBraceTypes();
1004   assert(FormatTok->BlockKind != BK_Unknown);
1005   if (FormatTok->BlockKind == BK_Block)
1006     return false;
1007   parseBracedList();
1008   return true;
1009 }
1010 
1011 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
1012   bool HasError = false;
1013   nextToken();
1014 
1015   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
1016   // replace this by using parseAssigmentExpression() inside.
1017   do {
1018     if (Style.Language == FormatStyle::LK_JavaScript &&
1019         FormatTok->is(Keywords.kw_function)) {
1020       tryToParseJSFunction();
1021       continue;
1022     }
1023     switch (FormatTok->Tok.getKind()) {
1024     case tok::caret:
1025       nextToken();
1026       if (FormatTok->is(tok::l_brace)) {
1027         parseChildBlock();
1028       }
1029       break;
1030     case tok::l_square:
1031       tryToParseLambda();
1032       break;
1033     case tok::l_brace:
1034       // Assume there are no blocks inside a braced init list apart
1035       // from the ones we explicitly parse out (like lambdas).
1036       FormatTok->BlockKind = BK_BracedInit;
1037       parseBracedList();
1038       break;
1039     case tok::r_paren:
1040       // JavaScript can just have free standing methods and getters/setters in
1041       // object literals. Detect them by a "{" following ")".
1042       if (Style.Language == FormatStyle::LK_JavaScript) {
1043         nextToken();
1044         if (FormatTok->is(tok::l_brace))
1045           parseChildBlock();
1046         break;
1047       }
1048       nextToken();
1049       break;
1050     case tok::r_brace:
1051       nextToken();
1052       return !HasError;
1053     case tok::semi:
1054       HasError = true;
1055       if (!ContinueOnSemicolons)
1056         return !HasError;
1057       nextToken();
1058       break;
1059     case tok::comma:
1060       nextToken();
1061       break;
1062     default:
1063       nextToken();
1064       break;
1065     }
1066   } while (!eof());
1067   return false;
1068 }
1069 
1070 void UnwrappedLineParser::parseParens() {
1071   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1072   nextToken();
1073   do {
1074     switch (FormatTok->Tok.getKind()) {
1075     case tok::l_paren:
1076       parseParens();
1077       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1078         parseChildBlock();
1079       break;
1080     case tok::r_paren:
1081       nextToken();
1082       return;
1083     case tok::r_brace:
1084       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1085       return;
1086     case tok::l_square:
1087       tryToParseLambda();
1088       break;
1089     case tok::l_brace:
1090       if (!tryToParseBracedList()) {
1091         parseChildBlock();
1092       }
1093       break;
1094     case tok::at:
1095       nextToken();
1096       if (FormatTok->Tok.is(tok::l_brace))
1097         parseBracedList();
1098       break;
1099     case tok::identifier:
1100       if (Style.Language == FormatStyle::LK_JavaScript &&
1101           FormatTok->is(Keywords.kw_function))
1102         tryToParseJSFunction();
1103       else
1104         nextToken();
1105       break;
1106     default:
1107       nextToken();
1108       break;
1109     }
1110   } while (!eof());
1111 }
1112 
1113 void UnwrappedLineParser::parseSquare() {
1114   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1115   if (tryToParseLambda())
1116     return;
1117   do {
1118     switch (FormatTok->Tok.getKind()) {
1119     case tok::l_paren:
1120       parseParens();
1121       break;
1122     case tok::r_square:
1123       nextToken();
1124       return;
1125     case tok::r_brace:
1126       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1127       return;
1128     case tok::l_square:
1129       parseSquare();
1130       break;
1131     case tok::l_brace: {
1132       if (!tryToParseBracedList()) {
1133         parseChildBlock();
1134       }
1135       break;
1136     }
1137     case tok::at:
1138       nextToken();
1139       if (FormatTok->Tok.is(tok::l_brace))
1140         parseBracedList();
1141       break;
1142     default:
1143       nextToken();
1144       break;
1145     }
1146   } while (!eof());
1147 }
1148 
1149 void UnwrappedLineParser::parseIfThenElse() {
1150   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1151   nextToken();
1152   if (FormatTok->Tok.is(tok::l_paren))
1153     parseParens();
1154   bool NeedsUnwrappedLine = false;
1155   if (FormatTok->Tok.is(tok::l_brace)) {
1156     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1157     parseBlock(/*MustBeDeclaration=*/false);
1158     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1159         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1160       addUnwrappedLine();
1161     } else {
1162       NeedsUnwrappedLine = true;
1163     }
1164   } else {
1165     addUnwrappedLine();
1166     ++Line->Level;
1167     parseStructuralElement();
1168     --Line->Level;
1169   }
1170   if (FormatTok->Tok.is(tok::kw_else)) {
1171     if (Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup)
1172       addUnwrappedLine();
1173     nextToken();
1174     if (FormatTok->Tok.is(tok::l_brace)) {
1175       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1176       parseBlock(/*MustBeDeclaration=*/false);
1177       addUnwrappedLine();
1178     } else if (FormatTok->Tok.is(tok::kw_if)) {
1179       parseIfThenElse();
1180     } else {
1181       addUnwrappedLine();
1182       ++Line->Level;
1183       parseStructuralElement();
1184       --Line->Level;
1185     }
1186   } else if (NeedsUnwrappedLine) {
1187     addUnwrappedLine();
1188   }
1189 }
1190 
1191 void UnwrappedLineParser::parseTryCatch() {
1192   assert(FormatTok->isOneOf(tok::kw_try, tok::kw___try) && "'try' expected");
1193   nextToken();
1194   bool NeedsUnwrappedLine = false;
1195   if (FormatTok->is(tok::colon)) {
1196     // We are in a function try block, what comes is an initializer list.
1197     nextToken();
1198     while (FormatTok->is(tok::identifier)) {
1199       nextToken();
1200       if (FormatTok->is(tok::l_paren))
1201         parseParens();
1202       else
1203         StructuralError = true;
1204       if (FormatTok->is(tok::comma))
1205         nextToken();
1206     }
1207   }
1208   // Parse try with resource.
1209   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1210     parseParens();
1211   }
1212   if (FormatTok->is(tok::l_brace)) {
1213     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1214     parseBlock(/*MustBeDeclaration=*/false);
1215     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1216         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1217         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1218       addUnwrappedLine();
1219     } else {
1220       NeedsUnwrappedLine = true;
1221     }
1222   } else if (!FormatTok->is(tok::kw_catch)) {
1223     // The C++ standard requires a compound-statement after a try.
1224     // If there's none, we try to assume there's a structuralElement
1225     // and try to continue.
1226     StructuralError = true;
1227     addUnwrappedLine();
1228     ++Line->Level;
1229     parseStructuralElement();
1230     --Line->Level;
1231   }
1232   while (1) {
1233     if (FormatTok->is(tok::at))
1234       nextToken();
1235     if (!(FormatTok->isOneOf(tok::kw_catch, Keywords.kw___except,
1236                              tok::kw___finally) ||
1237           ((Style.Language == FormatStyle::LK_Java ||
1238             Style.Language == FormatStyle::LK_JavaScript) &&
1239            FormatTok->is(Keywords.kw_finally)) ||
1240           (FormatTok->Tok.isObjCAtKeyword(tok::objc_catch) ||
1241            FormatTok->Tok.isObjCAtKeyword(tok::objc_finally))))
1242       break;
1243     nextToken();
1244     while (FormatTok->isNot(tok::l_brace)) {
1245       if (FormatTok->is(tok::l_paren)) {
1246         parseParens();
1247         continue;
1248       }
1249       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1250         return;
1251       nextToken();
1252     }
1253     NeedsUnwrappedLine = false;
1254     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1255     parseBlock(/*MustBeDeclaration=*/false);
1256     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1257         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1258         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1259       addUnwrappedLine();
1260     } else {
1261       NeedsUnwrappedLine = true;
1262     }
1263   }
1264   if (NeedsUnwrappedLine) {
1265     addUnwrappedLine();
1266   }
1267 }
1268 
1269 void UnwrappedLineParser::parseNamespace() {
1270   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1271 
1272   const FormatToken &InitialToken = *FormatTok;
1273   nextToken();
1274   if (FormatTok->Tok.is(tok::identifier))
1275     nextToken();
1276   if (FormatTok->Tok.is(tok::l_brace)) {
1277     if (ShouldBreakBeforeBrace(Style, InitialToken))
1278       addUnwrappedLine();
1279 
1280     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1281                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1282                      DeclarationScopeStack.size() > 1);
1283     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1284     // Munch the semicolon after a namespace. This is more common than one would
1285     // think. Puttin the semicolon into its own line is very ugly.
1286     if (FormatTok->Tok.is(tok::semi))
1287       nextToken();
1288     addUnwrappedLine();
1289   }
1290   // FIXME: Add error handling.
1291 }
1292 
1293 void UnwrappedLineParser::parseNew() {
1294   assert(FormatTok->is(tok::kw_new) && "'new' expected");
1295   nextToken();
1296   if (Style.Language != FormatStyle::LK_Java)
1297     return;
1298 
1299   // In Java, we can parse everything up to the parens, which aren't optional.
1300   do {
1301     // There should not be a ;, { or } before the new's open paren.
1302     if (FormatTok->isOneOf(tok::semi, tok::l_brace, tok::r_brace))
1303       return;
1304 
1305     // Consume the parens.
1306     if (FormatTok->is(tok::l_paren)) {
1307       parseParens();
1308 
1309       // If there is a class body of an anonymous class, consume that as child.
1310       if (FormatTok->is(tok::l_brace))
1311         parseChildBlock();
1312       return;
1313     }
1314     nextToken();
1315   } while (!eof());
1316 }
1317 
1318 void UnwrappedLineParser::parseForOrWhileLoop() {
1319   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1320           FormatTok->IsForEachMacro) &&
1321          "'for', 'while' or foreach macro expected");
1322   nextToken();
1323   if (FormatTok->Tok.is(tok::l_paren))
1324     parseParens();
1325   if (FormatTok->Tok.is(tok::l_brace)) {
1326     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1327     parseBlock(/*MustBeDeclaration=*/false);
1328     addUnwrappedLine();
1329   } else {
1330     addUnwrappedLine();
1331     ++Line->Level;
1332     parseStructuralElement();
1333     --Line->Level;
1334   }
1335 }
1336 
1337 void UnwrappedLineParser::parseDoWhile() {
1338   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1339   nextToken();
1340   if (FormatTok->Tok.is(tok::l_brace)) {
1341     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1342     parseBlock(/*MustBeDeclaration=*/false);
1343     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1344       addUnwrappedLine();
1345   } else {
1346     addUnwrappedLine();
1347     ++Line->Level;
1348     parseStructuralElement();
1349     --Line->Level;
1350   }
1351 
1352   // FIXME: Add error handling.
1353   if (!FormatTok->Tok.is(tok::kw_while)) {
1354     addUnwrappedLine();
1355     return;
1356   }
1357 
1358   nextToken();
1359   parseStructuralElement();
1360 }
1361 
1362 void UnwrappedLineParser::parseLabel() {
1363   nextToken();
1364   unsigned OldLineLevel = Line->Level;
1365   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1366     --Line->Level;
1367   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1368     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1369     parseBlock(/*MustBeDeclaration=*/false);
1370     if (FormatTok->Tok.is(tok::kw_break)) {
1371       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1372       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1373           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1374         addUnwrappedLine();
1375       }
1376       parseStructuralElement();
1377     }
1378     addUnwrappedLine();
1379   } else {
1380     addUnwrappedLine();
1381   }
1382   Line->Level = OldLineLevel;
1383 }
1384 
1385 void UnwrappedLineParser::parseCaseLabel() {
1386   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1387   // FIXME: fix handling of complex expressions here.
1388   do {
1389     nextToken();
1390   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1391   parseLabel();
1392 }
1393 
1394 void UnwrappedLineParser::parseSwitch() {
1395   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1396   nextToken();
1397   if (FormatTok->Tok.is(tok::l_paren))
1398     parseParens();
1399   if (FormatTok->Tok.is(tok::l_brace)) {
1400     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1401     parseBlock(/*MustBeDeclaration=*/false);
1402     addUnwrappedLine();
1403   } else {
1404     addUnwrappedLine();
1405     ++Line->Level;
1406     parseStructuralElement();
1407     --Line->Level;
1408   }
1409 }
1410 
1411 void UnwrappedLineParser::parseAccessSpecifier() {
1412   nextToken();
1413   // Understand Qt's slots.
1414   if (FormatTok->is(tok::identifier) &&
1415       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1416     nextToken();
1417   // Otherwise, we don't know what it is, and we'd better keep the next token.
1418   if (FormatTok->Tok.is(tok::colon))
1419     nextToken();
1420   addUnwrappedLine();
1421 }
1422 
1423 void UnwrappedLineParser::parseEnum() {
1424   // Won't be 'enum' for NS_ENUMs.
1425   if (FormatTok->Tok.is(tok::kw_enum))
1426     nextToken();
1427 
1428   // Eat up enum class ...
1429   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1430     nextToken();
1431   while (FormatTok->Tok.getIdentifierInfo() ||
1432          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1433                             tok::greater, tok::comma, tok::question)) {
1434     nextToken();
1435     // We can have macros or attributes in between 'enum' and the enum name.
1436     if (FormatTok->is(tok::l_paren))
1437       parseParens();
1438     if (FormatTok->is(tok::identifier))
1439       nextToken();
1440   }
1441 
1442   // Just a declaration or something is wrong.
1443   if (FormatTok->isNot(tok::l_brace))
1444     return;
1445   FormatTok->BlockKind = BK_Block;
1446 
1447   if (Style.Language == FormatStyle::LK_Java) {
1448     // Java enums are different.
1449     parseJavaEnumBody();
1450     return;
1451   }
1452 
1453   // Parse enum body.
1454   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1455   if (HasError) {
1456     if (FormatTok->is(tok::semi))
1457       nextToken();
1458     addUnwrappedLine();
1459   }
1460 
1461   // We fall through to parsing a structural element afterwards, so that in
1462   // enum A {} n, m;
1463   // "} n, m;" will end up in one unwrapped line.
1464 }
1465 
1466 void UnwrappedLineParser::parseJavaEnumBody() {
1467   // Determine whether the enum is simple, i.e. does not have a semicolon or
1468   // constants with class bodies. Simple enums can be formatted like braced
1469   // lists, contracted to a single line, etc.
1470   unsigned StoredPosition = Tokens->getPosition();
1471   bool IsSimple = true;
1472   FormatToken *Tok = Tokens->getNextToken();
1473   while (Tok) {
1474     if (Tok->is(tok::r_brace))
1475       break;
1476     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1477       IsSimple = false;
1478       break;
1479     }
1480     // FIXME: This will also mark enums with braces in the arguments to enum
1481     // constants as "not simple". This is probably fine in practice, though.
1482     Tok = Tokens->getNextToken();
1483   }
1484   FormatTok = Tokens->setPosition(StoredPosition);
1485 
1486   if (IsSimple) {
1487     parseBracedList();
1488     addUnwrappedLine();
1489     return;
1490   }
1491 
1492   // Parse the body of a more complex enum.
1493   // First add a line for everything up to the "{".
1494   nextToken();
1495   addUnwrappedLine();
1496   ++Line->Level;
1497 
1498   // Parse the enum constants.
1499   while (FormatTok) {
1500     if (FormatTok->is(tok::l_brace)) {
1501       // Parse the constant's class body.
1502       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1503                  /*MunchSemi=*/false);
1504     } else if (FormatTok->is(tok::l_paren)) {
1505       parseParens();
1506     } else if (FormatTok->is(tok::comma)) {
1507       nextToken();
1508       addUnwrappedLine();
1509     } else if (FormatTok->is(tok::semi)) {
1510       nextToken();
1511       addUnwrappedLine();
1512       break;
1513     } else if (FormatTok->is(tok::r_brace)) {
1514       addUnwrappedLine();
1515       break;
1516     } else {
1517       nextToken();
1518     }
1519   }
1520 
1521   // Parse the class body after the enum's ";" if any.
1522   parseLevel(/*HasOpeningBrace=*/true);
1523   nextToken();
1524   --Line->Level;
1525   addUnwrappedLine();
1526 }
1527 
1528 void UnwrappedLineParser::parseRecord() {
1529   const FormatToken &InitialToken = *FormatTok;
1530   nextToken();
1531   if (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::kw___attribute,
1532                          tok::kw___declspec, tok::kw_alignas)) {
1533     nextToken();
1534     // We can have macros or attributes in between 'class' and the class name.
1535     if (FormatTok->Tok.is(tok::l_paren)) {
1536       parseParens();
1537     }
1538     // The actual identifier can be a nested name specifier, and in macros
1539     // it is often token-pasted.
1540     while (FormatTok->is(tok::identifier) || FormatTok->is(tok::coloncolon) ||
1541            FormatTok->is(tok::hashhash) ||
1542            (Style.Language == FormatStyle::LK_Java &&
1543             FormatTok->isOneOf(tok::period, tok::comma)))
1544       nextToken();
1545 
1546     // Note that parsing away template declarations here leads to incorrectly
1547     // accepting function declarations as record declarations.
1548     // In general, we cannot solve this problem. Consider:
1549     // class A<int> B() {}
1550     // which can be a function definition or a class definition when B() is a
1551     // macro. If we find enough real-world cases where this is a problem, we
1552     // can parse for the 'template' keyword in the beginning of the statement,
1553     // and thus rule out the record production in case there is no template
1554     // (this would still leave us with an ambiguity between template function
1555     // and class declarations).
1556     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1557       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1558         if (FormatTok->Tok.is(tok::semi))
1559           return;
1560         nextToken();
1561       }
1562     }
1563   }
1564   if (FormatTok->Tok.is(tok::l_brace)) {
1565     if (ShouldBreakBeforeBrace(Style, InitialToken))
1566       addUnwrappedLine();
1567 
1568     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1569                /*MunchSemi=*/false);
1570   }
1571   // We fall through to parsing a structural element afterwards, so
1572   // class A {} n, m;
1573   // will end up in one unwrapped line.
1574   // This does not apply for Java.
1575   if (Style.Language == FormatStyle::LK_Java ||
1576       Style.Language == FormatStyle::LK_JavaScript)
1577     addUnwrappedLine();
1578 }
1579 
1580 void UnwrappedLineParser::parseObjCProtocolList() {
1581   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1582   do
1583     nextToken();
1584   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1585   nextToken(); // Skip '>'.
1586 }
1587 
1588 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1589   do {
1590     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1591       nextToken();
1592       addUnwrappedLine();
1593       break;
1594     }
1595     if (FormatTok->is(tok::l_brace)) {
1596       parseBlock(/*MustBeDeclaration=*/false);
1597       // In ObjC interfaces, nothing should be following the "}".
1598       addUnwrappedLine();
1599     } else if (FormatTok->is(tok::r_brace)) {
1600       // Ignore stray "}". parseStructuralElement doesn't consume them.
1601       nextToken();
1602       addUnwrappedLine();
1603     } else {
1604       parseStructuralElement();
1605     }
1606   } while (!eof());
1607 }
1608 
1609 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1610   nextToken();
1611   nextToken(); // interface name
1612 
1613   // @interface can be followed by either a base class, or a category.
1614   if (FormatTok->Tok.is(tok::colon)) {
1615     nextToken();
1616     nextToken(); // base class name
1617   } else if (FormatTok->Tok.is(tok::l_paren))
1618     // Skip category, if present.
1619     parseParens();
1620 
1621   if (FormatTok->Tok.is(tok::less))
1622     parseObjCProtocolList();
1623 
1624   if (FormatTok->Tok.is(tok::l_brace)) {
1625     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1626         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1627       addUnwrappedLine();
1628     parseBlock(/*MustBeDeclaration=*/true);
1629   }
1630 
1631   // With instance variables, this puts '}' on its own line.  Without instance
1632   // variables, this ends the @interface line.
1633   addUnwrappedLine();
1634 
1635   parseObjCUntilAtEnd();
1636 }
1637 
1638 void UnwrappedLineParser::parseObjCProtocol() {
1639   nextToken();
1640   nextToken(); // protocol name
1641 
1642   if (FormatTok->Tok.is(tok::less))
1643     parseObjCProtocolList();
1644 
1645   // Check for protocol declaration.
1646   if (FormatTok->Tok.is(tok::semi)) {
1647     nextToken();
1648     return addUnwrappedLine();
1649   }
1650 
1651   addUnwrappedLine();
1652   parseObjCUntilAtEnd();
1653 }
1654 
1655 void UnwrappedLineParser::parseJavaScriptEs6ImportExport() {
1656   assert(FormatTok->isOneOf(Keywords.kw_import, tok::kw_export));
1657   nextToken();
1658 
1659   if (FormatTok->isOneOf(tok::kw_const, tok::kw_class, Keywords.kw_function,
1660                          Keywords.kw_var))
1661     return; // Fall through to parsing the corresponding structure.
1662 
1663   if (FormatTok->is(tok::kw_default)) {
1664     nextToken(); // export default ..., fall through after eating 'default'.
1665     return;
1666   }
1667 
1668   if (FormatTok->is(tok::l_brace)) {
1669     FormatTok->BlockKind = BK_Block;
1670     parseBracedList();
1671   }
1672 
1673   while (!eof() && FormatTok->isNot(tok::semi) &&
1674          FormatTok->isNot(tok::l_brace)) {
1675     nextToken();
1676   }
1677 }
1678 
1679 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1680                                                  StringRef Prefix = "") {
1681   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1682                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1683   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1684                                                     E = Line.Tokens.end();
1685        I != E; ++I) {
1686     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1687   }
1688   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1689                                                     E = Line.Tokens.end();
1690        I != E; ++I) {
1691     const UnwrappedLineNode &Node = *I;
1692     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1693              I = Node.Children.begin(),
1694              E = Node.Children.end();
1695          I != E; ++I) {
1696       printDebugInfo(*I, "\nChild: ");
1697     }
1698   }
1699   llvm::dbgs() << "\n";
1700 }
1701 
1702 void UnwrappedLineParser::addUnwrappedLine() {
1703   if (Line->Tokens.empty())
1704     return;
1705   DEBUG({
1706     if (CurrentLines == &Lines)
1707       printDebugInfo(*Line);
1708   });
1709   CurrentLines->push_back(*Line);
1710   Line->Tokens.clear();
1711   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1712     for (SmallVectorImpl<UnwrappedLine>::iterator
1713              I = PreprocessorDirectives.begin(),
1714              E = PreprocessorDirectives.end();
1715          I != E; ++I) {
1716       CurrentLines->push_back(*I);
1717     }
1718     PreprocessorDirectives.clear();
1719   }
1720 }
1721 
1722 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1723 
1724 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
1725   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1726          FormatTok.NewlinesBefore > 0;
1727 }
1728 
1729 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1730   bool JustComments = Line->Tokens.empty();
1731   for (SmallVectorImpl<FormatToken *>::const_iterator
1732            I = CommentsBeforeNextToken.begin(),
1733            E = CommentsBeforeNextToken.end();
1734        I != E; ++I) {
1735     if (isOnNewLine(**I) && JustComments) {
1736       addUnwrappedLine();
1737     }
1738     pushToken(*I);
1739   }
1740   if (NewlineBeforeNext && JustComments) {
1741     addUnwrappedLine();
1742   }
1743   CommentsBeforeNextToken.clear();
1744 }
1745 
1746 void UnwrappedLineParser::nextToken() {
1747   if (eof())
1748     return;
1749   flushComments(isOnNewLine(*FormatTok));
1750   pushToken(FormatTok);
1751   readToken();
1752 }
1753 
1754 void UnwrappedLineParser::readToken() {
1755   bool CommentsInCurrentLine = true;
1756   do {
1757     FormatTok = Tokens->getNextToken();
1758     assert(FormatTok);
1759     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1760            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1761       // If there is an unfinished unwrapped line, we flush the preprocessor
1762       // directives only after that unwrapped line was finished later.
1763       bool SwitchToPreprocessorLines = !Line->Tokens.empty();
1764       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1765       // Comments stored before the preprocessor directive need to be output
1766       // before the preprocessor directive, at the same level as the
1767       // preprocessor directive, as we consider them to apply to the directive.
1768       flushComments(isOnNewLine(*FormatTok));
1769       parsePPDirective();
1770     }
1771     while (FormatTok->Type == TT_ConflictStart ||
1772            FormatTok->Type == TT_ConflictEnd ||
1773            FormatTok->Type == TT_ConflictAlternative) {
1774       if (FormatTok->Type == TT_ConflictStart) {
1775         conditionalCompilationStart(/*Unreachable=*/false);
1776       } else if (FormatTok->Type == TT_ConflictAlternative) {
1777         conditionalCompilationAlternative();
1778       } else if (FormatTok->Type == TT_ConflictEnd) {
1779         conditionalCompilationEnd();
1780       }
1781       FormatTok = Tokens->getNextToken();
1782       FormatTok->MustBreakBefore = true;
1783     }
1784 
1785     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1786         !Line->InPPDirective) {
1787       continue;
1788     }
1789 
1790     if (!FormatTok->Tok.is(tok::comment))
1791       return;
1792     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
1793       CommentsInCurrentLine = false;
1794     }
1795     if (CommentsInCurrentLine) {
1796       pushToken(FormatTok);
1797     } else {
1798       CommentsBeforeNextToken.push_back(FormatTok);
1799     }
1800   } while (!eof());
1801 }
1802 
1803 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1804   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1805   if (MustBreakBeforeNextToken) {
1806     Line->Tokens.back().Tok->MustBreakBefore = true;
1807     MustBreakBeforeNextToken = false;
1808   }
1809 }
1810 
1811 } // end namespace format
1812 } // end namespace clang
1813