1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "format-parser"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(nullptr) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
126     if (SwitchToPreprocessorLines)
127       Parser.CurrentLines = &Parser.PreprocessorDirectives;
128     else if (!Parser.Line->Tokens.empty())
129       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
130     PreBlockLine = std::move(Parser.Line);
131     Parser.Line = llvm::make_unique<UnwrappedLine>();
132     Parser.Line->Level = PreBlockLine->Level;
133     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
134   }
135 
136   ~ScopedLineState() {
137     if (!Parser.Line->Tokens.empty()) {
138       Parser.addUnwrappedLine();
139     }
140     assert(Parser.Line->Tokens.empty());
141     Parser.Line = std::move(PreBlockLine);
142     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
143       Parser.MustBreakBeforeNextToken = true;
144     Parser.CurrentLines = OriginalLines;
145   }
146 
147 private:
148   UnwrappedLineParser &Parser;
149 
150   std::unique_ptr<UnwrappedLine> PreBlockLine;
151   SmallVectorImpl<UnwrappedLine> *OriginalLines;
152 };
153 
154 class CompoundStatementIndenter {
155 public:
156   CompoundStatementIndenter(UnwrappedLineParser *Parser,
157                             const FormatStyle &Style, unsigned &LineLevel)
158       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
159     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
160       Parser->addUnwrappedLine();
161     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
162       Parser->addUnwrappedLine();
163       ++LineLevel;
164     }
165   }
166   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
167 
168 private:
169   unsigned &LineLevel;
170   unsigned OldLineLevel;
171 };
172 
173 namespace {
174 
175 class IndexedTokenSource : public FormatTokenSource {
176 public:
177   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
178       : Tokens(Tokens), Position(-1) {}
179 
180   FormatToken *getNextToken() override {
181     ++Position;
182     return Tokens[Position];
183   }
184 
185   unsigned getPosition() override {
186     assert(Position >= 0);
187     return Position;
188   }
189 
190   FormatToken *setPosition(unsigned P) override {
191     Position = P;
192     return Tokens[Position];
193   }
194 
195   void reset() { Position = -1; }
196 
197 private:
198   ArrayRef<FormatToken *> Tokens;
199   int Position;
200 };
201 
202 } // end anonymous namespace
203 
204 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
205                                          const AdditionalKeywords &Keywords,
206                                          ArrayRef<FormatToken *> Tokens,
207                                          UnwrappedLineConsumer &Callback)
208     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
209       CurrentLines(&Lines), StructuralError(false), Style(Style),
210       Keywords(Keywords), Tokens(nullptr), Callback(Callback),
211       AllTokens(Tokens), PPBranchLevel(-1) {}
212 
213 void UnwrappedLineParser::reset() {
214   PPBranchLevel = -1;
215   Line.reset(new UnwrappedLine);
216   CommentsBeforeNextToken.clear();
217   FormatTok = nullptr;
218   MustBreakBeforeNextToken = false;
219   PreprocessorDirectives.clear();
220   CurrentLines = &Lines;
221   DeclarationScopeStack.clear();
222   StructuralError = false;
223   PPStack.clear();
224 }
225 
226 bool UnwrappedLineParser::parse() {
227   IndexedTokenSource TokenSource(AllTokens);
228   do {
229     DEBUG(llvm::dbgs() << "----\n");
230     reset();
231     Tokens = &TokenSource;
232     TokenSource.reset();
233 
234     readToken();
235     parseFile();
236     // Create line with eof token.
237     pushToken(FormatTok);
238     addUnwrappedLine();
239 
240     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
241                                                   E = Lines.end();
242          I != E; ++I) {
243       Callback.consumeUnwrappedLine(*I);
244     }
245     Callback.finishRun();
246     Lines.clear();
247     while (!PPLevelBranchIndex.empty() &&
248            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
249       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
250       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
251     }
252     if (!PPLevelBranchIndex.empty()) {
253       ++PPLevelBranchIndex.back();
254       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
255       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
256     }
257   } while (!PPLevelBranchIndex.empty());
258 
259   return StructuralError;
260 }
261 
262 void UnwrappedLineParser::parseFile() {
263   ScopedDeclarationState DeclarationState(
264       *Line, DeclarationScopeStack,
265       /*MustBeDeclaration=*/ !Line->InPPDirective);
266   parseLevel(/*HasOpeningBrace=*/false);
267   // Make sure to format the remaining tokens.
268   flushComments(true);
269   addUnwrappedLine();
270 }
271 
272 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
273   bool SwitchLabelEncountered = false;
274   do {
275     switch (FormatTok->Tok.getKind()) {
276     case tok::comment:
277       nextToken();
278       addUnwrappedLine();
279       break;
280     case tok::l_brace:
281       // FIXME: Add parameter whether this can happen - if this happens, we must
282       // be in a non-declaration context.
283       parseBlock(/*MustBeDeclaration=*/false);
284       addUnwrappedLine();
285       break;
286     case tok::r_brace:
287       if (HasOpeningBrace)
288         return;
289       StructuralError = true;
290       nextToken();
291       addUnwrappedLine();
292       break;
293     case tok::kw_default:
294     case tok::kw_case:
295       if (!SwitchLabelEncountered &&
296           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
297         ++Line->Level;
298       SwitchLabelEncountered = true;
299       parseStructuralElement();
300       break;
301     default:
302       parseStructuralElement();
303       break;
304     }
305   } while (!eof());
306 }
307 
308 void UnwrappedLineParser::calculateBraceTypes() {
309   // We'll parse forward through the tokens until we hit
310   // a closing brace or eof - note that getNextToken() will
311   // parse macros, so this will magically work inside macro
312   // definitions, too.
313   unsigned StoredPosition = Tokens->getPosition();
314   FormatToken *Tok = FormatTok;
315   // Keep a stack of positions of lbrace tokens. We will
316   // update information about whether an lbrace starts a
317   // braced init list or a different block during the loop.
318   SmallVector<FormatToken *, 8> LBraceStack;
319   assert(Tok->Tok.is(tok::l_brace));
320   do {
321     // Get next none-comment token.
322     FormatToken *NextTok;
323     unsigned ReadTokens = 0;
324     do {
325       NextTok = Tokens->getNextToken();
326       ++ReadTokens;
327     } while (NextTok->is(tok::comment));
328 
329     switch (Tok->Tok.getKind()) {
330     case tok::l_brace:
331       LBraceStack.push_back(Tok);
332       break;
333     case tok::r_brace:
334       if (!LBraceStack.empty()) {
335         if (LBraceStack.back()->BlockKind == BK_Unknown) {
336           bool ProbablyBracedList = false;
337           if (Style.Language == FormatStyle::LK_Proto) {
338             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
339           } else {
340             // Using OriginalColumn to distinguish between ObjC methods and
341             // binary operators is a bit hacky.
342             bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
343                                     NextTok->OriginalColumn == 0;
344 
345             // If there is a comma, semicolon or right paren after the closing
346             // brace, we assume this is a braced initializer list.  Note that
347             // regardless how we mark inner braces here, we will overwrite the
348             // BlockKind later if we parse a braced list (where all blocks
349             // inside are by default braced lists), or when we explicitly detect
350             // blocks (for example while parsing lambdas).
351             //
352             // We exclude + and - as they can be ObjC visibility modifiers.
353             ProbablyBracedList =
354                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
355                                  tok::r_paren, tok::r_square, tok::l_brace,
356                                  tok::l_paren, tok::ellipsis) ||
357                 (NextTok->isBinaryOperator() && !NextIsObjCMethod);
358           }
359           if (ProbablyBracedList) {
360             Tok->BlockKind = BK_BracedInit;
361             LBraceStack.back()->BlockKind = BK_BracedInit;
362           } else {
363             Tok->BlockKind = BK_Block;
364             LBraceStack.back()->BlockKind = BK_Block;
365           }
366         }
367         LBraceStack.pop_back();
368       }
369       break;
370     case tok::at:
371     case tok::semi:
372     case tok::kw_if:
373     case tok::kw_while:
374     case tok::kw_for:
375     case tok::kw_switch:
376     case tok::kw_try:
377       if (!LBraceStack.empty())
378         LBraceStack.back()->BlockKind = BK_Block;
379       break;
380     default:
381       break;
382     }
383     Tok = NextTok;
384   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
385   // Assume other blocks for all unclosed opening braces.
386   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
387     if (LBraceStack[i]->BlockKind == BK_Unknown)
388       LBraceStack[i]->BlockKind = BK_Block;
389   }
390 
391   FormatTok = Tokens->setPosition(StoredPosition);
392 }
393 
394 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
395                                      bool MunchSemi) {
396   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
397   unsigned InitialLevel = Line->Level;
398   nextToken();
399 
400   addUnwrappedLine();
401 
402   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
403                                           MustBeDeclaration);
404   if (AddLevel)
405     ++Line->Level;
406   parseLevel(/*HasOpeningBrace=*/true);
407 
408   if (!FormatTok->Tok.is(tok::r_brace)) {
409     Line->Level = InitialLevel;
410     StructuralError = true;
411     return;
412   }
413 
414   nextToken(); // Munch the closing brace.
415   if (MunchSemi && FormatTok->Tok.is(tok::semi))
416     nextToken();
417   Line->Level = InitialLevel;
418 }
419 
420 static bool IsGoogScope(const UnwrappedLine &Line) {
421   // FIXME: Closure-library specific stuff should not be hard-coded but be
422   // configurable.
423   if (Line.Tokens.size() < 4)
424     return false;
425   auto I = Line.Tokens.begin();
426   if (I->Tok->TokenText != "goog")
427     return false;
428   ++I;
429   if (I->Tok->isNot(tok::period))
430     return false;
431   ++I;
432   if (I->Tok->TokenText != "scope")
433     return false;
434   ++I;
435   return I->Tok->is(tok::l_paren);
436 }
437 
438 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
439                                    const FormatToken &InitialToken) {
440   switch (Style.BreakBeforeBraces) {
441   case FormatStyle::BS_Linux:
442     return InitialToken.isOneOf(tok::kw_namespace, tok::kw_class);
443   case FormatStyle::BS_Allman:
444   case FormatStyle::BS_GNU:
445     return true;
446   default:
447     return false;
448   }
449 }
450 
451 void UnwrappedLineParser::parseChildBlock() {
452   FormatTok->BlockKind = BK_Block;
453   nextToken();
454   {
455     bool GoogScope =
456         Style.Language == FormatStyle::LK_JavaScript && IsGoogScope(*Line);
457     ScopedLineState LineState(*this);
458     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
459                                             /*MustBeDeclaration=*/false);
460     Line->Level += GoogScope ? 0 : 1;
461     parseLevel(/*HasOpeningBrace=*/true);
462     Line->Level -= GoogScope ? 0 : 1;
463   }
464   nextToken();
465 }
466 
467 void UnwrappedLineParser::parsePPDirective() {
468   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
469   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
470   nextToken();
471 
472   if (!FormatTok->Tok.getIdentifierInfo()) {
473     parsePPUnknown();
474     return;
475   }
476 
477   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
478   case tok::pp_define:
479     parsePPDefine();
480     return;
481   case tok::pp_if:
482     parsePPIf(/*IfDef=*/false);
483     break;
484   case tok::pp_ifdef:
485   case tok::pp_ifndef:
486     parsePPIf(/*IfDef=*/true);
487     break;
488   case tok::pp_else:
489     parsePPElse();
490     break;
491   case tok::pp_elif:
492     parsePPElIf();
493     break;
494   case tok::pp_endif:
495     parsePPEndIf();
496     break;
497   default:
498     parsePPUnknown();
499     break;
500   }
501 }
502 
503 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
504   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
505     PPStack.push_back(PP_Unreachable);
506   else
507     PPStack.push_back(PP_Conditional);
508 }
509 
510 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
511   ++PPBranchLevel;
512   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
513   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
514     PPLevelBranchIndex.push_back(0);
515     PPLevelBranchCount.push_back(0);
516   }
517   PPChainBranchIndex.push(0);
518   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
519   conditionalCompilationCondition(Unreachable || Skip);
520 }
521 
522 void UnwrappedLineParser::conditionalCompilationAlternative() {
523   if (!PPStack.empty())
524     PPStack.pop_back();
525   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
526   if (!PPChainBranchIndex.empty())
527     ++PPChainBranchIndex.top();
528   conditionalCompilationCondition(
529       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
530       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
531 }
532 
533 void UnwrappedLineParser::conditionalCompilationEnd() {
534   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
535   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
536     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
537       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
538     }
539   }
540   // Guard against #endif's without #if.
541   if (PPBranchLevel > 0)
542     --PPBranchLevel;
543   if (!PPChainBranchIndex.empty())
544     PPChainBranchIndex.pop();
545   if (!PPStack.empty())
546     PPStack.pop_back();
547 }
548 
549 void UnwrappedLineParser::parsePPIf(bool IfDef) {
550   nextToken();
551   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
552                          FormatTok->Tok.getLiteralData() != nullptr &&
553                          StringRef(FormatTok->Tok.getLiteralData(),
554                                    FormatTok->Tok.getLength()) == "0") ||
555                         FormatTok->Tok.is(tok::kw_false);
556   conditionalCompilationStart(!IfDef && IsLiteralFalse);
557   parsePPUnknown();
558 }
559 
560 void UnwrappedLineParser::parsePPElse() {
561   conditionalCompilationAlternative();
562   parsePPUnknown();
563 }
564 
565 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
566 
567 void UnwrappedLineParser::parsePPEndIf() {
568   conditionalCompilationEnd();
569   parsePPUnknown();
570 }
571 
572 void UnwrappedLineParser::parsePPDefine() {
573   nextToken();
574 
575   if (FormatTok->Tok.getKind() != tok::identifier) {
576     parsePPUnknown();
577     return;
578   }
579   nextToken();
580   if (FormatTok->Tok.getKind() == tok::l_paren &&
581       FormatTok->WhitespaceRange.getBegin() ==
582           FormatTok->WhitespaceRange.getEnd()) {
583     parseParens();
584   }
585   addUnwrappedLine();
586   Line->Level = 1;
587 
588   // Errors during a preprocessor directive can only affect the layout of the
589   // preprocessor directive, and thus we ignore them. An alternative approach
590   // would be to use the same approach we use on the file level (no
591   // re-indentation if there was a structural error) within the macro
592   // definition.
593   parseFile();
594 }
595 
596 void UnwrappedLineParser::parsePPUnknown() {
597   do {
598     nextToken();
599   } while (!eof());
600   addUnwrappedLine();
601 }
602 
603 // Here we blacklist certain tokens that are not usually the first token in an
604 // unwrapped line. This is used in attempt to distinguish macro calls without
605 // trailing semicolons from other constructs split to several lines.
606 bool tokenCanStartNewLine(clang::Token Tok) {
607   // Semicolon can be a null-statement, l_square can be a start of a macro or
608   // a C++11 attribute, but this doesn't seem to be common.
609   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
610          Tok.isNot(tok::l_square) &&
611          // Tokens that can only be used as binary operators and a part of
612          // overloaded operator names.
613          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
614          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
615          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
616          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
617          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
618          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
619          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
620          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
621          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
622          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
623          Tok.isNot(tok::lesslessequal) &&
624          // Colon is used in labels, base class lists, initializer lists,
625          // range-based for loops, ternary operator, but should never be the
626          // first token in an unwrapped line.
627          Tok.isNot(tok::colon) &&
628          // 'noexcept' is a trailing annotation.
629          Tok.isNot(tok::kw_noexcept);
630 }
631 
632 void UnwrappedLineParser::parseStructuralElement() {
633   assert(!FormatTok->Tok.is(tok::l_brace));
634   switch (FormatTok->Tok.getKind()) {
635   case tok::at:
636     nextToken();
637     if (FormatTok->Tok.is(tok::l_brace)) {
638       parseBracedList();
639       break;
640     }
641     switch (FormatTok->Tok.getObjCKeywordID()) {
642     case tok::objc_public:
643     case tok::objc_protected:
644     case tok::objc_package:
645     case tok::objc_private:
646       return parseAccessSpecifier();
647     case tok::objc_interface:
648     case tok::objc_implementation:
649       return parseObjCInterfaceOrImplementation();
650     case tok::objc_protocol:
651       return parseObjCProtocol();
652     case tok::objc_end:
653       return; // Handled by the caller.
654     case tok::objc_optional:
655     case tok::objc_required:
656       nextToken();
657       addUnwrappedLine();
658       return;
659     default:
660       break;
661     }
662     break;
663   case tok::kw_asm:
664     nextToken();
665     if (FormatTok->is(tok::l_brace)) {
666       nextToken();
667       while (FormatTok && FormatTok->isNot(tok::eof)) {
668         if (FormatTok->is(tok::r_brace)) {
669           nextToken();
670           break;
671         }
672         FormatTok->Finalized = true;
673         nextToken();
674       }
675     }
676     break;
677   case tok::kw_namespace:
678     parseNamespace();
679     return;
680   case tok::kw_inline:
681     nextToken();
682     if (FormatTok->Tok.is(tok::kw_namespace)) {
683       parseNamespace();
684       return;
685     }
686     break;
687   case tok::kw_public:
688   case tok::kw_protected:
689   case tok::kw_private:
690     if (Style.Language == FormatStyle::LK_Java)
691       nextToken();
692     else
693       parseAccessSpecifier();
694     return;
695   case tok::kw_if:
696     parseIfThenElse();
697     return;
698   case tok::kw_for:
699   case tok::kw_while:
700     parseForOrWhileLoop();
701     return;
702   case tok::kw_do:
703     parseDoWhile();
704     return;
705   case tok::kw_switch:
706     parseSwitch();
707     return;
708   case tok::kw_default:
709     nextToken();
710     parseLabel();
711     return;
712   case tok::kw_case:
713     parseCaseLabel();
714     return;
715   case tok::kw_try:
716     parseTryCatch();
717     return;
718   case tok::kw_extern:
719     nextToken();
720     if (FormatTok->Tok.is(tok::string_literal)) {
721       nextToken();
722       if (FormatTok->Tok.is(tok::l_brace)) {
723         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
724         addUnwrappedLine();
725         return;
726       }
727     }
728     break;
729   case tok::identifier:
730     if (FormatTok->IsForEachMacro) {
731       parseForOrWhileLoop();
732       return;
733     }
734     // In all other cases, parse the declaration.
735     break;
736   default:
737     break;
738   }
739   do {
740     switch (FormatTok->Tok.getKind()) {
741     case tok::at:
742       nextToken();
743       if (FormatTok->Tok.is(tok::l_brace))
744         parseBracedList();
745       break;
746     case tok::kw_enum:
747       parseEnum();
748       break;
749     case tok::kw_typedef:
750       nextToken();
751       if (FormatTok->isOneOf(Keywords.kw_NS_ENUM, Keywords.kw_NS_OPTIONS,
752                              Keywords.kw_CF_ENUM, Keywords.kw_CF_OPTIONS))
753         parseEnum();
754       break;
755     case tok::kw_struct:
756     case tok::kw_union:
757     case tok::kw_class:
758       parseRecord();
759       // A record declaration or definition is always the start of a structural
760       // element.
761       break;
762     case tok::period:
763       nextToken();
764       // In Java, classes have an implicit static member "class".
765       if (Style.Language == FormatStyle::LK_Java && FormatTok &&
766           FormatTok->is(tok::kw_class))
767         nextToken();
768       break;
769     case tok::semi:
770       nextToken();
771       addUnwrappedLine();
772       return;
773     case tok::r_brace:
774       addUnwrappedLine();
775       return;
776     case tok::l_paren:
777       parseParens();
778       break;
779     case tok::caret:
780       nextToken();
781       if (FormatTok->Tok.isAnyIdentifier() ||
782           FormatTok->isSimpleTypeSpecifier())
783         nextToken();
784       if (FormatTok->is(tok::l_paren))
785         parseParens();
786       if (FormatTok->is(tok::l_brace))
787         parseChildBlock();
788       break;
789     case tok::l_brace:
790       if (!tryToParseBracedList()) {
791         // A block outside of parentheses must be the last part of a
792         // structural element.
793         // FIXME: Figure out cases where this is not true, and add projections
794         // for them (the one we know is missing are lambdas).
795         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
796           addUnwrappedLine();
797         FormatTok->Type = TT_FunctionLBrace;
798         parseBlock(/*MustBeDeclaration=*/false);
799         addUnwrappedLine();
800         return;
801       }
802       // Otherwise this was a braced init list, and the structural
803       // element continues.
804       break;
805     case tok::kw_try:
806       // We arrive here when parsing function-try blocks.
807       parseTryCatch();
808       return;
809     case tok::identifier: {
810       StringRef Text = FormatTok->TokenText;
811       // Parse function literal unless 'function' is the first token in a line
812       // in which case this should be treated as a free-standing function.
813       if (Style.Language == FormatStyle::LK_JavaScript && Text == "function" &&
814           Line->Tokens.size() > 0) {
815         tryToParseJSFunction();
816         break;
817       }
818       nextToken();
819       if (Line->Tokens.size() == 1) {
820         if (FormatTok->Tok.is(tok::colon)) {
821           parseLabel();
822           return;
823         }
824         // Recognize function-like macro usages without trailing semicolon as
825         // well as free-standing macrose like Q_OBJECT.
826         bool FunctionLike = FormatTok->is(tok::l_paren);
827         if (FunctionLike)
828           parseParens();
829         if (FormatTok->NewlinesBefore > 0 &&
830             (Text.size() >= 5 || FunctionLike) &&
831             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
832           addUnwrappedLine();
833           return;
834         }
835       }
836       break;
837     }
838     case tok::equal:
839       nextToken();
840       if (FormatTok->Tok.is(tok::l_brace)) {
841         parseBracedList();
842       }
843       break;
844     case tok::l_square:
845       parseSquare();
846       break;
847     default:
848       nextToken();
849       break;
850     }
851   } while (!eof());
852 }
853 
854 bool UnwrappedLineParser::tryToParseLambda() {
855   // FIXME: This is a dirty way to access the previous token. Find a better
856   // solution.
857   if (!Line->Tokens.empty() &&
858       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator,
859                                         tok::kw_new, tok::kw_delete) ||
860        Line->Tokens.back().Tok->closesScope() ||
861        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
862     nextToken();
863     return false;
864   }
865   assert(FormatTok->is(tok::l_square));
866   FormatToken &LSquare = *FormatTok;
867   if (!tryToParseLambdaIntroducer())
868     return false;
869 
870   while (FormatTok->isNot(tok::l_brace)) {
871     if (FormatTok->isSimpleTypeSpecifier()) {
872       nextToken();
873       continue;
874     }
875     switch (FormatTok->Tok.getKind()) {
876     case tok::l_brace:
877       break;
878     case tok::l_paren:
879       parseParens();
880       break;
881     case tok::amp:
882     case tok::star:
883     case tok::kw_const:
884     case tok::comma:
885     case tok::less:
886     case tok::greater:
887     case tok::identifier:
888     case tok::coloncolon:
889     case tok::kw_mutable:
890       nextToken();
891       break;
892     case tok::arrow:
893       FormatTok->Type = TT_TrailingReturnArrow;
894       nextToken();
895       break;
896     default:
897       return true;
898     }
899   }
900   LSquare.Type = TT_LambdaLSquare;
901   parseChildBlock();
902   return true;
903 }
904 
905 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
906   nextToken();
907   if (FormatTok->is(tok::equal)) {
908     nextToken();
909     if (FormatTok->is(tok::r_square)) {
910       nextToken();
911       return true;
912     }
913     if (FormatTok->isNot(tok::comma))
914       return false;
915     nextToken();
916   } else if (FormatTok->is(tok::amp)) {
917     nextToken();
918     if (FormatTok->is(tok::r_square)) {
919       nextToken();
920       return true;
921     }
922     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
923       return false;
924     }
925     if (FormatTok->is(tok::comma))
926       nextToken();
927   } else if (FormatTok->is(tok::r_square)) {
928     nextToken();
929     return true;
930   }
931   do {
932     if (FormatTok->is(tok::amp))
933       nextToken();
934     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
935       return false;
936     nextToken();
937     if (FormatTok->is(tok::ellipsis))
938       nextToken();
939     if (FormatTok->is(tok::comma)) {
940       nextToken();
941     } else if (FormatTok->is(tok::r_square)) {
942       nextToken();
943       return true;
944     } else {
945       return false;
946     }
947   } while (!eof());
948   return false;
949 }
950 
951 void UnwrappedLineParser::tryToParseJSFunction() {
952   nextToken();
953 
954   // Consume function name.
955   if (FormatTok->is(tok::identifier))
956       nextToken();
957 
958   if (FormatTok->isNot(tok::l_paren))
959     return;
960   nextToken();
961   while (FormatTok->isNot(tok::l_brace)) {
962     // Err on the side of caution in order to avoid consuming the full file in
963     // case of incomplete code.
964     if (!FormatTok->isOneOf(tok::identifier, tok::comma, tok::r_paren,
965                             tok::comment))
966       return;
967     nextToken();
968   }
969   parseChildBlock();
970 }
971 
972 bool UnwrappedLineParser::tryToParseBracedList() {
973   if (FormatTok->BlockKind == BK_Unknown)
974     calculateBraceTypes();
975   assert(FormatTok->BlockKind != BK_Unknown);
976   if (FormatTok->BlockKind == BK_Block)
977     return false;
978   parseBracedList();
979   return true;
980 }
981 
982 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
983   bool HasError = false;
984   nextToken();
985 
986   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
987   // replace this by using parseAssigmentExpression() inside.
988   do {
989     if (Style.Language == FormatStyle::LK_JavaScript &&
990         FormatTok->is(Keywords.kw_function)) {
991       tryToParseJSFunction();
992       continue;
993     }
994     switch (FormatTok->Tok.getKind()) {
995     case tok::caret:
996       nextToken();
997       if (FormatTok->is(tok::l_brace)) {
998         parseChildBlock();
999       }
1000       break;
1001     case tok::l_square:
1002       tryToParseLambda();
1003       break;
1004     case tok::l_brace:
1005       // Assume there are no blocks inside a braced init list apart
1006       // from the ones we explicitly parse out (like lambdas).
1007       FormatTok->BlockKind = BK_BracedInit;
1008       parseBracedList();
1009       break;
1010     case tok::r_brace:
1011       nextToken();
1012       return !HasError;
1013     case tok::semi:
1014       HasError = true;
1015       if (!ContinueOnSemicolons)
1016         return !HasError;
1017       nextToken();
1018       break;
1019     case tok::comma:
1020       nextToken();
1021       break;
1022     default:
1023       nextToken();
1024       break;
1025     }
1026   } while (!eof());
1027   return false;
1028 }
1029 
1030 void UnwrappedLineParser::parseParens() {
1031   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1032   nextToken();
1033   do {
1034     switch (FormatTok->Tok.getKind()) {
1035     case tok::l_paren:
1036       parseParens();
1037       if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_brace))
1038         parseChildBlock();
1039       break;
1040     case tok::r_paren:
1041       nextToken();
1042       return;
1043     case tok::r_brace:
1044       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1045       return;
1046     case tok::l_square:
1047       tryToParseLambda();
1048       break;
1049     case tok::l_brace:
1050       if (!tryToParseBracedList()) {
1051         parseChildBlock();
1052       }
1053       break;
1054     case tok::at:
1055       nextToken();
1056       if (FormatTok->Tok.is(tok::l_brace))
1057         parseBracedList();
1058       break;
1059     case tok::identifier:
1060       if (Style.Language == FormatStyle::LK_JavaScript &&
1061           FormatTok->is(Keywords.kw_function))
1062         tryToParseJSFunction();
1063       else
1064         nextToken();
1065       break;
1066     default:
1067       nextToken();
1068       break;
1069     }
1070   } while (!eof());
1071 }
1072 
1073 void UnwrappedLineParser::parseSquare() {
1074   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1075   if (tryToParseLambda())
1076     return;
1077   do {
1078     switch (FormatTok->Tok.getKind()) {
1079     case tok::l_paren:
1080       parseParens();
1081       break;
1082     case tok::r_square:
1083       nextToken();
1084       return;
1085     case tok::r_brace:
1086       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1087       return;
1088     case tok::l_square:
1089       parseSquare();
1090       break;
1091     case tok::l_brace: {
1092       if (!tryToParseBracedList()) {
1093         parseChildBlock();
1094       }
1095       break;
1096     }
1097     case tok::at:
1098       nextToken();
1099       if (FormatTok->Tok.is(tok::l_brace))
1100         parseBracedList();
1101       break;
1102     default:
1103       nextToken();
1104       break;
1105     }
1106   } while (!eof());
1107 }
1108 
1109 void UnwrappedLineParser::parseIfThenElse() {
1110   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1111   nextToken();
1112   if (FormatTok->Tok.is(tok::l_paren))
1113     parseParens();
1114   bool NeedsUnwrappedLine = false;
1115   if (FormatTok->Tok.is(tok::l_brace)) {
1116     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1117     parseBlock(/*MustBeDeclaration=*/false);
1118     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1119         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1120       addUnwrappedLine();
1121     } else {
1122       NeedsUnwrappedLine = true;
1123     }
1124   } else {
1125     addUnwrappedLine();
1126     ++Line->Level;
1127     parseStructuralElement();
1128     --Line->Level;
1129   }
1130   if (FormatTok->Tok.is(tok::kw_else)) {
1131     if (Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup)
1132       addUnwrappedLine();
1133     nextToken();
1134     if (FormatTok->Tok.is(tok::l_brace)) {
1135       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1136       parseBlock(/*MustBeDeclaration=*/false);
1137       addUnwrappedLine();
1138     } else if (FormatTok->Tok.is(tok::kw_if)) {
1139       parseIfThenElse();
1140     } else {
1141       addUnwrappedLine();
1142       ++Line->Level;
1143       parseStructuralElement();
1144       --Line->Level;
1145     }
1146   } else if (NeedsUnwrappedLine) {
1147     addUnwrappedLine();
1148   }
1149 }
1150 
1151 void UnwrappedLineParser::parseTryCatch() {
1152   assert(FormatTok->is(tok::kw_try) && "'try' expected");
1153   nextToken();
1154   bool NeedsUnwrappedLine = false;
1155   if (FormatTok->is(tok::colon)) {
1156     // We are in a function try block, what comes is an initializer list.
1157     nextToken();
1158     while (FormatTok->is(tok::identifier)) {
1159       nextToken();
1160       if (FormatTok->is(tok::l_paren))
1161         parseParens();
1162       else
1163         StructuralError = true;
1164       if (FormatTok->is(tok::comma))
1165         nextToken();
1166     }
1167   }
1168   // Parse try with resource.
1169   if (Style.Language == FormatStyle::LK_Java && FormatTok->is(tok::l_paren)) {
1170     parseParens();
1171   }
1172   if (FormatTok->is(tok::l_brace)) {
1173     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1174     parseBlock(/*MustBeDeclaration=*/false);
1175     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1176         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1177         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1178       addUnwrappedLine();
1179     } else {
1180       NeedsUnwrappedLine = true;
1181     }
1182   } else if (!FormatTok->is(tok::kw_catch)) {
1183     // The C++ standard requires a compound-statement after a try.
1184     // If there's none, we try to assume there's a structuralElement
1185     // and try to continue.
1186     StructuralError = true;
1187     addUnwrappedLine();
1188     ++Line->Level;
1189     parseStructuralElement();
1190     --Line->Level;
1191   }
1192   while (FormatTok->is(tok::kw_catch) ||
1193          ((Style.Language == FormatStyle::LK_Java ||
1194            Style.Language == FormatStyle::LK_JavaScript) &&
1195           FormatTok->is(Keywords.kw_finally))) {
1196     nextToken();
1197     while (FormatTok->isNot(tok::l_brace)) {
1198       if (FormatTok->is(tok::l_paren)) {
1199         parseParens();
1200         continue;
1201       }
1202       if (FormatTok->isOneOf(tok::semi, tok::r_brace, tok::eof))
1203         return;
1204       nextToken();
1205     }
1206     NeedsUnwrappedLine = false;
1207     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1208     parseBlock(/*MustBeDeclaration=*/false);
1209     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1210         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1211         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1212       addUnwrappedLine();
1213     } else {
1214       NeedsUnwrappedLine = true;
1215     }
1216   }
1217   if (NeedsUnwrappedLine) {
1218     addUnwrappedLine();
1219   }
1220 }
1221 
1222 void UnwrappedLineParser::parseNamespace() {
1223   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1224 
1225   const FormatToken &InitialToken = *FormatTok;
1226   nextToken();
1227   if (FormatTok->Tok.is(tok::identifier))
1228     nextToken();
1229   if (FormatTok->Tok.is(tok::l_brace)) {
1230     if (ShouldBreakBeforeBrace(Style, InitialToken))
1231       addUnwrappedLine();
1232 
1233     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1234                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1235                      DeclarationScopeStack.size() > 1);
1236     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1237     // Munch the semicolon after a namespace. This is more common than one would
1238     // think. Puttin the semicolon into its own line is very ugly.
1239     if (FormatTok->Tok.is(tok::semi))
1240       nextToken();
1241     addUnwrappedLine();
1242   }
1243   // FIXME: Add error handling.
1244 }
1245 
1246 void UnwrappedLineParser::parseForOrWhileLoop() {
1247   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1248           FormatTok->IsForEachMacro) &&
1249          "'for', 'while' or foreach macro expected");
1250   nextToken();
1251   if (FormatTok->Tok.is(tok::l_paren))
1252     parseParens();
1253   if (FormatTok->Tok.is(tok::l_brace)) {
1254     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1255     parseBlock(/*MustBeDeclaration=*/false);
1256     addUnwrappedLine();
1257   } else {
1258     addUnwrappedLine();
1259     ++Line->Level;
1260     parseStructuralElement();
1261     --Line->Level;
1262   }
1263 }
1264 
1265 void UnwrappedLineParser::parseDoWhile() {
1266   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1267   nextToken();
1268   if (FormatTok->Tok.is(tok::l_brace)) {
1269     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1270     parseBlock(/*MustBeDeclaration=*/false);
1271     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1272       addUnwrappedLine();
1273   } else {
1274     addUnwrappedLine();
1275     ++Line->Level;
1276     parseStructuralElement();
1277     --Line->Level;
1278   }
1279 
1280   // FIXME: Add error handling.
1281   if (!FormatTok->Tok.is(tok::kw_while)) {
1282     addUnwrappedLine();
1283     return;
1284   }
1285 
1286   nextToken();
1287   parseStructuralElement();
1288 }
1289 
1290 void UnwrappedLineParser::parseLabel() {
1291   nextToken();
1292   unsigned OldLineLevel = Line->Level;
1293   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1294     --Line->Level;
1295   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1296     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1297     parseBlock(/*MustBeDeclaration=*/false);
1298     if (FormatTok->Tok.is(tok::kw_break)) {
1299       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1300       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1301           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1302         addUnwrappedLine();
1303       }
1304       parseStructuralElement();
1305     }
1306     addUnwrappedLine();
1307   } else {
1308     addUnwrappedLine();
1309   }
1310   Line->Level = OldLineLevel;
1311 }
1312 
1313 void UnwrappedLineParser::parseCaseLabel() {
1314   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1315   // FIXME: fix handling of complex expressions here.
1316   do {
1317     nextToken();
1318   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1319   parseLabel();
1320 }
1321 
1322 void UnwrappedLineParser::parseSwitch() {
1323   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1324   nextToken();
1325   if (FormatTok->Tok.is(tok::l_paren))
1326     parseParens();
1327   if (FormatTok->Tok.is(tok::l_brace)) {
1328     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1329     parseBlock(/*MustBeDeclaration=*/false);
1330     addUnwrappedLine();
1331   } else {
1332     addUnwrappedLine();
1333     ++Line->Level;
1334     parseStructuralElement();
1335     --Line->Level;
1336   }
1337 }
1338 
1339 void UnwrappedLineParser::parseAccessSpecifier() {
1340   nextToken();
1341   // Understand Qt's slots.
1342   if (FormatTok->is(tok::identifier) &&
1343       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1344     nextToken();
1345   // Otherwise, we don't know what it is, and we'd better keep the next token.
1346   if (FormatTok->Tok.is(tok::colon))
1347     nextToken();
1348   addUnwrappedLine();
1349 }
1350 
1351 void UnwrappedLineParser::parseEnum() {
1352   // Won't be 'enum' for NS_ENUMs.
1353   if (FormatTok->Tok.is(tok::kw_enum))
1354     nextToken();
1355 
1356   // Eat up enum class ...
1357   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1358     nextToken();
1359   while (FormatTok->Tok.getIdentifierInfo() ||
1360          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1361                             tok::greater, tok::comma, tok::question)) {
1362     nextToken();
1363     // We can have macros or attributes in between 'enum' and the enum name.
1364     if (FormatTok->is(tok::l_paren))
1365       parseParens();
1366     if (FormatTok->is(tok::identifier))
1367       nextToken();
1368   }
1369 
1370   // Just a declaration or something is wrong.
1371   if (FormatTok->isNot(tok::l_brace))
1372     return;
1373   FormatTok->BlockKind = BK_Block;
1374 
1375   if (Style.Language == FormatStyle::LK_Java) {
1376     // Java enums are different.
1377     parseJavaEnumBody();
1378     return;
1379   }
1380 
1381   // Parse enum body.
1382   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1383   if (HasError) {
1384     if (FormatTok->is(tok::semi))
1385       nextToken();
1386     addUnwrappedLine();
1387   }
1388 
1389   // We fall through to parsing a structural element afterwards, so that in
1390   // enum A {} n, m;
1391   // "} n, m;" will end up in one unwrapped line.
1392 }
1393 
1394 void UnwrappedLineParser::parseJavaEnumBody() {
1395   // Determine whether the enum is simple, i.e. does not have a semicolon or
1396   // constants with class bodies. Simple enums can be formatted like braced
1397   // lists, contracted to a single line, etc.
1398   unsigned StoredPosition = Tokens->getPosition();
1399   bool IsSimple = true;
1400   FormatToken *Tok = Tokens->getNextToken();
1401   while (Tok) {
1402     if (Tok->is(tok::r_brace))
1403       break;
1404     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1405       IsSimple = false;
1406       break;
1407     }
1408     // FIXME: This will also mark enums with braces in the arguments to enum
1409     // constants as "not simple". This is probably fine in practice, though.
1410     Tok = Tokens->getNextToken();
1411   }
1412   FormatTok = Tokens->setPosition(StoredPosition);
1413 
1414   if (IsSimple) {
1415     parseBracedList();
1416     addUnwrappedLine();
1417     return;
1418   }
1419 
1420   // Parse the body of a more complex enum.
1421   // First add a line for everything up to the "{".
1422   nextToken();
1423   addUnwrappedLine();
1424   ++Line->Level;
1425 
1426   // Parse the enum constants.
1427   while (FormatTok) {
1428     if (FormatTok->is(tok::l_brace)) {
1429       // Parse the constant's class body.
1430       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1431                  /*MunchSemi=*/false);
1432     } else if (FormatTok->is(tok::l_paren)) {
1433       parseParens();
1434     } else if (FormatTok->is(tok::comma)) {
1435       nextToken();
1436       addUnwrappedLine();
1437     } else if (FormatTok->is(tok::semi)) {
1438       nextToken();
1439       addUnwrappedLine();
1440       break;
1441     } else if (FormatTok->is(tok::r_brace)) {
1442       addUnwrappedLine();
1443       break;
1444     } else {
1445       nextToken();
1446     }
1447   }
1448 
1449   // Parse the class body after the enum's ";" if any.
1450   parseLevel(/*HasOpeningBrace=*/true);
1451   nextToken();
1452   --Line->Level;
1453   addUnwrappedLine();
1454 }
1455 
1456 void UnwrappedLineParser::parseRecord() {
1457   const FormatToken &InitialToken = *FormatTok;
1458   nextToken();
1459   if (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::kw___attribute,
1460                          tok::kw___declspec, tok::kw_alignas)) {
1461     nextToken();
1462     // We can have macros or attributes in between 'class' and the class name.
1463     if (FormatTok->Tok.is(tok::l_paren)) {
1464       parseParens();
1465     }
1466     // The actual identifier can be a nested name specifier, and in macros
1467     // it is often token-pasted.
1468     while (FormatTok->is(tok::identifier) || FormatTok->is(tok::coloncolon) ||
1469            FormatTok->is(tok::hashhash) ||
1470            (Style.Language == FormatStyle::LK_Java &&
1471             FormatTok->isOneOf(tok::period, tok::comma)))
1472       nextToken();
1473 
1474     // Note that parsing away template declarations here leads to incorrectly
1475     // accepting function declarations as record declarations.
1476     // In general, we cannot solve this problem. Consider:
1477     // class A<int> B() {}
1478     // which can be a function definition or a class definition when B() is a
1479     // macro. If we find enough real-world cases where this is a problem, we
1480     // can parse for the 'template' keyword in the beginning of the statement,
1481     // and thus rule out the record production in case there is no template
1482     // (this would still leave us with an ambiguity between template function
1483     // and class declarations).
1484     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1485       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1486         if (FormatTok->Tok.is(tok::semi))
1487           return;
1488         nextToken();
1489       }
1490     }
1491   }
1492   if (FormatTok->Tok.is(tok::l_brace)) {
1493     if (ShouldBreakBeforeBrace(Style, InitialToken))
1494       addUnwrappedLine();
1495 
1496     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1497                /*MunchSemi=*/false);
1498   }
1499   // We fall through to parsing a structural element afterwards, so
1500   // class A {} n, m;
1501   // will end up in one unwrapped line.
1502   // This does not apply for Java.
1503   if (Style.Language == FormatStyle::LK_Java)
1504     addUnwrappedLine();
1505 }
1506 
1507 void UnwrappedLineParser::parseObjCProtocolList() {
1508   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1509   do
1510     nextToken();
1511   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1512   nextToken(); // Skip '>'.
1513 }
1514 
1515 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1516   do {
1517     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1518       nextToken();
1519       addUnwrappedLine();
1520       break;
1521     }
1522     if (FormatTok->is(tok::l_brace)) {
1523       parseBlock(/*MustBeDeclaration=*/false);
1524       // In ObjC interfaces, nothing should be following the "}".
1525       addUnwrappedLine();
1526     } else if (FormatTok->is(tok::r_brace)) {
1527       // Ignore stray "}". parseStructuralElement doesn't consume them.
1528       nextToken();
1529       addUnwrappedLine();
1530     } else {
1531       parseStructuralElement();
1532     }
1533   } while (!eof());
1534 }
1535 
1536 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1537   nextToken();
1538   nextToken(); // interface name
1539 
1540   // @interface can be followed by either a base class, or a category.
1541   if (FormatTok->Tok.is(tok::colon)) {
1542     nextToken();
1543     nextToken(); // base class name
1544   } else if (FormatTok->Tok.is(tok::l_paren))
1545     // Skip category, if present.
1546     parseParens();
1547 
1548   if (FormatTok->Tok.is(tok::less))
1549     parseObjCProtocolList();
1550 
1551   if (FormatTok->Tok.is(tok::l_brace)) {
1552     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1553         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1554       addUnwrappedLine();
1555     parseBlock(/*MustBeDeclaration=*/true);
1556   }
1557 
1558   // With instance variables, this puts '}' on its own line.  Without instance
1559   // variables, this ends the @interface line.
1560   addUnwrappedLine();
1561 
1562   parseObjCUntilAtEnd();
1563 }
1564 
1565 void UnwrappedLineParser::parseObjCProtocol() {
1566   nextToken();
1567   nextToken(); // protocol name
1568 
1569   if (FormatTok->Tok.is(tok::less))
1570     parseObjCProtocolList();
1571 
1572   // Check for protocol declaration.
1573   if (FormatTok->Tok.is(tok::semi)) {
1574     nextToken();
1575     return addUnwrappedLine();
1576   }
1577 
1578   addUnwrappedLine();
1579   parseObjCUntilAtEnd();
1580 }
1581 
1582 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1583                                                  StringRef Prefix = "") {
1584   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1585                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1586   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1587                                                     E = Line.Tokens.end();
1588        I != E; ++I) {
1589     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1590   }
1591   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1592                                                     E = Line.Tokens.end();
1593        I != E; ++I) {
1594     const UnwrappedLineNode &Node = *I;
1595     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1596              I = Node.Children.begin(),
1597              E = Node.Children.end();
1598          I != E; ++I) {
1599       printDebugInfo(*I, "\nChild: ");
1600     }
1601   }
1602   llvm::dbgs() << "\n";
1603 }
1604 
1605 void UnwrappedLineParser::addUnwrappedLine() {
1606   if (Line->Tokens.empty())
1607     return;
1608   DEBUG({
1609     if (CurrentLines == &Lines)
1610       printDebugInfo(*Line);
1611   });
1612   CurrentLines->push_back(*Line);
1613   Line->Tokens.clear();
1614   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1615     for (SmallVectorImpl<UnwrappedLine>::iterator
1616              I = PreprocessorDirectives.begin(),
1617              E = PreprocessorDirectives.end();
1618          I != E; ++I) {
1619       CurrentLines->push_back(*I);
1620     }
1621     PreprocessorDirectives.clear();
1622   }
1623 }
1624 
1625 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1626 
1627 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
1628   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1629          FormatTok.NewlinesBefore > 0;
1630 }
1631 
1632 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1633   bool JustComments = Line->Tokens.empty();
1634   for (SmallVectorImpl<FormatToken *>::const_iterator
1635            I = CommentsBeforeNextToken.begin(),
1636            E = CommentsBeforeNextToken.end();
1637        I != E; ++I) {
1638     if (isOnNewLine(**I) && JustComments) {
1639       addUnwrappedLine();
1640     }
1641     pushToken(*I);
1642   }
1643   if (NewlineBeforeNext && JustComments) {
1644     addUnwrappedLine();
1645   }
1646   CommentsBeforeNextToken.clear();
1647 }
1648 
1649 void UnwrappedLineParser::nextToken() {
1650   if (eof())
1651     return;
1652   flushComments(isOnNewLine(*FormatTok));
1653   pushToken(FormatTok);
1654   readToken();
1655 }
1656 
1657 void UnwrappedLineParser::readToken() {
1658   bool CommentsInCurrentLine = true;
1659   do {
1660     FormatTok = Tokens->getNextToken();
1661     assert(FormatTok);
1662     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1663            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1664       // If there is an unfinished unwrapped line, we flush the preprocessor
1665       // directives only after that unwrapped line was finished later.
1666       bool SwitchToPreprocessorLines =
1667           !Line->Tokens.empty() && CurrentLines == &Lines;
1668       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1669       // Comments stored before the preprocessor directive need to be output
1670       // before the preprocessor directive, at the same level as the
1671       // preprocessor directive, as we consider them to apply to the directive.
1672       flushComments(isOnNewLine(*FormatTok));
1673       parsePPDirective();
1674     }
1675     while (FormatTok->Type == TT_ConflictStart ||
1676            FormatTok->Type == TT_ConflictEnd ||
1677            FormatTok->Type == TT_ConflictAlternative) {
1678       if (FormatTok->Type == TT_ConflictStart) {
1679         conditionalCompilationStart(/*Unreachable=*/false);
1680       } else if (FormatTok->Type == TT_ConflictAlternative) {
1681         conditionalCompilationAlternative();
1682       } else if (FormatTok->Type == TT_ConflictEnd) {
1683         conditionalCompilationEnd();
1684       }
1685       FormatTok = Tokens->getNextToken();
1686       FormatTok->MustBreakBefore = true;
1687     }
1688 
1689     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1690         !Line->InPPDirective) {
1691       continue;
1692     }
1693 
1694     if (!FormatTok->Tok.is(tok::comment))
1695       return;
1696     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
1697       CommentsInCurrentLine = false;
1698     }
1699     if (CommentsInCurrentLine) {
1700       pushToken(FormatTok);
1701     } else {
1702       CommentsBeforeNextToken.push_back(FormatTok);
1703     }
1704   } while (!eof());
1705 }
1706 
1707 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1708   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1709   if (MustBreakBeforeNextToken) {
1710     Line->Tokens.back().Tok->MustBreakBefore = true;
1711     MustBreakBeforeNextToken = false;
1712   }
1713 }
1714 
1715 } // end namespace format
1716 } // end namespace clang
1717