1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "format-parser"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(nullptr) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser), OriginalLines(Parser.CurrentLines) {
126     if (SwitchToPreprocessorLines)
127       Parser.CurrentLines = &Parser.PreprocessorDirectives;
128     else if (!Parser.Line->Tokens.empty())
129       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
130     PreBlockLine = std::move(Parser.Line);
131     Parser.Line = llvm::make_unique<UnwrappedLine>();
132     Parser.Line->Level = PreBlockLine->Level;
133     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
134   }
135 
136   ~ScopedLineState() {
137     if (!Parser.Line->Tokens.empty()) {
138       Parser.addUnwrappedLine();
139     }
140     assert(Parser.Line->Tokens.empty());
141     Parser.Line = std::move(PreBlockLine);
142     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
143       Parser.MustBreakBeforeNextToken = true;
144     Parser.CurrentLines = OriginalLines;
145   }
146 
147 private:
148   UnwrappedLineParser &Parser;
149 
150   std::unique_ptr<UnwrappedLine> PreBlockLine;
151   SmallVectorImpl<UnwrappedLine> *OriginalLines;
152 };
153 
154 class CompoundStatementIndenter {
155 public:
156   CompoundStatementIndenter(UnwrappedLineParser *Parser,
157                             const FormatStyle &Style, unsigned &LineLevel)
158       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
159     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
160       Parser->addUnwrappedLine();
161     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
162       Parser->addUnwrappedLine();
163       ++LineLevel;
164     }
165   }
166   ~CompoundStatementIndenter() { LineLevel = OldLineLevel; }
167 
168 private:
169   unsigned &LineLevel;
170   unsigned OldLineLevel;
171 };
172 
173 namespace {
174 
175 class IndexedTokenSource : public FormatTokenSource {
176 public:
177   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
178       : Tokens(Tokens), Position(-1) {}
179 
180   FormatToken *getNextToken() override {
181     ++Position;
182     return Tokens[Position];
183   }
184 
185   unsigned getPosition() override {
186     assert(Position >= 0);
187     return Position;
188   }
189 
190   FormatToken *setPosition(unsigned P) override {
191     Position = P;
192     return Tokens[Position];
193   }
194 
195   void reset() { Position = -1; }
196 
197 private:
198   ArrayRef<FormatToken *> Tokens;
199   int Position;
200 };
201 
202 } // end anonymous namespace
203 
204 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
205                                          const AdditionalKeywords &Keywords,
206                                          ArrayRef<FormatToken *> Tokens,
207                                          UnwrappedLineConsumer &Callback)
208     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
209       CurrentLines(&Lines), StructuralError(false), Style(Style),
210       Keywords(Keywords), Tokens(nullptr), Callback(Callback),
211       AllTokens(Tokens), PPBranchLevel(-1) {}
212 
213 void UnwrappedLineParser::reset() {
214   PPBranchLevel = -1;
215   Line.reset(new UnwrappedLine);
216   CommentsBeforeNextToken.clear();
217   FormatTok = nullptr;
218   MustBreakBeforeNextToken = false;
219   PreprocessorDirectives.clear();
220   CurrentLines = &Lines;
221   DeclarationScopeStack.clear();
222   StructuralError = false;
223   PPStack.clear();
224 }
225 
226 bool UnwrappedLineParser::parse() {
227   IndexedTokenSource TokenSource(AllTokens);
228   do {
229     DEBUG(llvm::dbgs() << "----\n");
230     reset();
231     Tokens = &TokenSource;
232     TokenSource.reset();
233 
234     readToken();
235     parseFile();
236     // Create line with eof token.
237     pushToken(FormatTok);
238     addUnwrappedLine();
239 
240     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
241                                                   E = Lines.end();
242          I != E; ++I) {
243       Callback.consumeUnwrappedLine(*I);
244     }
245     Callback.finishRun();
246     Lines.clear();
247     while (!PPLevelBranchIndex.empty() &&
248            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
249       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
250       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
251     }
252     if (!PPLevelBranchIndex.empty()) {
253       ++PPLevelBranchIndex.back();
254       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
255       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
256     }
257   } while (!PPLevelBranchIndex.empty());
258 
259   return StructuralError;
260 }
261 
262 void UnwrappedLineParser::parseFile() {
263   ScopedDeclarationState DeclarationState(
264       *Line, DeclarationScopeStack,
265       /*MustBeDeclaration=*/ !Line->InPPDirective);
266   parseLevel(/*HasOpeningBrace=*/false);
267   // Make sure to format the remaining tokens.
268   flushComments(true);
269   addUnwrappedLine();
270 }
271 
272 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
273   bool SwitchLabelEncountered = false;
274   do {
275     switch (FormatTok->Tok.getKind()) {
276     case tok::comment:
277       nextToken();
278       addUnwrappedLine();
279       break;
280     case tok::l_brace:
281       // FIXME: Add parameter whether this can happen - if this happens, we must
282       // be in a non-declaration context.
283       parseBlock(/*MustBeDeclaration=*/false);
284       addUnwrappedLine();
285       break;
286     case tok::r_brace:
287       if (HasOpeningBrace)
288         return;
289       StructuralError = true;
290       nextToken();
291       addUnwrappedLine();
292       break;
293     case tok::kw_default:
294     case tok::kw_case:
295       if (!SwitchLabelEncountered &&
296           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
297         ++Line->Level;
298       SwitchLabelEncountered = true;
299       parseStructuralElement();
300       break;
301     default:
302       parseStructuralElement();
303       break;
304     }
305   } while (!eof());
306 }
307 
308 void UnwrappedLineParser::calculateBraceTypes() {
309   // We'll parse forward through the tokens until we hit
310   // a closing brace or eof - note that getNextToken() will
311   // parse macros, so this will magically work inside macro
312   // definitions, too.
313   unsigned StoredPosition = Tokens->getPosition();
314   FormatToken *Tok = FormatTok;
315   // Keep a stack of positions of lbrace tokens. We will
316   // update information about whether an lbrace starts a
317   // braced init list or a different block during the loop.
318   SmallVector<FormatToken *, 8> LBraceStack;
319   assert(Tok->Tok.is(tok::l_brace));
320   do {
321     // Get next none-comment token.
322     FormatToken *NextTok;
323     unsigned ReadTokens = 0;
324     do {
325       NextTok = Tokens->getNextToken();
326       ++ReadTokens;
327     } while (NextTok->is(tok::comment));
328 
329     switch (Tok->Tok.getKind()) {
330     case tok::l_brace:
331       LBraceStack.push_back(Tok);
332       break;
333     case tok::r_brace:
334       if (!LBraceStack.empty()) {
335         if (LBraceStack.back()->BlockKind == BK_Unknown) {
336           bool ProbablyBracedList = false;
337           if (Style.Language == FormatStyle::LK_Proto) {
338             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
339           } else {
340             // Using OriginalColumn to distinguish between ObjC methods and
341             // binary operators is a bit hacky.
342             bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
343                                     NextTok->OriginalColumn == 0;
344 
345             // If there is a comma, semicolon or right paren after the closing
346             // brace, we assume this is a braced initializer list.  Note that
347             // regardless how we mark inner braces here, we will overwrite the
348             // BlockKind later if we parse a braced list (where all blocks
349             // inside are by default braced lists), or when we explicitly detect
350             // blocks (for example while parsing lambdas).
351             //
352             // We exclude + and - as they can be ObjC visibility modifiers.
353             ProbablyBracedList =
354                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
355                                  tok::r_paren, tok::r_square, tok::l_brace,
356                                  tok::l_paren, tok::ellipsis) ||
357                 (NextTok->isBinaryOperator() && !NextIsObjCMethod);
358           }
359           if (ProbablyBracedList) {
360             Tok->BlockKind = BK_BracedInit;
361             LBraceStack.back()->BlockKind = BK_BracedInit;
362           } else {
363             Tok->BlockKind = BK_Block;
364             LBraceStack.back()->BlockKind = BK_Block;
365           }
366         }
367         LBraceStack.pop_back();
368       }
369       break;
370     case tok::at:
371     case tok::semi:
372     case tok::kw_if:
373     case tok::kw_while:
374     case tok::kw_for:
375     case tok::kw_switch:
376     case tok::kw_try:
377       if (!LBraceStack.empty())
378         LBraceStack.back()->BlockKind = BK_Block;
379       break;
380     default:
381       break;
382     }
383     Tok = NextTok;
384   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
385   // Assume other blocks for all unclosed opening braces.
386   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
387     if (LBraceStack[i]->BlockKind == BK_Unknown)
388       LBraceStack[i]->BlockKind = BK_Block;
389   }
390 
391   FormatTok = Tokens->setPosition(StoredPosition);
392 }
393 
394 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
395                                      bool MunchSemi) {
396   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
397   unsigned InitialLevel = Line->Level;
398   nextToken();
399 
400   addUnwrappedLine();
401 
402   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
403                                           MustBeDeclaration);
404   if (AddLevel)
405     ++Line->Level;
406   parseLevel(/*HasOpeningBrace=*/true);
407 
408   if (!FormatTok->Tok.is(tok::r_brace)) {
409     Line->Level = InitialLevel;
410     StructuralError = true;
411     return;
412   }
413 
414   nextToken(); // Munch the closing brace.
415   if (MunchSemi && FormatTok->Tok.is(tok::semi))
416     nextToken();
417   Line->Level = InitialLevel;
418 }
419 
420 static bool IsGoogScope(const UnwrappedLine &Line) {
421   if (Line.Tokens.size() < 4)
422     return false;
423   auto I = Line.Tokens.begin();
424   if (I->Tok->TokenText != "goog")
425     return false;
426   ++I;
427   if (I->Tok->isNot(tok::period))
428     return false;
429   ++I;
430   if (I->Tok->TokenText != "scope")
431     return false;
432   ++I;
433   return I->Tok->is(tok::l_paren);
434 }
435 
436 static bool ShouldBreakBeforeBrace(const FormatStyle &Style,
437                                    const FormatToken &InitialToken) {
438   switch (Style.BreakBeforeBraces) {
439   case FormatStyle::BS_Linux:
440     return InitialToken.isOneOf(tok::kw_namespace, tok::kw_class);
441   case FormatStyle::BS_Allman:
442   case FormatStyle::BS_GNU:
443     return true;
444   default:
445     return false;
446   }
447 }
448 
449 void UnwrappedLineParser::parseChildBlock() {
450   FormatTok->BlockKind = BK_Block;
451   nextToken();
452   {
453     bool GoogScope =
454         Style.Language == FormatStyle::LK_JavaScript && IsGoogScope(*Line);
455     ScopedLineState LineState(*this);
456     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
457                                             /*MustBeDeclaration=*/false);
458     Line->Level += GoogScope ? 0 : 1;
459     parseLevel(/*HasOpeningBrace=*/true);
460     Line->Level -= GoogScope ? 0 : 1;
461   }
462   nextToken();
463 }
464 
465 void UnwrappedLineParser::parsePPDirective() {
466   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
467   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
468   nextToken();
469 
470   if (!FormatTok->Tok.getIdentifierInfo()) {
471     parsePPUnknown();
472     return;
473   }
474 
475   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
476   case tok::pp_define:
477     parsePPDefine();
478     return;
479   case tok::pp_if:
480     parsePPIf(/*IfDef=*/false);
481     break;
482   case tok::pp_ifdef:
483   case tok::pp_ifndef:
484     parsePPIf(/*IfDef=*/true);
485     break;
486   case tok::pp_else:
487     parsePPElse();
488     break;
489   case tok::pp_elif:
490     parsePPElIf();
491     break;
492   case tok::pp_endif:
493     parsePPEndIf();
494     break;
495   default:
496     parsePPUnknown();
497     break;
498   }
499 }
500 
501 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
502   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
503     PPStack.push_back(PP_Unreachable);
504   else
505     PPStack.push_back(PP_Conditional);
506 }
507 
508 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
509   ++PPBranchLevel;
510   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
511   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
512     PPLevelBranchIndex.push_back(0);
513     PPLevelBranchCount.push_back(0);
514   }
515   PPChainBranchIndex.push(0);
516   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
517   conditionalCompilationCondition(Unreachable || Skip);
518 }
519 
520 void UnwrappedLineParser::conditionalCompilationAlternative() {
521   if (!PPStack.empty())
522     PPStack.pop_back();
523   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
524   if (!PPChainBranchIndex.empty())
525     ++PPChainBranchIndex.top();
526   conditionalCompilationCondition(
527       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
528       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
529 }
530 
531 void UnwrappedLineParser::conditionalCompilationEnd() {
532   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
533   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
534     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
535       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
536     }
537   }
538   // Guard against #endif's without #if.
539   if (PPBranchLevel > 0)
540     --PPBranchLevel;
541   if (!PPChainBranchIndex.empty())
542     PPChainBranchIndex.pop();
543   if (!PPStack.empty())
544     PPStack.pop_back();
545 }
546 
547 void UnwrappedLineParser::parsePPIf(bool IfDef) {
548   nextToken();
549   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
550                          StringRef(FormatTok->Tok.getLiteralData(),
551                                    FormatTok->Tok.getLength()) == "0") ||
552                         FormatTok->Tok.is(tok::kw_false);
553   conditionalCompilationStart(!IfDef && IsLiteralFalse);
554   parsePPUnknown();
555 }
556 
557 void UnwrappedLineParser::parsePPElse() {
558   conditionalCompilationAlternative();
559   parsePPUnknown();
560 }
561 
562 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
563 
564 void UnwrappedLineParser::parsePPEndIf() {
565   conditionalCompilationEnd();
566   parsePPUnknown();
567 }
568 
569 void UnwrappedLineParser::parsePPDefine() {
570   nextToken();
571 
572   if (FormatTok->Tok.getKind() != tok::identifier) {
573     parsePPUnknown();
574     return;
575   }
576   nextToken();
577   if (FormatTok->Tok.getKind() == tok::l_paren &&
578       FormatTok->WhitespaceRange.getBegin() ==
579           FormatTok->WhitespaceRange.getEnd()) {
580     parseParens();
581   }
582   addUnwrappedLine();
583   Line->Level = 1;
584 
585   // Errors during a preprocessor directive can only affect the layout of the
586   // preprocessor directive, and thus we ignore them. An alternative approach
587   // would be to use the same approach we use on the file level (no
588   // re-indentation if there was a structural error) within the macro
589   // definition.
590   parseFile();
591 }
592 
593 void UnwrappedLineParser::parsePPUnknown() {
594   do {
595     nextToken();
596   } while (!eof());
597   addUnwrappedLine();
598 }
599 
600 // Here we blacklist certain tokens that are not usually the first token in an
601 // unwrapped line. This is used in attempt to distinguish macro calls without
602 // trailing semicolons from other constructs split to several lines.
603 bool tokenCanStartNewLine(clang::Token Tok) {
604   // Semicolon can be a null-statement, l_square can be a start of a macro or
605   // a C++11 attribute, but this doesn't seem to be common.
606   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
607          Tok.isNot(tok::l_square) &&
608          // Tokens that can only be used as binary operators and a part of
609          // overloaded operator names.
610          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
611          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
612          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
613          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
614          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
615          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
616          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
617          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
618          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
619          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
620          Tok.isNot(tok::lesslessequal) &&
621          // Colon is used in labels, base class lists, initializer lists,
622          // range-based for loops, ternary operator, but should never be the
623          // first token in an unwrapped line.
624          Tok.isNot(tok::colon) &&
625          // 'noexcept' is a trailing annotation.
626          Tok.isNot(tok::kw_noexcept);
627 }
628 
629 void UnwrappedLineParser::parseStructuralElement() {
630   assert(!FormatTok->Tok.is(tok::l_brace));
631   switch (FormatTok->Tok.getKind()) {
632   case tok::at:
633     nextToken();
634     if (FormatTok->Tok.is(tok::l_brace)) {
635       parseBracedList();
636       break;
637     }
638     switch (FormatTok->Tok.getObjCKeywordID()) {
639     case tok::objc_public:
640     case tok::objc_protected:
641     case tok::objc_package:
642     case tok::objc_private:
643       return parseAccessSpecifier();
644     case tok::objc_interface:
645     case tok::objc_implementation:
646       return parseObjCInterfaceOrImplementation();
647     case tok::objc_protocol:
648       return parseObjCProtocol();
649     case tok::objc_end:
650       return; // Handled by the caller.
651     case tok::objc_optional:
652     case tok::objc_required:
653       nextToken();
654       addUnwrappedLine();
655       return;
656     default:
657       break;
658     }
659     break;
660   case tok::kw_asm:
661     FormatTok->Finalized = true;
662     nextToken();
663     if (FormatTok->is(tok::l_brace)) {
664       while (FormatTok && FormatTok->isNot(tok::eof)) {
665         FormatTok->Finalized = true;
666         if (FormatTok->is(tok::r_brace)) {
667           nextToken();
668           break;
669         }
670         nextToken();
671       }
672     }
673     break;
674   case tok::kw_namespace:
675     parseNamespace();
676     return;
677   case tok::kw_inline:
678     nextToken();
679     if (FormatTok->Tok.is(tok::kw_namespace)) {
680       parseNamespace();
681       return;
682     }
683     break;
684   case tok::kw_public:
685   case tok::kw_protected:
686   case tok::kw_private:
687     if (Style.Language == FormatStyle::LK_Java)
688       nextToken();
689     else
690       parseAccessSpecifier();
691     return;
692   case tok::kw_if:
693     parseIfThenElse();
694     return;
695   case tok::kw_for:
696   case tok::kw_while:
697     parseForOrWhileLoop();
698     return;
699   case tok::kw_do:
700     parseDoWhile();
701     return;
702   case tok::kw_switch:
703     parseSwitch();
704     return;
705   case tok::kw_default:
706     nextToken();
707     parseLabel();
708     return;
709   case tok::kw_case:
710     parseCaseLabel();
711     return;
712   case tok::kw_try:
713     parseTryCatch();
714     return;
715   case tok::kw_extern:
716     nextToken();
717     if (FormatTok->Tok.is(tok::string_literal)) {
718       nextToken();
719       if (FormatTok->Tok.is(tok::l_brace)) {
720         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
721         addUnwrappedLine();
722         return;
723       }
724     }
725     break;
726   case tok::identifier:
727     if (FormatTok->IsForEachMacro) {
728       parseForOrWhileLoop();
729       return;
730     }
731     // In all other cases, parse the declaration.
732     break;
733   default:
734     break;
735   }
736   do {
737     switch (FormatTok->Tok.getKind()) {
738     case tok::at:
739       nextToken();
740       if (FormatTok->Tok.is(tok::l_brace))
741         parseBracedList();
742       break;
743     case tok::kw_enum:
744       parseEnum();
745       break;
746     case tok::kw_typedef:
747       nextToken();
748       if (FormatTok->is(Keywords.kw_NS_ENUM))
749         parseEnum();
750       break;
751     case tok::kw_struct:
752     case tok::kw_union:
753     case tok::kw_class:
754       parseRecord();
755       // A record declaration or definition is always the start of a structural
756       // element.
757       break;
758     case tok::semi:
759       nextToken();
760       addUnwrappedLine();
761       return;
762     case tok::r_brace:
763       addUnwrappedLine();
764       return;
765     case tok::l_paren:
766       parseParens();
767       break;
768     case tok::caret:
769       nextToken();
770       if (FormatTok->Tok.isAnyIdentifier() ||
771           FormatTok->isSimpleTypeSpecifier())
772         nextToken();
773       if (FormatTok->is(tok::l_paren))
774         parseParens();
775       if (FormatTok->is(tok::l_brace))
776         parseChildBlock();
777       break;
778     case tok::l_brace:
779       if (!tryToParseBracedList()) {
780         // A block outside of parentheses must be the last part of a
781         // structural element.
782         // FIXME: Figure out cases where this is not true, and add projections
783         // for them (the one we know is missing are lambdas).
784         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
785           addUnwrappedLine();
786         FormatTok->Type = TT_FunctionLBrace;
787         parseBlock(/*MustBeDeclaration=*/false);
788         addUnwrappedLine();
789         return;
790       }
791       // Otherwise this was a braced init list, and the structural
792       // element continues.
793       break;
794     case tok::kw_try:
795       // We arrive here when parsing function-try blocks.
796       parseTryCatch();
797       return;
798     case tok::identifier: {
799       StringRef Text = FormatTok->TokenText;
800       // Parse function literal unless 'function' is the first token in a line
801       // in which case this should be treated as a free-standing function.
802       if (Style.Language == FormatStyle::LK_JavaScript && Text == "function" &&
803           Line->Tokens.size() > 0) {
804         tryToParseJSFunction();
805         break;
806       }
807       nextToken();
808       if (Line->Tokens.size() == 1) {
809         if (FormatTok->Tok.is(tok::colon)) {
810           parseLabel();
811           return;
812         }
813         // Recognize function-like macro usages without trailing semicolon as
814         // well as free-standing macrose like Q_OBJECT.
815         bool FunctionLike = FormatTok->is(tok::l_paren);
816         if (FunctionLike)
817           parseParens();
818         if (FormatTok->NewlinesBefore > 0 &&
819             (Text.size() >= 5 || FunctionLike) &&
820             tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
821           addUnwrappedLine();
822           return;
823         }
824       }
825       break;
826     }
827     case tok::equal:
828       nextToken();
829       if (FormatTok->Tok.is(tok::l_brace)) {
830         parseBracedList();
831       }
832       break;
833     case tok::l_square:
834       parseSquare();
835       break;
836     default:
837       nextToken();
838       break;
839     }
840   } while (!eof());
841 }
842 
843 bool UnwrappedLineParser::tryToParseLambda() {
844   // FIXME: This is a dirty way to access the previous token. Find a better
845   // solution.
846   if (!Line->Tokens.empty() &&
847       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator,
848                                         tok::kw_new, tok::kw_delete) ||
849        Line->Tokens.back().Tok->closesScope() ||
850        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
851     nextToken();
852     return false;
853   }
854   assert(FormatTok->is(tok::l_square));
855   FormatToken &LSquare = *FormatTok;
856   if (!tryToParseLambdaIntroducer())
857     return false;
858 
859   while (FormatTok->isNot(tok::l_brace)) {
860     if (FormatTok->isSimpleTypeSpecifier()) {
861       nextToken();
862       continue;
863     }
864     switch (FormatTok->Tok.getKind()) {
865     case tok::l_brace:
866       break;
867     case tok::l_paren:
868       parseParens();
869       break;
870     case tok::less:
871     case tok::greater:
872     case tok::identifier:
873     case tok::coloncolon:
874     case tok::kw_mutable:
875       nextToken();
876       break;
877     case tok::arrow:
878       FormatTok->Type = TT_TrailingReturnArrow;
879       nextToken();
880       break;
881     default:
882       return true;
883     }
884   }
885   LSquare.Type = TT_LambdaLSquare;
886   parseChildBlock();
887   return true;
888 }
889 
890 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
891   nextToken();
892   if (FormatTok->is(tok::equal)) {
893     nextToken();
894     if (FormatTok->is(tok::r_square)) {
895       nextToken();
896       return true;
897     }
898     if (FormatTok->isNot(tok::comma))
899       return false;
900     nextToken();
901   } else if (FormatTok->is(tok::amp)) {
902     nextToken();
903     if (FormatTok->is(tok::r_square)) {
904       nextToken();
905       return true;
906     }
907     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
908       return false;
909     }
910     if (FormatTok->is(tok::comma))
911       nextToken();
912   } else if (FormatTok->is(tok::r_square)) {
913     nextToken();
914     return true;
915   }
916   do {
917     if (FormatTok->is(tok::amp))
918       nextToken();
919     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
920       return false;
921     nextToken();
922     if (FormatTok->is(tok::ellipsis))
923       nextToken();
924     if (FormatTok->is(tok::comma)) {
925       nextToken();
926     } else if (FormatTok->is(tok::r_square)) {
927       nextToken();
928       return true;
929     } else {
930       return false;
931     }
932   } while (!eof());
933   return false;
934 }
935 
936 void UnwrappedLineParser::tryToParseJSFunction() {
937   nextToken();
938 
939   // Consume function name.
940   if (FormatTok->is(tok::identifier))
941       nextToken();
942 
943   if (FormatTok->isNot(tok::l_paren))
944     return;
945   nextToken();
946   while (FormatTok->isNot(tok::l_brace)) {
947     // Err on the side of caution in order to avoid consuming the full file in
948     // case of incomplete code.
949     if (!FormatTok->isOneOf(tok::identifier, tok::comma, tok::r_paren,
950                             tok::comment))
951       return;
952     nextToken();
953   }
954   parseChildBlock();
955 }
956 
957 bool UnwrappedLineParser::tryToParseBracedList() {
958   if (FormatTok->BlockKind == BK_Unknown)
959     calculateBraceTypes();
960   assert(FormatTok->BlockKind != BK_Unknown);
961   if (FormatTok->BlockKind == BK_Block)
962     return false;
963   parseBracedList();
964   return true;
965 }
966 
967 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
968   bool HasError = false;
969   nextToken();
970 
971   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
972   // replace this by using parseAssigmentExpression() inside.
973   do {
974     if (Style.Language == FormatStyle::LK_JavaScript &&
975         FormatTok->is(Keywords.kw_function)) {
976       tryToParseJSFunction();
977       continue;
978     }
979     switch (FormatTok->Tok.getKind()) {
980     case tok::caret:
981       nextToken();
982       if (FormatTok->is(tok::l_brace)) {
983         parseChildBlock();
984       }
985       break;
986     case tok::l_square:
987       tryToParseLambda();
988       break;
989     case tok::l_brace:
990       // Assume there are no blocks inside a braced init list apart
991       // from the ones we explicitly parse out (like lambdas).
992       FormatTok->BlockKind = BK_BracedInit;
993       parseBracedList();
994       break;
995     case tok::r_brace:
996       nextToken();
997       return !HasError;
998     case tok::semi:
999       HasError = true;
1000       if (!ContinueOnSemicolons)
1001         return !HasError;
1002       nextToken();
1003       break;
1004     case tok::comma:
1005       nextToken();
1006       break;
1007     default:
1008       nextToken();
1009       break;
1010     }
1011   } while (!eof());
1012   return false;
1013 }
1014 
1015 void UnwrappedLineParser::parseParens() {
1016   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
1017   nextToken();
1018   do {
1019     switch (FormatTok->Tok.getKind()) {
1020     case tok::l_paren:
1021       parseParens();
1022       break;
1023     case tok::r_paren:
1024       nextToken();
1025       return;
1026     case tok::r_brace:
1027       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1028       return;
1029     case tok::l_square:
1030       tryToParseLambda();
1031       break;
1032     case tok::l_brace: {
1033       if (!tryToParseBracedList()) {
1034         parseChildBlock();
1035       }
1036       break;
1037     }
1038     case tok::at:
1039       nextToken();
1040       if (FormatTok->Tok.is(tok::l_brace))
1041         parseBracedList();
1042       break;
1043     case tok::identifier:
1044       if (Style.Language == FormatStyle::LK_JavaScript &&
1045           FormatTok->is(Keywords.kw_function))
1046         tryToParseJSFunction();
1047       else
1048         nextToken();
1049       break;
1050     default:
1051       nextToken();
1052       break;
1053     }
1054   } while (!eof());
1055 }
1056 
1057 void UnwrappedLineParser::parseSquare() {
1058   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
1059   if (tryToParseLambda())
1060     return;
1061   do {
1062     switch (FormatTok->Tok.getKind()) {
1063     case tok::l_paren:
1064       parseParens();
1065       break;
1066     case tok::r_square:
1067       nextToken();
1068       return;
1069     case tok::r_brace:
1070       // A "}" inside parenthesis is an error if there wasn't a matching "{".
1071       return;
1072     case tok::l_square:
1073       parseSquare();
1074       break;
1075     case tok::l_brace: {
1076       if (!tryToParseBracedList()) {
1077         parseChildBlock();
1078       }
1079       break;
1080     }
1081     case tok::at:
1082       nextToken();
1083       if (FormatTok->Tok.is(tok::l_brace))
1084         parseBracedList();
1085       break;
1086     default:
1087       nextToken();
1088       break;
1089     }
1090   } while (!eof());
1091 }
1092 
1093 void UnwrappedLineParser::parseIfThenElse() {
1094   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1095   nextToken();
1096   if (FormatTok->Tok.is(tok::l_paren))
1097     parseParens();
1098   bool NeedsUnwrappedLine = false;
1099   if (FormatTok->Tok.is(tok::l_brace)) {
1100     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1101     parseBlock(/*MustBeDeclaration=*/false);
1102     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1103         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1104       addUnwrappedLine();
1105     } else {
1106       NeedsUnwrappedLine = true;
1107     }
1108   } else {
1109     addUnwrappedLine();
1110     ++Line->Level;
1111     parseStructuralElement();
1112     --Line->Level;
1113   }
1114   if (FormatTok->Tok.is(tok::kw_else)) {
1115     if (Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup)
1116       addUnwrappedLine();
1117     nextToken();
1118     if (FormatTok->Tok.is(tok::l_brace)) {
1119       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1120       parseBlock(/*MustBeDeclaration=*/false);
1121       addUnwrappedLine();
1122     } else if (FormatTok->Tok.is(tok::kw_if)) {
1123       parseIfThenElse();
1124     } else {
1125       addUnwrappedLine();
1126       ++Line->Level;
1127       parseStructuralElement();
1128       --Line->Level;
1129     }
1130   } else if (NeedsUnwrappedLine) {
1131     addUnwrappedLine();
1132   }
1133 }
1134 
1135 void UnwrappedLineParser::parseTryCatch() {
1136   assert(FormatTok->is(tok::kw_try) && "'try' expected");
1137   nextToken();
1138   bool NeedsUnwrappedLine = false;
1139   if (FormatTok->is(tok::colon)) {
1140     // We are in a function try block, what comes is an initializer list.
1141     nextToken();
1142     while (FormatTok->is(tok::identifier)) {
1143       nextToken();
1144       if (FormatTok->is(tok::l_paren))
1145         parseParens();
1146       else
1147         StructuralError = true;
1148       if (FormatTok->is(tok::comma))
1149         nextToken();
1150     }
1151   }
1152   if (FormatTok->is(tok::l_brace)) {
1153     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1154     parseBlock(/*MustBeDeclaration=*/false);
1155     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1156         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1157         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1158       addUnwrappedLine();
1159     } else {
1160       NeedsUnwrappedLine = true;
1161     }
1162   } else if (!FormatTok->is(tok::kw_catch)) {
1163     // The C++ standard requires a compound-statement after a try.
1164     // If there's none, we try to assume there's a structuralElement
1165     // and try to continue.
1166     StructuralError = true;
1167     addUnwrappedLine();
1168     ++Line->Level;
1169     parseStructuralElement();
1170     --Line->Level;
1171   }
1172   while (FormatTok->is(tok::kw_catch) ||
1173          ((Style.Language == FormatStyle::LK_Java ||
1174            Style.Language == FormatStyle::LK_JavaScript) &&
1175           FormatTok->is(Keywords.kw_finally))) {
1176     nextToken();
1177     while (FormatTok->isNot(tok::l_brace)) {
1178       if (FormatTok->is(tok::l_paren)) {
1179         parseParens();
1180         continue;
1181       }
1182       if (FormatTok->isOneOf(tok::semi, tok::r_brace))
1183         return;
1184       nextToken();
1185     }
1186     NeedsUnwrappedLine = false;
1187     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1188     parseBlock(/*MustBeDeclaration=*/false);
1189     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1190         Style.BreakBeforeBraces == FormatStyle::BS_GNU ||
1191         Style.BreakBeforeBraces == FormatStyle::BS_Stroustrup) {
1192       addUnwrappedLine();
1193     } else {
1194       NeedsUnwrappedLine = true;
1195     }
1196   }
1197   if (NeedsUnwrappedLine) {
1198     addUnwrappedLine();
1199   }
1200 }
1201 
1202 void UnwrappedLineParser::parseNamespace() {
1203   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1204 
1205   const FormatToken &InitialToken = *FormatTok;
1206   nextToken();
1207   if (FormatTok->Tok.is(tok::identifier))
1208     nextToken();
1209   if (FormatTok->Tok.is(tok::l_brace)) {
1210     if (ShouldBreakBeforeBrace(Style, InitialToken))
1211       addUnwrappedLine();
1212 
1213     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1214                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1215                      DeclarationScopeStack.size() > 1);
1216     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1217     // Munch the semicolon after a namespace. This is more common than one would
1218     // think. Puttin the semicolon into its own line is very ugly.
1219     if (FormatTok->Tok.is(tok::semi))
1220       nextToken();
1221     addUnwrappedLine();
1222   }
1223   // FIXME: Add error handling.
1224 }
1225 
1226 void UnwrappedLineParser::parseForOrWhileLoop() {
1227   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1228           FormatTok->IsForEachMacro) &&
1229          "'for', 'while' or foreach macro expected");
1230   nextToken();
1231   if (FormatTok->Tok.is(tok::l_paren))
1232     parseParens();
1233   if (FormatTok->Tok.is(tok::l_brace)) {
1234     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1235     parseBlock(/*MustBeDeclaration=*/false);
1236     addUnwrappedLine();
1237   } else {
1238     addUnwrappedLine();
1239     ++Line->Level;
1240     parseStructuralElement();
1241     --Line->Level;
1242   }
1243 }
1244 
1245 void UnwrappedLineParser::parseDoWhile() {
1246   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1247   nextToken();
1248   if (FormatTok->Tok.is(tok::l_brace)) {
1249     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1250     parseBlock(/*MustBeDeclaration=*/false);
1251     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1252       addUnwrappedLine();
1253   } else {
1254     addUnwrappedLine();
1255     ++Line->Level;
1256     parseStructuralElement();
1257     --Line->Level;
1258   }
1259 
1260   // FIXME: Add error handling.
1261   if (!FormatTok->Tok.is(tok::kw_while)) {
1262     addUnwrappedLine();
1263     return;
1264   }
1265 
1266   nextToken();
1267   parseStructuralElement();
1268 }
1269 
1270 void UnwrappedLineParser::parseLabel() {
1271   nextToken();
1272   unsigned OldLineLevel = Line->Level;
1273   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1274     --Line->Level;
1275   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1276     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1277     parseBlock(/*MustBeDeclaration=*/false);
1278     if (FormatTok->Tok.is(tok::kw_break)) {
1279       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1280       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1281           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1282         addUnwrappedLine();
1283       }
1284       parseStructuralElement();
1285     }
1286     addUnwrappedLine();
1287   } else {
1288     addUnwrappedLine();
1289   }
1290   Line->Level = OldLineLevel;
1291 }
1292 
1293 void UnwrappedLineParser::parseCaseLabel() {
1294   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1295   // FIXME: fix handling of complex expressions here.
1296   do {
1297     nextToken();
1298   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1299   parseLabel();
1300 }
1301 
1302 void UnwrappedLineParser::parseSwitch() {
1303   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1304   nextToken();
1305   if (FormatTok->Tok.is(tok::l_paren))
1306     parseParens();
1307   if (FormatTok->Tok.is(tok::l_brace)) {
1308     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1309     parseBlock(/*MustBeDeclaration=*/false);
1310     addUnwrappedLine();
1311   } else {
1312     addUnwrappedLine();
1313     ++Line->Level;
1314     parseStructuralElement();
1315     --Line->Level;
1316   }
1317 }
1318 
1319 void UnwrappedLineParser::parseAccessSpecifier() {
1320   nextToken();
1321   // Understand Qt's slots.
1322   if (FormatTok->is(tok::identifier) &&
1323       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1324     nextToken();
1325   // Otherwise, we don't know what it is, and we'd better keep the next token.
1326   if (FormatTok->Tok.is(tok::colon))
1327     nextToken();
1328   addUnwrappedLine();
1329 }
1330 
1331 void UnwrappedLineParser::parseEnum() {
1332   // Won't be 'enum' for NS_ENUMs.
1333   if (FormatTok->Tok.is(tok::kw_enum))
1334     nextToken();
1335 
1336   // Eat up enum class ...
1337   if (FormatTok->Tok.is(tok::kw_class) || FormatTok->Tok.is(tok::kw_struct))
1338     nextToken();
1339   while (FormatTok->Tok.getIdentifierInfo() ||
1340          FormatTok->isOneOf(tok::colon, tok::coloncolon, tok::less,
1341                             tok::greater, tok::comma, tok::question)) {
1342     nextToken();
1343     // We can have macros or attributes in between 'enum' and the enum name.
1344     if (FormatTok->is(tok::l_paren))
1345       parseParens();
1346     if (FormatTok->is(tok::identifier))
1347       nextToken();
1348   }
1349 
1350   // Just a declaration or something is wrong.
1351   if (FormatTok->isNot(tok::l_brace))
1352     return;
1353   FormatTok->BlockKind = BK_Block;
1354 
1355   if (Style.Language == FormatStyle::LK_Java) {
1356     // Java enums are different.
1357     parseJavaEnumBody();
1358     return;
1359   }
1360 
1361   // Parse enum body.
1362   bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1363   if (HasError) {
1364     if (FormatTok->is(tok::semi))
1365       nextToken();
1366     addUnwrappedLine();
1367   }
1368 
1369   // We fall through to parsing a structural element afterwards, so that in
1370   // enum A {} n, m;
1371   // "} n, m;" will end up in one unwrapped line.
1372 }
1373 
1374 void UnwrappedLineParser::parseJavaEnumBody() {
1375   // Determine whether the enum is simple, i.e. does not have a semicolon or
1376   // constants with class bodies. Simple enums can be formatted like braced
1377   // lists, contracted to a single line, etc.
1378   unsigned StoredPosition = Tokens->getPosition();
1379   bool IsSimple = true;
1380   FormatToken *Tok = Tokens->getNextToken();
1381   while (Tok) {
1382     if (Tok->is(tok::r_brace))
1383       break;
1384     if (Tok->isOneOf(tok::l_brace, tok::semi)) {
1385       IsSimple = false;
1386       break;
1387     }
1388     // FIXME: This will also mark enums with braces in the arguments to enum
1389     // constants as "not simple". This is probably fine in practice, though.
1390     Tok = Tokens->getNextToken();
1391   }
1392   FormatTok = Tokens->setPosition(StoredPosition);
1393 
1394   if (IsSimple) {
1395     parseBracedList();
1396     addUnwrappedLine();
1397     return;
1398   }
1399 
1400   // Parse the body of a more complex enum.
1401   // First add a line for everything up to the "{".
1402   nextToken();
1403   addUnwrappedLine();
1404   ++Line->Level;
1405 
1406   // Parse the enum constants.
1407   while (FormatTok) {
1408     if (FormatTok->is(tok::l_brace)) {
1409       // Parse the constant's class body.
1410       parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1411                  /*MunchSemi=*/false);
1412     } else if (FormatTok->is(tok::l_paren)) {
1413       parseParens();
1414     } else if (FormatTok->is(tok::comma)) {
1415       nextToken();
1416       addUnwrappedLine();
1417     } else if (FormatTok->is(tok::semi)) {
1418       nextToken();
1419       addUnwrappedLine();
1420       break;
1421     } else if (FormatTok->is(tok::r_brace)) {
1422       addUnwrappedLine();
1423       break;
1424     } else {
1425       nextToken();
1426     }
1427   }
1428 
1429   // Parse the class body after the enum's ";" if any.
1430   parseLevel(/*HasOpeningBrace=*/true);
1431   nextToken();
1432   --Line->Level;
1433   addUnwrappedLine();
1434 }
1435 
1436 void UnwrappedLineParser::parseRecord() {
1437   const FormatToken &InitialToken = *FormatTok;
1438   nextToken();
1439   if (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::kw___attribute,
1440                          tok::kw___declspec, tok::kw_alignas)) {
1441     nextToken();
1442     // We can have macros or attributes in between 'class' and the class name.
1443     if (FormatTok->Tok.is(tok::l_paren)) {
1444       parseParens();
1445     }
1446     // The actual identifier can be a nested name specifier, and in macros
1447     // it is often token-pasted.
1448     while (FormatTok->is(tok::identifier) || FormatTok->is(tok::coloncolon) ||
1449            FormatTok->is(tok::hashhash) ||
1450            (Style.Language == FormatStyle::LK_Java &&
1451             FormatTok->isOneOf(tok::period, tok::comma)))
1452       nextToken();
1453 
1454     // Note that parsing away template declarations here leads to incorrectly
1455     // accepting function declarations as record declarations.
1456     // In general, we cannot solve this problem. Consider:
1457     // class A<int> B() {}
1458     // which can be a function definition or a class definition when B() is a
1459     // macro. If we find enough real-world cases where this is a problem, we
1460     // can parse for the 'template' keyword in the beginning of the statement,
1461     // and thus rule out the record production in case there is no template
1462     // (this would still leave us with an ambiguity between template function
1463     // and class declarations).
1464     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1465       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1466         if (FormatTok->Tok.is(tok::semi))
1467           return;
1468         nextToken();
1469       }
1470     }
1471   }
1472   if (FormatTok->Tok.is(tok::l_brace)) {
1473     if (ShouldBreakBeforeBrace(Style, InitialToken))
1474       addUnwrappedLine();
1475 
1476     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1477                /*MunchSemi=*/false);
1478   }
1479   // We fall through to parsing a structural element afterwards, so
1480   // class A {} n, m;
1481   // will end up in one unwrapped line.
1482   // This does not apply for Java.
1483   if (Style.Language == FormatStyle::LK_Java)
1484     addUnwrappedLine();
1485 }
1486 
1487 void UnwrappedLineParser::parseObjCProtocolList() {
1488   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1489   do
1490     nextToken();
1491   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1492   nextToken(); // Skip '>'.
1493 }
1494 
1495 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1496   do {
1497     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1498       nextToken();
1499       addUnwrappedLine();
1500       break;
1501     }
1502     if (FormatTok->is(tok::l_brace)) {
1503       parseBlock(/*MustBeDeclaration=*/false);
1504       // In ObjC interfaces, nothing should be following the "}".
1505       addUnwrappedLine();
1506     } else if (FormatTok->is(tok::r_brace)) {
1507       // Ignore stray "}". parseStructuralElement doesn't consume them.
1508       nextToken();
1509       addUnwrappedLine();
1510     } else {
1511       parseStructuralElement();
1512     }
1513   } while (!eof());
1514 }
1515 
1516 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1517   nextToken();
1518   nextToken(); // interface name
1519 
1520   // @interface can be followed by either a base class, or a category.
1521   if (FormatTok->Tok.is(tok::colon)) {
1522     nextToken();
1523     nextToken(); // base class name
1524   } else if (FormatTok->Tok.is(tok::l_paren))
1525     // Skip category, if present.
1526     parseParens();
1527 
1528   if (FormatTok->Tok.is(tok::less))
1529     parseObjCProtocolList();
1530 
1531   if (FormatTok->Tok.is(tok::l_brace)) {
1532     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1533         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1534       addUnwrappedLine();
1535     parseBlock(/*MustBeDeclaration=*/true);
1536   }
1537 
1538   // With instance variables, this puts '}' on its own line.  Without instance
1539   // variables, this ends the @interface line.
1540   addUnwrappedLine();
1541 
1542   parseObjCUntilAtEnd();
1543 }
1544 
1545 void UnwrappedLineParser::parseObjCProtocol() {
1546   nextToken();
1547   nextToken(); // protocol name
1548 
1549   if (FormatTok->Tok.is(tok::less))
1550     parseObjCProtocolList();
1551 
1552   // Check for protocol declaration.
1553   if (FormatTok->Tok.is(tok::semi)) {
1554     nextToken();
1555     return addUnwrappedLine();
1556   }
1557 
1558   addUnwrappedLine();
1559   parseObjCUntilAtEnd();
1560 }
1561 
1562 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1563                                                  StringRef Prefix = "") {
1564   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1565                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1566   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1567                                                     E = Line.Tokens.end();
1568        I != E; ++I) {
1569     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1570   }
1571   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1572                                                     E = Line.Tokens.end();
1573        I != E; ++I) {
1574     const UnwrappedLineNode &Node = *I;
1575     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1576              I = Node.Children.begin(),
1577              E = Node.Children.end();
1578          I != E; ++I) {
1579       printDebugInfo(*I, "\nChild: ");
1580     }
1581   }
1582   llvm::dbgs() << "\n";
1583 }
1584 
1585 void UnwrappedLineParser::addUnwrappedLine() {
1586   if (Line->Tokens.empty())
1587     return;
1588   DEBUG({
1589     if (CurrentLines == &Lines)
1590       printDebugInfo(*Line);
1591   });
1592   CurrentLines->push_back(*Line);
1593   Line->Tokens.clear();
1594   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1595     for (SmallVectorImpl<UnwrappedLine>::iterator
1596              I = PreprocessorDirectives.begin(),
1597              E = PreprocessorDirectives.end();
1598          I != E; ++I) {
1599       CurrentLines->push_back(*I);
1600     }
1601     PreprocessorDirectives.clear();
1602   }
1603 }
1604 
1605 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1606 
1607 bool UnwrappedLineParser::isOnNewLine(const FormatToken &FormatTok) {
1608   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1609          FormatTok.NewlinesBefore > 0;
1610 }
1611 
1612 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1613   bool JustComments = Line->Tokens.empty();
1614   for (SmallVectorImpl<FormatToken *>::const_iterator
1615            I = CommentsBeforeNextToken.begin(),
1616            E = CommentsBeforeNextToken.end();
1617        I != E; ++I) {
1618     if (isOnNewLine(**I) && JustComments) {
1619       addUnwrappedLine();
1620     }
1621     pushToken(*I);
1622   }
1623   if (NewlineBeforeNext && JustComments) {
1624     addUnwrappedLine();
1625   }
1626   CommentsBeforeNextToken.clear();
1627 }
1628 
1629 void UnwrappedLineParser::nextToken() {
1630   if (eof())
1631     return;
1632   flushComments(isOnNewLine(*FormatTok));
1633   pushToken(FormatTok);
1634   readToken();
1635 }
1636 
1637 void UnwrappedLineParser::readToken() {
1638   bool CommentsInCurrentLine = true;
1639   do {
1640     FormatTok = Tokens->getNextToken();
1641     assert(FormatTok);
1642     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1643            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1644       // If there is an unfinished unwrapped line, we flush the preprocessor
1645       // directives only after that unwrapped line was finished later.
1646       bool SwitchToPreprocessorLines =
1647           !Line->Tokens.empty() && CurrentLines == &Lines;
1648       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1649       // Comments stored before the preprocessor directive need to be output
1650       // before the preprocessor directive, at the same level as the
1651       // preprocessor directive, as we consider them to apply to the directive.
1652       flushComments(isOnNewLine(*FormatTok));
1653       parsePPDirective();
1654     }
1655     while (FormatTok->Type == TT_ConflictStart ||
1656            FormatTok->Type == TT_ConflictEnd ||
1657            FormatTok->Type == TT_ConflictAlternative) {
1658       if (FormatTok->Type == TT_ConflictStart) {
1659         conditionalCompilationStart(/*Unreachable=*/false);
1660       } else if (FormatTok->Type == TT_ConflictAlternative) {
1661         conditionalCompilationAlternative();
1662       } else if (FormatTok->Type == TT_ConflictEnd) {
1663         conditionalCompilationEnd();
1664       }
1665       FormatTok = Tokens->getNextToken();
1666       FormatTok->MustBreakBefore = true;
1667     }
1668 
1669     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1670         !Line->InPPDirective) {
1671       continue;
1672     }
1673 
1674     if (!FormatTok->Tok.is(tok::comment))
1675       return;
1676     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
1677       CommentsInCurrentLine = false;
1678     }
1679     if (CommentsInCurrentLine) {
1680       pushToken(FormatTok);
1681     } else {
1682       CommentsBeforeNextToken.push_back(FormatTok);
1683     }
1684   } while (!eof());
1685 }
1686 
1687 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1688   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1689   if (MustBreakBeforeNextToken) {
1690     Line->Tokens.back().Tok->MustBreakBefore = true;
1691     MustBreakBeforeNextToken = false;
1692   }
1693 }
1694 
1695 } // end namespace format
1696 } // end namespace clang
1697