1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "llvm/Support/Debug.h"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   virtual FormatToken *getNextToken() {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   virtual unsigned getPosition() { return PreviousTokenSource->getPosition(); }
88 
89   virtual FormatToken *setPosition(unsigned Position) {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.take();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   virtual FormatToken *getNextToken() {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   virtual unsigned getPosition() {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   virtual FormatToken *setPosition(unsigned P) {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           // If there is a comma, semicolon or right paren after the closing
339           // brace, we assume this is a braced initializer list.  Note that
340           // regardless how we mark inner braces here, we will overwrite the
341           // BlockKind later if we parse a braced list (where all blocks inside
342           // are by default braced lists), or when we explicitly detect blocks
343           // (for example while parsing lambdas).
344           //
345           // We exclude + and - as they can be ObjC visibility modifiers.
346           if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren, tok::period,
347                                tok::r_square, tok::l_brace, tok::colon) ||
348               (NextTok->isBinaryOperator() &&
349                !NextTok->isOneOf(tok::plus, tok::minus))) {
350             Tok->BlockKind = BK_BracedInit;
351             LBraceStack.back()->BlockKind = BK_BracedInit;
352           } else {
353             Tok->BlockKind = BK_Block;
354             LBraceStack.back()->BlockKind = BK_Block;
355           }
356         }
357         LBraceStack.pop_back();
358       }
359       break;
360     case tok::semi:
361     case tok::kw_if:
362     case tok::kw_while:
363     case tok::kw_for:
364     case tok::kw_switch:
365     case tok::kw_try:
366       if (!LBraceStack.empty())
367         LBraceStack.back()->BlockKind = BK_Block;
368       break;
369     default:
370       break;
371     }
372     Tok = NextTok;
373     Position += ReadTokens;
374   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
375   // Assume other blocks for all unclosed opening braces.
376   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
377     if (LBraceStack[i]->BlockKind == BK_Unknown)
378       LBraceStack[i]->BlockKind = BK_Block;
379   }
380 
381   FormatTok = Tokens->setPosition(StoredPosition);
382 }
383 
384 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
385                                      bool MunchSemi) {
386   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
387   unsigned InitialLevel = Line->Level;
388   nextToken();
389 
390   addUnwrappedLine();
391 
392   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
393                                           MustBeDeclaration);
394   if (AddLevel)
395     ++Line->Level;
396   parseLevel(/*HasOpeningBrace=*/true);
397 
398   if (!FormatTok->Tok.is(tok::r_brace)) {
399     Line->Level = InitialLevel;
400     StructuralError = true;
401     return;
402   }
403 
404   nextToken(); // Munch the closing brace.
405   if (MunchSemi && FormatTok->Tok.is(tok::semi))
406     nextToken();
407   Line->Level = InitialLevel;
408 }
409 
410 void UnwrappedLineParser::parseChildBlock() {
411   FormatTok->BlockKind = BK_Block;
412   nextToken();
413   {
414     ScopedLineState LineState(*this);
415     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
416                                             /*MustBeDeclaration=*/false);
417     Line->Level += 1;
418     parseLevel(/*HasOpeningBrace=*/true);
419     Line->Level -= 1;
420   }
421   nextToken();
422 }
423 
424 void UnwrappedLineParser::parsePPDirective() {
425   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
426   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
427   nextToken();
428 
429   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
430     parsePPUnknown();
431     return;
432   }
433 
434   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
435   case tok::pp_define:
436     parsePPDefine();
437     return;
438   case tok::pp_if:
439     parsePPIf(/*IfDef=*/false);
440     break;
441   case tok::pp_ifdef:
442   case tok::pp_ifndef:
443     parsePPIf(/*IfDef=*/true);
444     break;
445   case tok::pp_else:
446     parsePPElse();
447     break;
448   case tok::pp_elif:
449     parsePPElIf();
450     break;
451   case tok::pp_endif:
452     parsePPEndIf();
453     break;
454   default:
455     parsePPUnknown();
456     break;
457   }
458 }
459 
460 void UnwrappedLineParser::pushPPConditional() {
461   if (!PPStack.empty() && PPStack.back() == PP_Unreachable)
462     PPStack.push_back(PP_Unreachable);
463   else
464     PPStack.push_back(PP_Conditional);
465 }
466 
467 void UnwrappedLineParser::parsePPIf(bool IfDef) {
468   ++PPBranchLevel;
469   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
470   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
471     PPLevelBranchIndex.push_back(0);
472     PPLevelBranchCount.push_back(0);
473   }
474   PPChainBranchIndex.push(0);
475   nextToken();
476   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
477                          StringRef(FormatTok->Tok.getLiteralData(),
478                                    FormatTok->Tok.getLength()) == "0") ||
479                         FormatTok->Tok.is(tok::kw_false);
480   if ((!IfDef && IsLiteralFalse) || PPLevelBranchIndex[PPBranchLevel] > 0) {
481     PPStack.push_back(PP_Unreachable);
482   } else {
483     pushPPConditional();
484   }
485   parsePPUnknown();
486 }
487 
488 void UnwrappedLineParser::parsePPElse() {
489   if (!PPStack.empty())
490     PPStack.pop_back();
491   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
492   if (!PPChainBranchIndex.empty())
493     ++PPChainBranchIndex.top();
494   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
495       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top()) {
496     PPStack.push_back(PP_Unreachable);
497   } else {
498     pushPPConditional();
499   }
500   parsePPUnknown();
501 }
502 
503 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
504 
505 void UnwrappedLineParser::parsePPEndIf() {
506   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
507   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
508     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
509       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
510     }
511   }
512   // Guard against #endif's without #if.
513   if (PPBranchLevel > 0)
514     --PPBranchLevel;
515   if (!PPChainBranchIndex.empty())
516     PPChainBranchIndex.pop();
517   if (!PPStack.empty())
518     PPStack.pop_back();
519   parsePPUnknown();
520 }
521 
522 void UnwrappedLineParser::parsePPDefine() {
523   nextToken();
524 
525   if (FormatTok->Tok.getKind() != tok::identifier) {
526     parsePPUnknown();
527     return;
528   }
529   nextToken();
530   if (FormatTok->Tok.getKind() == tok::l_paren &&
531       FormatTok->WhitespaceRange.getBegin() ==
532           FormatTok->WhitespaceRange.getEnd()) {
533     parseParens();
534   }
535   addUnwrappedLine();
536   Line->Level = 1;
537 
538   // Errors during a preprocessor directive can only affect the layout of the
539   // preprocessor directive, and thus we ignore them. An alternative approach
540   // would be to use the same approach we use on the file level (no
541   // re-indentation if there was a structural error) within the macro
542   // definition.
543   parseFile();
544 }
545 
546 void UnwrappedLineParser::parsePPUnknown() {
547   do {
548     nextToken();
549   } while (!eof());
550   addUnwrappedLine();
551 }
552 
553 // Here we blacklist certain tokens that are not usually the first token in an
554 // unwrapped line. This is used in attempt to distinguish macro calls without
555 // trailing semicolons from other constructs split to several lines.
556 bool tokenCanStartNewLine(clang::Token Tok) {
557   // Semicolon can be a null-statement, l_square can be a start of a macro or
558   // a C++11 attribute, but this doesn't seem to be common.
559   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
560          Tok.isNot(tok::l_square) &&
561          // Tokens that can only be used as binary operators and a part of
562          // overloaded operator names.
563          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
564          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
565          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
566          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
567          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
568          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
569          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
570          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
571          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
572          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
573          Tok.isNot(tok::lesslessequal) &&
574          // Colon is used in labels, base class lists, initializer lists,
575          // range-based for loops, ternary operator, but should never be the
576          // first token in an unwrapped line.
577          Tok.isNot(tok::colon);
578 }
579 
580 void UnwrappedLineParser::parseStructuralElement() {
581   assert(!FormatTok->Tok.is(tok::l_brace));
582   switch (FormatTok->Tok.getKind()) {
583   case tok::at:
584     nextToken();
585     if (FormatTok->Tok.is(tok::l_brace)) {
586       parseBracedList();
587       break;
588     }
589     switch (FormatTok->Tok.getObjCKeywordID()) {
590     case tok::objc_public:
591     case tok::objc_protected:
592     case tok::objc_package:
593     case tok::objc_private:
594       return parseAccessSpecifier();
595     case tok::objc_interface:
596     case tok::objc_implementation:
597       return parseObjCInterfaceOrImplementation();
598     case tok::objc_protocol:
599       return parseObjCProtocol();
600     case tok::objc_end:
601       return; // Handled by the caller.
602     case tok::objc_optional:
603     case tok::objc_required:
604       nextToken();
605       addUnwrappedLine();
606       return;
607     default:
608       break;
609     }
610     break;
611   case tok::kw_namespace:
612     parseNamespace();
613     return;
614   case tok::kw_inline:
615     nextToken();
616     if (FormatTok->Tok.is(tok::kw_namespace)) {
617       parseNamespace();
618       return;
619     }
620     break;
621   case tok::kw_public:
622   case tok::kw_protected:
623   case tok::kw_private:
624     parseAccessSpecifier();
625     return;
626   case tok::kw_if:
627     parseIfThenElse();
628     return;
629   case tok::kw_for:
630   case tok::kw_while:
631     parseForOrWhileLoop();
632     return;
633   case tok::kw_do:
634     parseDoWhile();
635     return;
636   case tok::kw_switch:
637     parseSwitch();
638     return;
639   case tok::kw_default:
640     nextToken();
641     parseLabel();
642     return;
643   case tok::kw_case:
644     parseCaseLabel();
645     return;
646   case tok::kw_extern:
647     nextToken();
648     if (FormatTok->Tok.is(tok::string_literal)) {
649       nextToken();
650       if (FormatTok->Tok.is(tok::l_brace)) {
651         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
652         addUnwrappedLine();
653         return;
654       }
655     }
656     // In all other cases, parse the declaration.
657     break;
658   default:
659     break;
660   }
661   do {
662     switch (FormatTok->Tok.getKind()) {
663     case tok::at:
664       nextToken();
665       if (FormatTok->Tok.is(tok::l_brace))
666         parseBracedList();
667       break;
668     case tok::kw_enum:
669       parseEnum();
670       break;
671     case tok::kw_struct:
672     case tok::kw_union:
673     case tok::kw_class:
674       parseRecord();
675       // A record declaration or definition is always the start of a structural
676       // element.
677       break;
678     case tok::semi:
679       nextToken();
680       addUnwrappedLine();
681       return;
682     case tok::r_brace:
683       addUnwrappedLine();
684       return;
685     case tok::l_paren:
686       parseParens();
687       break;
688     case tok::caret:
689       nextToken();
690       if (FormatTok->is(tok::l_brace)) {
691         parseChildBlock();
692       }
693       break;
694     case tok::l_brace:
695       if (!tryToParseBracedList()) {
696         // A block outside of parentheses must be the last part of a
697         // structural element.
698         // FIXME: Figure out cases where this is not true, and add projections
699         // for them (the one we know is missing are lambdas).
700         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
701           addUnwrappedLine();
702         FormatTok->Type = TT_FunctionLBrace;
703         parseBlock(/*MustBeDeclaration=*/false);
704         addUnwrappedLine();
705         return;
706       }
707       // Otherwise this was a braced init list, and the structural
708       // element continues.
709       break;
710     case tok::identifier: {
711       StringRef Text = FormatTok->TokenText;
712       nextToken();
713       if (Line->Tokens.size() == 1) {
714         if (FormatTok->Tok.is(tok::colon)) {
715           parseLabel();
716           return;
717         }
718         // Recognize function-like macro usages without trailing semicolon.
719         if (FormatTok->Tok.is(tok::l_paren)) {
720           parseParens();
721           if (FormatTok->NewlinesBefore > 0 &&
722               tokenCanStartNewLine(FormatTok->Tok)) {
723             addUnwrappedLine();
724             return;
725           }
726         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
727                    Text == Text.upper()) {
728           // Recognize free-standing macros like Q_OBJECT.
729           addUnwrappedLine();
730           return;
731         }
732       }
733       break;
734     }
735     case tok::equal:
736       nextToken();
737       if (FormatTok->Tok.is(tok::l_brace)) {
738         parseBracedList();
739       }
740       break;
741     case tok::l_square:
742       parseSquare();
743       break;
744     default:
745       nextToken();
746       break;
747     }
748   } while (!eof());
749 }
750 
751 bool UnwrappedLineParser::tryToParseLambda() {
752   // FIXME: This is a dirty way to access the previous token. Find a better
753   // solution.
754   if (!Line->Tokens.empty() &&
755       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator) ||
756        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
757     nextToken();
758     return false;
759   }
760   assert(FormatTok->is(tok::l_square));
761   FormatToken &LSquare = *FormatTok;
762   if (!tryToParseLambdaIntroducer())
763     return false;
764 
765   while (FormatTok && FormatTok->isNot(tok::l_brace)) {
766     if (FormatTok->isSimpleTypeSpecifier()) {
767       nextToken();
768       continue;
769     }
770     switch (FormatTok->Tok.getKind()) {
771     case tok::l_brace:
772       break;
773     case tok::l_paren:
774       parseParens();
775       break;
776     case tok::less:
777     case tok::greater:
778     case tok::identifier:
779     case tok::kw_mutable:
780     case tok::arrow:
781       nextToken();
782       break;
783     default:
784       return true;
785     }
786   }
787   LSquare.Type = TT_LambdaLSquare;
788   parseChildBlock();
789   return true;
790 }
791 
792 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
793   nextToken();
794   if (FormatTok->is(tok::equal)) {
795     nextToken();
796     if (FormatTok->is(tok::r_square)) {
797       nextToken();
798       return true;
799     }
800     if (FormatTok->isNot(tok::comma))
801       return false;
802     nextToken();
803   } else if (FormatTok->is(tok::amp)) {
804     nextToken();
805     if (FormatTok->is(tok::r_square)) {
806       nextToken();
807       return true;
808     }
809     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
810       return false;
811     }
812     if (FormatTok->is(tok::comma))
813       nextToken();
814   } else if (FormatTok->is(tok::r_square)) {
815     nextToken();
816     return true;
817   }
818   do {
819     if (FormatTok->is(tok::amp))
820       nextToken();
821     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
822       return false;
823     nextToken();
824     if (FormatTok->is(tok::comma)) {
825       nextToken();
826     } else if (FormatTok->is(tok::r_square)) {
827       nextToken();
828       return true;
829     } else {
830       return false;
831     }
832   } while (!eof());
833   return false;
834 }
835 
836 bool UnwrappedLineParser::tryToParseBracedList() {
837   if (FormatTok->BlockKind == BK_Unknown)
838     calculateBraceTypes();
839   assert(FormatTok->BlockKind != BK_Unknown);
840   if (FormatTok->BlockKind == BK_Block)
841     return false;
842   parseBracedList();
843   return true;
844 }
845 
846 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
847   bool HasError = false;
848   nextToken();
849 
850   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
851   // replace this by using parseAssigmentExpression() inside.
852   do {
853     // FIXME: When we start to support lambdas, we'll want to parse them away
854     // here, otherwise our bail-out scenarios below break. The better solution
855     // might be to just implement a more or less complete expression parser.
856     switch (FormatTok->Tok.getKind()) {
857     case tok::caret:
858       nextToken();
859       if (FormatTok->is(tok::l_brace)) {
860         parseChildBlock();
861       }
862       break;
863     case tok::l_square:
864       tryToParseLambda();
865       break;
866     case tok::l_brace:
867       // Assume there are no blocks inside a braced init list apart
868       // from the ones we explicitly parse out (like lambdas).
869       FormatTok->BlockKind = BK_BracedInit;
870       parseBracedList();
871       break;
872     case tok::r_brace:
873       nextToken();
874       return !HasError;
875     case tok::semi:
876       HasError = true;
877       if (!ContinueOnSemicolons)
878         return !HasError;
879       nextToken();
880       break;
881     case tok::comma:
882       nextToken();
883       break;
884     default:
885       nextToken();
886       break;
887     }
888   } while (!eof());
889   return false;
890 }
891 
892 void UnwrappedLineParser::parseParens() {
893   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
894   nextToken();
895   do {
896     switch (FormatTok->Tok.getKind()) {
897     case tok::l_paren:
898       parseParens();
899       break;
900     case tok::r_paren:
901       nextToken();
902       return;
903     case tok::r_brace:
904       // A "}" inside parenthesis is an error if there wasn't a matching "{".
905       return;
906     case tok::l_square:
907       tryToParseLambda();
908       break;
909     case tok::l_brace: {
910       if (!tryToParseBracedList()) {
911         parseChildBlock();
912       }
913       break;
914     }
915     case tok::at:
916       nextToken();
917       if (FormatTok->Tok.is(tok::l_brace))
918         parseBracedList();
919       break;
920     default:
921       nextToken();
922       break;
923     }
924   } while (!eof());
925 }
926 
927 void UnwrappedLineParser::parseSquare() {
928   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
929   if (tryToParseLambda())
930     return;
931   do {
932     switch (FormatTok->Tok.getKind()) {
933     case tok::l_paren:
934       parseParens();
935       break;
936     case tok::r_square:
937       nextToken();
938       return;
939     case tok::r_brace:
940       // A "}" inside parenthesis is an error if there wasn't a matching "{".
941       return;
942     case tok::l_square:
943       parseSquare();
944       break;
945     case tok::l_brace: {
946       if (!tryToParseBracedList()) {
947         parseChildBlock();
948       }
949       break;
950     }
951     case tok::at:
952       nextToken();
953       if (FormatTok->Tok.is(tok::l_brace))
954         parseBracedList();
955       break;
956     default:
957       nextToken();
958       break;
959     }
960   } while (!eof());
961 }
962 
963 void UnwrappedLineParser::parseIfThenElse() {
964   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
965   nextToken();
966   if (FormatTok->Tok.is(tok::l_paren))
967     parseParens();
968   bool NeedsUnwrappedLine = false;
969   if (FormatTok->Tok.is(tok::l_brace)) {
970     CompoundStatementIndenter Indenter(this, Style, Line->Level);
971     parseBlock(/*MustBeDeclaration=*/false);
972     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
973         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
974       addUnwrappedLine();
975     } else {
976       NeedsUnwrappedLine = true;
977     }
978   } else {
979     addUnwrappedLine();
980     ++Line->Level;
981     parseStructuralElement();
982     --Line->Level;
983   }
984   if (FormatTok->Tok.is(tok::kw_else)) {
985     nextToken();
986     if (FormatTok->Tok.is(tok::l_brace)) {
987       CompoundStatementIndenter Indenter(this, Style, Line->Level);
988       parseBlock(/*MustBeDeclaration=*/false);
989       addUnwrappedLine();
990     } else if (FormatTok->Tok.is(tok::kw_if)) {
991       parseIfThenElse();
992     } else {
993       addUnwrappedLine();
994       ++Line->Level;
995       parseStructuralElement();
996       --Line->Level;
997     }
998   } else if (NeedsUnwrappedLine) {
999     addUnwrappedLine();
1000   }
1001 }
1002 
1003 void UnwrappedLineParser::parseNamespace() {
1004   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1005   nextToken();
1006   if (FormatTok->Tok.is(tok::identifier))
1007     nextToken();
1008   if (FormatTok->Tok.is(tok::l_brace)) {
1009     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1010         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1011         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1012       addUnwrappedLine();
1013 
1014     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1015                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1016                      DeclarationScopeStack.size() > 1);
1017     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1018     // Munch the semicolon after a namespace. This is more common than one would
1019     // think. Puttin the semicolon into its own line is very ugly.
1020     if (FormatTok->Tok.is(tok::semi))
1021       nextToken();
1022     addUnwrappedLine();
1023   }
1024   // FIXME: Add error handling.
1025 }
1026 
1027 void UnwrappedLineParser::parseForOrWhileLoop() {
1028   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while)) &&
1029          "'for' or 'while' expected");
1030   nextToken();
1031   if (FormatTok->Tok.is(tok::l_paren))
1032     parseParens();
1033   if (FormatTok->Tok.is(tok::l_brace)) {
1034     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1035     parseBlock(/*MustBeDeclaration=*/false);
1036     addUnwrappedLine();
1037   } else {
1038     addUnwrappedLine();
1039     ++Line->Level;
1040     parseStructuralElement();
1041     --Line->Level;
1042   }
1043 }
1044 
1045 void UnwrappedLineParser::parseDoWhile() {
1046   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1047   nextToken();
1048   if (FormatTok->Tok.is(tok::l_brace)) {
1049     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1050     parseBlock(/*MustBeDeclaration=*/false);
1051     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1052       addUnwrappedLine();
1053   } else {
1054     addUnwrappedLine();
1055     ++Line->Level;
1056     parseStructuralElement();
1057     --Line->Level;
1058   }
1059 
1060   // FIXME: Add error handling.
1061   if (!FormatTok->Tok.is(tok::kw_while)) {
1062     addUnwrappedLine();
1063     return;
1064   }
1065 
1066   nextToken();
1067   parseStructuralElement();
1068 }
1069 
1070 void UnwrappedLineParser::parseLabel() {
1071   nextToken();
1072   unsigned OldLineLevel = Line->Level;
1073   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1074     --Line->Level;
1075   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1076     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1077     parseBlock(/*MustBeDeclaration=*/false);
1078     if (FormatTok->Tok.is(tok::kw_break)) {
1079       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1080       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1081           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1082         addUnwrappedLine();
1083       }
1084       parseStructuralElement();
1085     }
1086     addUnwrappedLine();
1087   } else {
1088     addUnwrappedLine();
1089   }
1090   Line->Level = OldLineLevel;
1091 }
1092 
1093 void UnwrappedLineParser::parseCaseLabel() {
1094   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1095   // FIXME: fix handling of complex expressions here.
1096   do {
1097     nextToken();
1098   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1099   parseLabel();
1100 }
1101 
1102 void UnwrappedLineParser::parseSwitch() {
1103   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1104   nextToken();
1105   if (FormatTok->Tok.is(tok::l_paren))
1106     parseParens();
1107   if (FormatTok->Tok.is(tok::l_brace)) {
1108     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1109     parseBlock(/*MustBeDeclaration=*/false);
1110     addUnwrappedLine();
1111   } else {
1112     addUnwrappedLine();
1113     ++Line->Level;
1114     parseStructuralElement();
1115     --Line->Level;
1116   }
1117 }
1118 
1119 void UnwrappedLineParser::parseAccessSpecifier() {
1120   nextToken();
1121   // Understand Qt's slots.
1122   if (FormatTok->is(tok::identifier) &&
1123       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1124     nextToken();
1125   // Otherwise, we don't know what it is, and we'd better keep the next token.
1126   if (FormatTok->Tok.is(tok::colon))
1127     nextToken();
1128   addUnwrappedLine();
1129 }
1130 
1131 void UnwrappedLineParser::parseEnum() {
1132   nextToken();
1133   // Eat up enum class ...
1134   if (FormatTok->Tok.is(tok::kw_class) ||
1135       FormatTok->Tok.is(tok::kw_struct))
1136       nextToken();
1137   while (FormatTok->Tok.getIdentifierInfo() ||
1138          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1139     nextToken();
1140     // We can have macros or attributes in between 'enum' and the enum name.
1141     if (FormatTok->Tok.is(tok::l_paren)) {
1142       parseParens();
1143     }
1144     if (FormatTok->Tok.is(tok::identifier))
1145       nextToken();
1146   }
1147   if (FormatTok->Tok.is(tok::l_brace)) {
1148     FormatTok->BlockKind = BK_Block;
1149     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1150     if (HasError) {
1151       if (FormatTok->is(tok::semi))
1152         nextToken();
1153       addUnwrappedLine();
1154     }
1155   }
1156   // We fall through to parsing a structural element afterwards, so that in
1157   // enum A {} n, m;
1158   // "} n, m;" will end up in one unwrapped line.
1159 }
1160 
1161 void UnwrappedLineParser::parseRecord() {
1162   nextToken();
1163   if (FormatTok->Tok.is(tok::identifier) ||
1164       FormatTok->Tok.is(tok::kw___attribute) ||
1165       FormatTok->Tok.is(tok::kw___declspec) ||
1166       FormatTok->Tok.is(tok::kw_alignas)) {
1167     nextToken();
1168     // We can have macros or attributes in between 'class' and the class name.
1169     if (FormatTok->Tok.is(tok::l_paren)) {
1170       parseParens();
1171     }
1172     // The actual identifier can be a nested name specifier, and in macros
1173     // it is often token-pasted.
1174     while (FormatTok->Tok.is(tok::identifier) ||
1175            FormatTok->Tok.is(tok::coloncolon) ||
1176            FormatTok->Tok.is(tok::hashhash))
1177       nextToken();
1178 
1179     // Note that parsing away template declarations here leads to incorrectly
1180     // accepting function declarations as record declarations.
1181     // In general, we cannot solve this problem. Consider:
1182     // class A<int> B() {}
1183     // which can be a function definition or a class definition when B() is a
1184     // macro. If we find enough real-world cases where this is a problem, we
1185     // can parse for the 'template' keyword in the beginning of the statement,
1186     // and thus rule out the record production in case there is no template
1187     // (this would still leave us with an ambiguity between template function
1188     // and class declarations).
1189     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1190       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1191         if (FormatTok->Tok.is(tok::semi))
1192           return;
1193         nextToken();
1194       }
1195     }
1196   }
1197   if (FormatTok->Tok.is(tok::l_brace)) {
1198     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1199         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1200         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1201       addUnwrappedLine();
1202 
1203     parseBlock(/*MustBeDeclaration=*/true, /*Addlevel=*/true,
1204                /*MunchSemi=*/false);
1205   }
1206   // We fall through to parsing a structural element afterwards, so
1207   // class A {} n, m;
1208   // will end up in one unwrapped line.
1209 }
1210 
1211 void UnwrappedLineParser::parseObjCProtocolList() {
1212   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1213   do
1214     nextToken();
1215   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1216   nextToken(); // Skip '>'.
1217 }
1218 
1219 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1220   do {
1221     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1222       nextToken();
1223       addUnwrappedLine();
1224       break;
1225     }
1226     if (FormatTok->is(tok::l_brace)) {
1227       parseBlock(/*MustBeDeclaration=*/false);
1228       // In ObjC interfaces, nothing should be following the "}".
1229       addUnwrappedLine();
1230     } else if (FormatTok->is(tok::r_brace)) {
1231       // Ignore stray "}". parseStructuralElement doesn't consume them.
1232       nextToken();
1233       addUnwrappedLine();
1234     } else {
1235       parseStructuralElement();
1236     }
1237   } while (!eof());
1238 }
1239 
1240 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1241   nextToken();
1242   nextToken(); // interface name
1243 
1244   // @interface can be followed by either a base class, or a category.
1245   if (FormatTok->Tok.is(tok::colon)) {
1246     nextToken();
1247     nextToken(); // base class name
1248   } else if (FormatTok->Tok.is(tok::l_paren))
1249     // Skip category, if present.
1250     parseParens();
1251 
1252   if (FormatTok->Tok.is(tok::less))
1253     parseObjCProtocolList();
1254 
1255   // If instance variables are present, keep the '{' on the first line too.
1256   if (FormatTok->Tok.is(tok::l_brace))
1257     parseBlock(/*MustBeDeclaration=*/true);
1258 
1259   // With instance variables, this puts '}' on its own line.  Without instance
1260   // variables, this ends the @interface line.
1261   addUnwrappedLine();
1262 
1263   parseObjCUntilAtEnd();
1264 }
1265 
1266 void UnwrappedLineParser::parseObjCProtocol() {
1267   nextToken();
1268   nextToken(); // protocol name
1269 
1270   if (FormatTok->Tok.is(tok::less))
1271     parseObjCProtocolList();
1272 
1273   // Check for protocol declaration.
1274   if (FormatTok->Tok.is(tok::semi)) {
1275     nextToken();
1276     return addUnwrappedLine();
1277   }
1278 
1279   addUnwrappedLine();
1280   parseObjCUntilAtEnd();
1281 }
1282 
1283 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1284                                                  StringRef Prefix = "") {
1285   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1286                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1287   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1288                                                     E = Line.Tokens.end();
1289        I != E; ++I) {
1290     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1291   }
1292   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1293                                                     E = Line.Tokens.end();
1294        I != E; ++I) {
1295     const UnwrappedLineNode &Node = *I;
1296     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1297              I = Node.Children.begin(),
1298              E = Node.Children.end();
1299          I != E; ++I) {
1300       printDebugInfo(*I, "\nChild: ");
1301     }
1302   }
1303   llvm::dbgs() << "\n";
1304 }
1305 
1306 void UnwrappedLineParser::addUnwrappedLine() {
1307   if (Line->Tokens.empty())
1308     return;
1309   DEBUG({
1310     if (CurrentLines == &Lines)
1311       printDebugInfo(*Line);
1312   });
1313   CurrentLines->push_back(*Line);
1314   Line->Tokens.clear();
1315   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1316     for (SmallVectorImpl<UnwrappedLine>::iterator
1317              I = PreprocessorDirectives.begin(),
1318              E = PreprocessorDirectives.end();
1319          I != E; ++I) {
1320       CurrentLines->push_back(*I);
1321     }
1322     PreprocessorDirectives.clear();
1323   }
1324 }
1325 
1326 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1327 
1328 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1329   bool JustComments = Line->Tokens.empty();
1330   for (SmallVectorImpl<FormatToken *>::const_iterator
1331            I = CommentsBeforeNextToken.begin(),
1332            E = CommentsBeforeNextToken.end();
1333        I != E; ++I) {
1334     if ((*I)->NewlinesBefore && JustComments) {
1335       addUnwrappedLine();
1336     }
1337     pushToken(*I);
1338   }
1339   if (NewlineBeforeNext && JustComments) {
1340     addUnwrappedLine();
1341   }
1342   CommentsBeforeNextToken.clear();
1343 }
1344 
1345 void UnwrappedLineParser::nextToken() {
1346   if (eof())
1347     return;
1348   flushComments(FormatTok->NewlinesBefore > 0);
1349   pushToken(FormatTok);
1350   readToken();
1351 }
1352 
1353 void UnwrappedLineParser::readToken() {
1354   bool CommentsInCurrentLine = true;
1355   do {
1356     FormatTok = Tokens->getNextToken();
1357     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1358            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1359       // If there is an unfinished unwrapped line, we flush the preprocessor
1360       // directives only after that unwrapped line was finished later.
1361       bool SwitchToPreprocessorLines =
1362           !Line->Tokens.empty() && CurrentLines == &Lines;
1363       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1364       // Comments stored before the preprocessor directive need to be output
1365       // before the preprocessor directive, at the same level as the
1366       // preprocessor directive, as we consider them to apply to the directive.
1367       flushComments(FormatTok->NewlinesBefore > 0);
1368       parsePPDirective();
1369     }
1370 
1371     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1372         !Line->InPPDirective) {
1373       continue;
1374     }
1375 
1376     if (!FormatTok->Tok.is(tok::comment))
1377       return;
1378     if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
1379       CommentsInCurrentLine = false;
1380     }
1381     if (CommentsInCurrentLine) {
1382       pushToken(FormatTok);
1383     } else {
1384       CommentsBeforeNextToken.push_back(FormatTok);
1385     }
1386   } while (!eof());
1387 }
1388 
1389 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1390   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1391   if (MustBreakBeforeNextToken) {
1392     Line->Tokens.back().Tok->MustBreakBefore = true;
1393     MustBreakBeforeNextToken = false;
1394   }
1395 }
1396 
1397 } // end namespace format
1398 } // end namespace clang
1399