1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "llvm/Support/Debug.h"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   virtual FormatToken *getNextToken() {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   virtual unsigned getPosition() { return PreviousTokenSource->getPosition(); }
88 
89   virtual FormatToken *setPosition(unsigned Position) {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.take();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   virtual FormatToken *getNextToken() {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   virtual unsigned getPosition() {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   virtual FormatToken *setPosition(unsigned P) {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           // If there is a comma, semicolon or right paren after the closing
339           // brace, we assume this is a braced initializer list.  Note that
340           // regardless how we mark inner braces here, we will overwrite the
341           // BlockKind later if we parse a braced list (where all blocks inside
342           // are by default braced lists), or when we explicitly detect blocks
343           // (for example while parsing lambdas).
344           //
345           // We exclude + and - as they can be ObjC visibility modifiers.
346           if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren, tok::period,
347                                tok::r_square, tok::l_brace, tok::colon) ||
348               (NextTok->isBinaryOperator() &&
349                !NextTok->isOneOf(tok::plus, tok::minus))) {
350             Tok->BlockKind = BK_BracedInit;
351             LBraceStack.back()->BlockKind = BK_BracedInit;
352           } else {
353             Tok->BlockKind = BK_Block;
354             LBraceStack.back()->BlockKind = BK_Block;
355           }
356         }
357         LBraceStack.pop_back();
358       }
359       break;
360     case tok::semi:
361     case tok::kw_if:
362     case tok::kw_while:
363     case tok::kw_for:
364     case tok::kw_switch:
365     case tok::kw_try:
366       if (!LBraceStack.empty())
367         LBraceStack.back()->BlockKind = BK_Block;
368       break;
369     default:
370       break;
371     }
372     Tok = NextTok;
373     Position += ReadTokens;
374   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
375   // Assume other blocks for all unclosed opening braces.
376   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
377     if (LBraceStack[i]->BlockKind == BK_Unknown)
378       LBraceStack[i]->BlockKind = BK_Block;
379   }
380 
381   FormatTok = Tokens->setPosition(StoredPosition);
382 }
383 
384 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
385                                      bool MunchSemi) {
386   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
387   unsigned InitialLevel = Line->Level;
388   nextToken();
389 
390   addUnwrappedLine();
391 
392   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
393                                           MustBeDeclaration);
394   if (AddLevel)
395     ++Line->Level;
396   parseLevel(/*HasOpeningBrace=*/true);
397 
398   if (!FormatTok->Tok.is(tok::r_brace)) {
399     Line->Level = InitialLevel;
400     StructuralError = true;
401     return;
402   }
403 
404   nextToken(); // Munch the closing brace.
405   if (MunchSemi && FormatTok->Tok.is(tok::semi))
406     nextToken();
407   Line->Level = InitialLevel;
408 }
409 
410 void UnwrappedLineParser::parseChildBlock() {
411   FormatTok->BlockKind = BK_Block;
412   nextToken();
413   {
414     ScopedLineState LineState(*this);
415     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
416                                             /*MustBeDeclaration=*/false);
417     Line->Level += 1;
418     parseLevel(/*HasOpeningBrace=*/true);
419     Line->Level -= 1;
420   }
421   nextToken();
422 }
423 
424 void UnwrappedLineParser::parsePPDirective() {
425   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
426   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
427   nextToken();
428 
429   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
430     parsePPUnknown();
431     return;
432   }
433 
434   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
435   case tok::pp_define:
436     parsePPDefine();
437     return;
438   case tok::pp_if:
439     parsePPIf(/*IfDef=*/false);
440     break;
441   case tok::pp_ifdef:
442   case tok::pp_ifndef:
443     parsePPIf(/*IfDef=*/true);
444     break;
445   case tok::pp_else:
446     parsePPElse();
447     break;
448   case tok::pp_elif:
449     parsePPElIf();
450     break;
451   case tok::pp_endif:
452     parsePPEndIf();
453     break;
454   default:
455     parsePPUnknown();
456     break;
457   }
458 }
459 
460 void UnwrappedLineParser::pushPPConditional() {
461   if (!PPStack.empty() && PPStack.back() == PP_Unreachable)
462     PPStack.push_back(PP_Unreachable);
463   else
464     PPStack.push_back(PP_Conditional);
465 }
466 
467 void UnwrappedLineParser::parsePPIf(bool IfDef) {
468   ++PPBranchLevel;
469   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
470   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
471     PPLevelBranchIndex.push_back(0);
472     PPLevelBranchCount.push_back(0);
473   }
474   PPChainBranchIndex.push(0);
475   nextToken();
476   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
477                          StringRef(FormatTok->Tok.getLiteralData(),
478                                    FormatTok->Tok.getLength()) == "0") ||
479                         FormatTok->Tok.is(tok::kw_false);
480   if ((!IfDef && IsLiteralFalse) || PPLevelBranchIndex[PPBranchLevel] > 0) {
481     PPStack.push_back(PP_Unreachable);
482   } else {
483     pushPPConditional();
484   }
485   parsePPUnknown();
486 }
487 
488 void UnwrappedLineParser::parsePPElse() {
489   if (!PPStack.empty())
490     PPStack.pop_back();
491   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
492   if (!PPChainBranchIndex.empty())
493     ++PPChainBranchIndex.top();
494   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
495       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top()) {
496     PPStack.push_back(PP_Unreachable);
497   } else {
498     pushPPConditional();
499   }
500   parsePPUnknown();
501 }
502 
503 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
504 
505 void UnwrappedLineParser::parsePPEndIf() {
506   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
507   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
508     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
509       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
510     }
511   }
512   // Guard against #endif's without #if.
513   if (PPBranchLevel > 0)
514     --PPBranchLevel;
515   if (!PPChainBranchIndex.empty())
516     PPChainBranchIndex.pop();
517   if (!PPStack.empty())
518     PPStack.pop_back();
519   parsePPUnknown();
520 }
521 
522 void UnwrappedLineParser::parsePPDefine() {
523   nextToken();
524 
525   if (FormatTok->Tok.getKind() != tok::identifier) {
526     parsePPUnknown();
527     return;
528   }
529   nextToken();
530   if (FormatTok->Tok.getKind() == tok::l_paren &&
531       FormatTok->WhitespaceRange.getBegin() ==
532           FormatTok->WhitespaceRange.getEnd()) {
533     parseParens();
534   }
535   addUnwrappedLine();
536   Line->Level = 1;
537 
538   // Errors during a preprocessor directive can only affect the layout of the
539   // preprocessor directive, and thus we ignore them. An alternative approach
540   // would be to use the same approach we use on the file level (no
541   // re-indentation if there was a structural error) within the macro
542   // definition.
543   parseFile();
544 }
545 
546 void UnwrappedLineParser::parsePPUnknown() {
547   do {
548     nextToken();
549   } while (!eof());
550   addUnwrappedLine();
551 }
552 
553 // Here we blacklist certain tokens that are not usually the first token in an
554 // unwrapped line. This is used in attempt to distinguish macro calls without
555 // trailing semicolons from other constructs split to several lines.
556 bool tokenCanStartNewLine(clang::Token Tok) {
557   // Semicolon can be a null-statement, l_square can be a start of a macro or
558   // a C++11 attribute, but this doesn't seem to be common.
559   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
560          Tok.isNot(tok::l_square) &&
561          // Tokens that can only be used as binary operators and a part of
562          // overloaded operator names.
563          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
564          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
565          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
566          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
567          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
568          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
569          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
570          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
571          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
572          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
573          Tok.isNot(tok::lesslessequal) &&
574          // Colon is used in labels, base class lists, initializer lists,
575          // range-based for loops, ternary operator, but should never be the
576          // first token in an unwrapped line.
577          Tok.isNot(tok::colon);
578 }
579 
580 void UnwrappedLineParser::parseStructuralElement() {
581   assert(!FormatTok->Tok.is(tok::l_brace));
582   switch (FormatTok->Tok.getKind()) {
583   case tok::at:
584     nextToken();
585     if (FormatTok->Tok.is(tok::l_brace)) {
586       parseBracedList();
587       break;
588     }
589     switch (FormatTok->Tok.getObjCKeywordID()) {
590     case tok::objc_public:
591     case tok::objc_protected:
592     case tok::objc_package:
593     case tok::objc_private:
594       return parseAccessSpecifier();
595     case tok::objc_interface:
596     case tok::objc_implementation:
597       return parseObjCInterfaceOrImplementation();
598     case tok::objc_protocol:
599       return parseObjCProtocol();
600     case tok::objc_end:
601       return; // Handled by the caller.
602     case tok::objc_optional:
603     case tok::objc_required:
604       nextToken();
605       addUnwrappedLine();
606       return;
607     default:
608       break;
609     }
610     break;
611   case tok::kw_namespace:
612     parseNamespace();
613     return;
614   case tok::kw_inline:
615     nextToken();
616     if (FormatTok->Tok.is(tok::kw_namespace)) {
617       parseNamespace();
618       return;
619     }
620     break;
621   case tok::kw_public:
622   case tok::kw_protected:
623   case tok::kw_private:
624     parseAccessSpecifier();
625     return;
626   case tok::kw_if:
627     parseIfThenElse();
628     return;
629   case tok::kw_for:
630   case tok::kw_while:
631     parseForOrWhileLoop();
632     return;
633   case tok::kw_do:
634     parseDoWhile();
635     return;
636   case tok::kw_switch:
637     parseSwitch();
638     return;
639   case tok::kw_default:
640     nextToken();
641     parseLabel();
642     return;
643   case tok::kw_case:
644     parseCaseLabel();
645     return;
646   case tok::kw_extern:
647     nextToken();
648     if (FormatTok->Tok.is(tok::string_literal)) {
649       nextToken();
650       if (FormatTok->Tok.is(tok::l_brace)) {
651         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
652         addUnwrappedLine();
653         return;
654       }
655     }
656     // In all other cases, parse the declaration.
657     break;
658   default:
659     break;
660   }
661   do {
662     switch (FormatTok->Tok.getKind()) {
663     case tok::at:
664       nextToken();
665       if (FormatTok->Tok.is(tok::l_brace))
666         parseBracedList();
667       break;
668     case tok::kw_enum:
669       parseEnum();
670       break;
671     case tok::kw_typedef:
672       nextToken();
673       // FIXME: Use the IdentifierTable instead.
674       if (FormatTok->TokenText == "NS_ENUM")
675         parseEnum();
676       break;
677     case tok::kw_struct:
678     case tok::kw_union:
679     case tok::kw_class:
680       parseRecord();
681       // A record declaration or definition is always the start of a structural
682       // element.
683       break;
684     case tok::semi:
685       nextToken();
686       addUnwrappedLine();
687       return;
688     case tok::r_brace:
689       addUnwrappedLine();
690       return;
691     case tok::l_paren:
692       parseParens();
693       break;
694     case tok::caret:
695       nextToken();
696       if (FormatTok->is(tok::l_brace)) {
697         parseChildBlock();
698       }
699       break;
700     case tok::l_brace:
701       if (!tryToParseBracedList()) {
702         // A block outside of parentheses must be the last part of a
703         // structural element.
704         // FIXME: Figure out cases where this is not true, and add projections
705         // for them (the one we know is missing are lambdas).
706         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
707           addUnwrappedLine();
708         FormatTok->Type = TT_FunctionLBrace;
709         parseBlock(/*MustBeDeclaration=*/false);
710         addUnwrappedLine();
711         return;
712       }
713       // Otherwise this was a braced init list, and the structural
714       // element continues.
715       break;
716     case tok::identifier: {
717       StringRef Text = FormatTok->TokenText;
718       nextToken();
719       if (Line->Tokens.size() == 1) {
720         if (FormatTok->Tok.is(tok::colon)) {
721           parseLabel();
722           return;
723         }
724         // Recognize function-like macro usages without trailing semicolon.
725         if (FormatTok->Tok.is(tok::l_paren)) {
726           parseParens();
727           if (FormatTok->NewlinesBefore > 0 &&
728               tokenCanStartNewLine(FormatTok->Tok)) {
729             addUnwrappedLine();
730             return;
731           }
732         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
733                    Text == Text.upper()) {
734           // Recognize free-standing macros like Q_OBJECT.
735           addUnwrappedLine();
736           return;
737         }
738       }
739       break;
740     }
741     case tok::equal:
742       nextToken();
743       if (FormatTok->Tok.is(tok::l_brace)) {
744         parseBracedList();
745       }
746       break;
747     case tok::l_square:
748       parseSquare();
749       break;
750     default:
751       nextToken();
752       break;
753     }
754   } while (!eof());
755 }
756 
757 bool UnwrappedLineParser::tryToParseLambda() {
758   // FIXME: This is a dirty way to access the previous token. Find a better
759   // solution.
760   if (!Line->Tokens.empty() &&
761       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator) ||
762        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
763     nextToken();
764     return false;
765   }
766   assert(FormatTok->is(tok::l_square));
767   FormatToken &LSquare = *FormatTok;
768   if (!tryToParseLambdaIntroducer())
769     return false;
770 
771   while (FormatTok && FormatTok->isNot(tok::l_brace)) {
772     if (FormatTok->isSimpleTypeSpecifier()) {
773       nextToken();
774       continue;
775     }
776     switch (FormatTok->Tok.getKind()) {
777     case tok::l_brace:
778       break;
779     case tok::l_paren:
780       parseParens();
781       break;
782     case tok::less:
783     case tok::greater:
784     case tok::identifier:
785     case tok::coloncolon:
786     case tok::kw_mutable:
787     case tok::arrow:
788       nextToken();
789       break;
790     default:
791       return true;
792     }
793   }
794   LSquare.Type = TT_LambdaLSquare;
795   parseChildBlock();
796   return true;
797 }
798 
799 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
800   nextToken();
801   if (FormatTok->is(tok::equal)) {
802     nextToken();
803     if (FormatTok->is(tok::r_square)) {
804       nextToken();
805       return true;
806     }
807     if (FormatTok->isNot(tok::comma))
808       return false;
809     nextToken();
810   } else if (FormatTok->is(tok::amp)) {
811     nextToken();
812     if (FormatTok->is(tok::r_square)) {
813       nextToken();
814       return true;
815     }
816     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
817       return false;
818     }
819     if (FormatTok->is(tok::comma))
820       nextToken();
821   } else if (FormatTok->is(tok::r_square)) {
822     nextToken();
823     return true;
824   }
825   do {
826     if (FormatTok->is(tok::amp))
827       nextToken();
828     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
829       return false;
830     nextToken();
831     if (FormatTok->is(tok::comma)) {
832       nextToken();
833     } else if (FormatTok->is(tok::r_square)) {
834       nextToken();
835       return true;
836     } else {
837       return false;
838     }
839   } while (!eof());
840   return false;
841 }
842 
843 bool UnwrappedLineParser::tryToParseBracedList() {
844   if (FormatTok->BlockKind == BK_Unknown)
845     calculateBraceTypes();
846   assert(FormatTok->BlockKind != BK_Unknown);
847   if (FormatTok->BlockKind == BK_Block)
848     return false;
849   parseBracedList();
850   return true;
851 }
852 
853 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
854   bool HasError = false;
855   nextToken();
856 
857   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
858   // replace this by using parseAssigmentExpression() inside.
859   do {
860     // FIXME: When we start to support lambdas, we'll want to parse them away
861     // here, otherwise our bail-out scenarios below break. The better solution
862     // might be to just implement a more or less complete expression parser.
863     switch (FormatTok->Tok.getKind()) {
864     case tok::caret:
865       nextToken();
866       if (FormatTok->is(tok::l_brace)) {
867         parseChildBlock();
868       }
869       break;
870     case tok::l_square:
871       tryToParseLambda();
872       break;
873     case tok::l_brace:
874       // Assume there are no blocks inside a braced init list apart
875       // from the ones we explicitly parse out (like lambdas).
876       FormatTok->BlockKind = BK_BracedInit;
877       parseBracedList();
878       break;
879     case tok::r_brace:
880       nextToken();
881       return !HasError;
882     case tok::semi:
883       HasError = true;
884       if (!ContinueOnSemicolons)
885         return !HasError;
886       nextToken();
887       break;
888     case tok::comma:
889       nextToken();
890       break;
891     default:
892       nextToken();
893       break;
894     }
895   } while (!eof());
896   return false;
897 }
898 
899 void UnwrappedLineParser::parseParens() {
900   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
901   nextToken();
902   do {
903     switch (FormatTok->Tok.getKind()) {
904     case tok::l_paren:
905       parseParens();
906       break;
907     case tok::r_paren:
908       nextToken();
909       return;
910     case tok::r_brace:
911       // A "}" inside parenthesis is an error if there wasn't a matching "{".
912       return;
913     case tok::l_square:
914       tryToParseLambda();
915       break;
916     case tok::l_brace: {
917       if (!tryToParseBracedList()) {
918         parseChildBlock();
919       }
920       break;
921     }
922     case tok::at:
923       nextToken();
924       if (FormatTok->Tok.is(tok::l_brace))
925         parseBracedList();
926       break;
927     default:
928       nextToken();
929       break;
930     }
931   } while (!eof());
932 }
933 
934 void UnwrappedLineParser::parseSquare() {
935   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
936   if (tryToParseLambda())
937     return;
938   do {
939     switch (FormatTok->Tok.getKind()) {
940     case tok::l_paren:
941       parseParens();
942       break;
943     case tok::r_square:
944       nextToken();
945       return;
946     case tok::r_brace:
947       // A "}" inside parenthesis is an error if there wasn't a matching "{".
948       return;
949     case tok::l_square:
950       parseSquare();
951       break;
952     case tok::l_brace: {
953       if (!tryToParseBracedList()) {
954         parseChildBlock();
955       }
956       break;
957     }
958     case tok::at:
959       nextToken();
960       if (FormatTok->Tok.is(tok::l_brace))
961         parseBracedList();
962       break;
963     default:
964       nextToken();
965       break;
966     }
967   } while (!eof());
968 }
969 
970 void UnwrappedLineParser::parseIfThenElse() {
971   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
972   nextToken();
973   if (FormatTok->Tok.is(tok::l_paren))
974     parseParens();
975   bool NeedsUnwrappedLine = false;
976   if (FormatTok->Tok.is(tok::l_brace)) {
977     CompoundStatementIndenter Indenter(this, Style, Line->Level);
978     parseBlock(/*MustBeDeclaration=*/false);
979     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
980         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
981       addUnwrappedLine();
982     } else {
983       NeedsUnwrappedLine = true;
984     }
985   } else {
986     addUnwrappedLine();
987     ++Line->Level;
988     parseStructuralElement();
989     --Line->Level;
990   }
991   if (FormatTok->Tok.is(tok::kw_else)) {
992     nextToken();
993     if (FormatTok->Tok.is(tok::l_brace)) {
994       CompoundStatementIndenter Indenter(this, Style, Line->Level);
995       parseBlock(/*MustBeDeclaration=*/false);
996       addUnwrappedLine();
997     } else if (FormatTok->Tok.is(tok::kw_if)) {
998       parseIfThenElse();
999     } else {
1000       addUnwrappedLine();
1001       ++Line->Level;
1002       parseStructuralElement();
1003       --Line->Level;
1004     }
1005   } else if (NeedsUnwrappedLine) {
1006     addUnwrappedLine();
1007   }
1008 }
1009 
1010 void UnwrappedLineParser::parseNamespace() {
1011   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1012   nextToken();
1013   if (FormatTok->Tok.is(tok::identifier))
1014     nextToken();
1015   if (FormatTok->Tok.is(tok::l_brace)) {
1016     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1017         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1018         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1019       addUnwrappedLine();
1020 
1021     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1022                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1023                      DeclarationScopeStack.size() > 1);
1024     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1025     // Munch the semicolon after a namespace. This is more common than one would
1026     // think. Puttin the semicolon into its own line is very ugly.
1027     if (FormatTok->Tok.is(tok::semi))
1028       nextToken();
1029     addUnwrappedLine();
1030   }
1031   // FIXME: Add error handling.
1032 }
1033 
1034 void UnwrappedLineParser::parseForOrWhileLoop() {
1035   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while)) &&
1036          "'for' or 'while' expected");
1037   nextToken();
1038   if (FormatTok->Tok.is(tok::l_paren))
1039     parseParens();
1040   if (FormatTok->Tok.is(tok::l_brace)) {
1041     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1042     parseBlock(/*MustBeDeclaration=*/false);
1043     addUnwrappedLine();
1044   } else {
1045     addUnwrappedLine();
1046     ++Line->Level;
1047     parseStructuralElement();
1048     --Line->Level;
1049   }
1050 }
1051 
1052 void UnwrappedLineParser::parseDoWhile() {
1053   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1054   nextToken();
1055   if (FormatTok->Tok.is(tok::l_brace)) {
1056     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1057     parseBlock(/*MustBeDeclaration=*/false);
1058     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1059       addUnwrappedLine();
1060   } else {
1061     addUnwrappedLine();
1062     ++Line->Level;
1063     parseStructuralElement();
1064     --Line->Level;
1065   }
1066 
1067   // FIXME: Add error handling.
1068   if (!FormatTok->Tok.is(tok::kw_while)) {
1069     addUnwrappedLine();
1070     return;
1071   }
1072 
1073   nextToken();
1074   parseStructuralElement();
1075 }
1076 
1077 void UnwrappedLineParser::parseLabel() {
1078   nextToken();
1079   unsigned OldLineLevel = Line->Level;
1080   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1081     --Line->Level;
1082   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1083     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1084     parseBlock(/*MustBeDeclaration=*/false);
1085     if (FormatTok->Tok.is(tok::kw_break)) {
1086       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1087       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1088           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1089         addUnwrappedLine();
1090       }
1091       parseStructuralElement();
1092     }
1093     addUnwrappedLine();
1094   } else {
1095     addUnwrappedLine();
1096   }
1097   Line->Level = OldLineLevel;
1098 }
1099 
1100 void UnwrappedLineParser::parseCaseLabel() {
1101   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1102   // FIXME: fix handling of complex expressions here.
1103   do {
1104     nextToken();
1105   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1106   parseLabel();
1107 }
1108 
1109 void UnwrappedLineParser::parseSwitch() {
1110   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1111   nextToken();
1112   if (FormatTok->Tok.is(tok::l_paren))
1113     parseParens();
1114   if (FormatTok->Tok.is(tok::l_brace)) {
1115     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1116     parseBlock(/*MustBeDeclaration=*/false);
1117     addUnwrappedLine();
1118   } else {
1119     addUnwrappedLine();
1120     ++Line->Level;
1121     parseStructuralElement();
1122     --Line->Level;
1123   }
1124 }
1125 
1126 void UnwrappedLineParser::parseAccessSpecifier() {
1127   nextToken();
1128   // Understand Qt's slots.
1129   if (FormatTok->is(tok::identifier) &&
1130       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1131     nextToken();
1132   // Otherwise, we don't know what it is, and we'd better keep the next token.
1133   if (FormatTok->Tok.is(tok::colon))
1134     nextToken();
1135   addUnwrappedLine();
1136 }
1137 
1138 void UnwrappedLineParser::parseEnum() {
1139   if (FormatTok->Tok.is(tok::kw_enum)) {
1140     // Won't be 'enum' for NS_ENUMs.
1141     nextToken();
1142   }
1143   // Eat up enum class ...
1144   if (FormatTok->Tok.is(tok::kw_class) ||
1145       FormatTok->Tok.is(tok::kw_struct))
1146       nextToken();
1147   while (FormatTok->Tok.getIdentifierInfo() ||
1148          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1149     nextToken();
1150     // We can have macros or attributes in between 'enum' and the enum name.
1151     if (FormatTok->Tok.is(tok::l_paren)) {
1152       parseParens();
1153     }
1154     if (FormatTok->Tok.is(tok::identifier))
1155       nextToken();
1156   }
1157   if (FormatTok->Tok.is(tok::l_brace)) {
1158     FormatTok->BlockKind = BK_Block;
1159     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1160     if (HasError) {
1161       if (FormatTok->is(tok::semi))
1162         nextToken();
1163       addUnwrappedLine();
1164     }
1165   }
1166   // We fall through to parsing a structural element afterwards, so that in
1167   // enum A {} n, m;
1168   // "} n, m;" will end up in one unwrapped line.
1169 }
1170 
1171 void UnwrappedLineParser::parseRecord() {
1172   nextToken();
1173   if (FormatTok->Tok.is(tok::identifier) ||
1174       FormatTok->Tok.is(tok::kw___attribute) ||
1175       FormatTok->Tok.is(tok::kw___declspec) ||
1176       FormatTok->Tok.is(tok::kw_alignas)) {
1177     nextToken();
1178     // We can have macros or attributes in between 'class' and the class name.
1179     if (FormatTok->Tok.is(tok::l_paren)) {
1180       parseParens();
1181     }
1182     // The actual identifier can be a nested name specifier, and in macros
1183     // it is often token-pasted.
1184     while (FormatTok->Tok.is(tok::identifier) ||
1185            FormatTok->Tok.is(tok::coloncolon) ||
1186            FormatTok->Tok.is(tok::hashhash))
1187       nextToken();
1188 
1189     // Note that parsing away template declarations here leads to incorrectly
1190     // accepting function declarations as record declarations.
1191     // In general, we cannot solve this problem. Consider:
1192     // class A<int> B() {}
1193     // which can be a function definition or a class definition when B() is a
1194     // macro. If we find enough real-world cases where this is a problem, we
1195     // can parse for the 'template' keyword in the beginning of the statement,
1196     // and thus rule out the record production in case there is no template
1197     // (this would still leave us with an ambiguity between template function
1198     // and class declarations).
1199     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1200       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1201         if (FormatTok->Tok.is(tok::semi))
1202           return;
1203         nextToken();
1204       }
1205     }
1206   }
1207   if (FormatTok->Tok.is(tok::l_brace)) {
1208     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1209         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1210         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1211       addUnwrappedLine();
1212 
1213     parseBlock(/*MustBeDeclaration=*/true, /*Addlevel=*/true,
1214                /*MunchSemi=*/false);
1215   }
1216   // We fall through to parsing a structural element afterwards, so
1217   // class A {} n, m;
1218   // will end up in one unwrapped line.
1219 }
1220 
1221 void UnwrappedLineParser::parseObjCProtocolList() {
1222   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1223   do
1224     nextToken();
1225   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1226   nextToken(); // Skip '>'.
1227 }
1228 
1229 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1230   do {
1231     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1232       nextToken();
1233       addUnwrappedLine();
1234       break;
1235     }
1236     if (FormatTok->is(tok::l_brace)) {
1237       parseBlock(/*MustBeDeclaration=*/false);
1238       // In ObjC interfaces, nothing should be following the "}".
1239       addUnwrappedLine();
1240     } else if (FormatTok->is(tok::r_brace)) {
1241       // Ignore stray "}". parseStructuralElement doesn't consume them.
1242       nextToken();
1243       addUnwrappedLine();
1244     } else {
1245       parseStructuralElement();
1246     }
1247   } while (!eof());
1248 }
1249 
1250 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1251   nextToken();
1252   nextToken(); // interface name
1253 
1254   // @interface can be followed by either a base class, or a category.
1255   if (FormatTok->Tok.is(tok::colon)) {
1256     nextToken();
1257     nextToken(); // base class name
1258   } else if (FormatTok->Tok.is(tok::l_paren))
1259     // Skip category, if present.
1260     parseParens();
1261 
1262   if (FormatTok->Tok.is(tok::less))
1263     parseObjCProtocolList();
1264 
1265   // If instance variables are present, keep the '{' on the first line too.
1266   if (FormatTok->Tok.is(tok::l_brace))
1267     parseBlock(/*MustBeDeclaration=*/true);
1268 
1269   // With instance variables, this puts '}' on its own line.  Without instance
1270   // variables, this ends the @interface line.
1271   addUnwrappedLine();
1272 
1273   parseObjCUntilAtEnd();
1274 }
1275 
1276 void UnwrappedLineParser::parseObjCProtocol() {
1277   nextToken();
1278   nextToken(); // protocol name
1279 
1280   if (FormatTok->Tok.is(tok::less))
1281     parseObjCProtocolList();
1282 
1283   // Check for protocol declaration.
1284   if (FormatTok->Tok.is(tok::semi)) {
1285     nextToken();
1286     return addUnwrappedLine();
1287   }
1288 
1289   addUnwrappedLine();
1290   parseObjCUntilAtEnd();
1291 }
1292 
1293 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1294                                                  StringRef Prefix = "") {
1295   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1296                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1297   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1298                                                     E = Line.Tokens.end();
1299        I != E; ++I) {
1300     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1301   }
1302   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1303                                                     E = Line.Tokens.end();
1304        I != E; ++I) {
1305     const UnwrappedLineNode &Node = *I;
1306     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1307              I = Node.Children.begin(),
1308              E = Node.Children.end();
1309          I != E; ++I) {
1310       printDebugInfo(*I, "\nChild: ");
1311     }
1312   }
1313   llvm::dbgs() << "\n";
1314 }
1315 
1316 void UnwrappedLineParser::addUnwrappedLine() {
1317   if (Line->Tokens.empty())
1318     return;
1319   DEBUG({
1320     if (CurrentLines == &Lines)
1321       printDebugInfo(*Line);
1322   });
1323   CurrentLines->push_back(*Line);
1324   Line->Tokens.clear();
1325   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1326     for (SmallVectorImpl<UnwrappedLine>::iterator
1327              I = PreprocessorDirectives.begin(),
1328              E = PreprocessorDirectives.end();
1329          I != E; ++I) {
1330       CurrentLines->push_back(*I);
1331     }
1332     PreprocessorDirectives.clear();
1333   }
1334 }
1335 
1336 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1337 
1338 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1339   bool JustComments = Line->Tokens.empty();
1340   for (SmallVectorImpl<FormatToken *>::const_iterator
1341            I = CommentsBeforeNextToken.begin(),
1342            E = CommentsBeforeNextToken.end();
1343        I != E; ++I) {
1344     if ((*I)->NewlinesBefore && JustComments) {
1345       addUnwrappedLine();
1346     }
1347     pushToken(*I);
1348   }
1349   if (NewlineBeforeNext && JustComments) {
1350     addUnwrappedLine();
1351   }
1352   CommentsBeforeNextToken.clear();
1353 }
1354 
1355 void UnwrappedLineParser::nextToken() {
1356   if (eof())
1357     return;
1358   flushComments(FormatTok->NewlinesBefore > 0);
1359   pushToken(FormatTok);
1360   readToken();
1361 }
1362 
1363 void UnwrappedLineParser::readToken() {
1364   bool CommentsInCurrentLine = true;
1365   do {
1366     FormatTok = Tokens->getNextToken();
1367     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1368            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1369       // If there is an unfinished unwrapped line, we flush the preprocessor
1370       // directives only after that unwrapped line was finished later.
1371       bool SwitchToPreprocessorLines =
1372           !Line->Tokens.empty() && CurrentLines == &Lines;
1373       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1374       // Comments stored before the preprocessor directive need to be output
1375       // before the preprocessor directive, at the same level as the
1376       // preprocessor directive, as we consider them to apply to the directive.
1377       flushComments(FormatTok->NewlinesBefore > 0);
1378       parsePPDirective();
1379     }
1380 
1381     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1382         !Line->InPPDirective) {
1383       continue;
1384     }
1385 
1386     if (!FormatTok->Tok.is(tok::comment))
1387       return;
1388     if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
1389       CommentsInCurrentLine = false;
1390     }
1391     if (CommentsInCurrentLine) {
1392       pushToken(FormatTok);
1393     } else {
1394       CommentsBeforeNextToken.push_back(FormatTok);
1395     }
1396   } while (!eof());
1397 }
1398 
1399 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1400   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1401   if (MustBreakBeforeNextToken) {
1402     Line->Tokens.back().Tok->MustBreakBefore = true;
1403     MustBreakBeforeNextToken = false;
1404   }
1405 }
1406 
1407 } // end namespace format
1408 } // end namespace clang
1409