1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "llvm/Support/Debug.h"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.release();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   FormatToken *getNextToken() override {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   unsigned getPosition() override {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   FormatToken *setPosition(unsigned P) override {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           // If there is a comma, semicolon or right paren after the closing
339           // brace, we assume this is a braced initializer list.  Note that
340           // regardless how we mark inner braces here, we will overwrite the
341           // BlockKind later if we parse a braced list (where all blocks inside
342           // are by default braced lists), or when we explicitly detect blocks
343           // (for example while parsing lambdas).
344           //
345           // We exclude + and - as they can be ObjC visibility modifiers.
346           if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren, tok::period,
347                                tok::r_square, tok::l_brace, tok::colon) ||
348               (NextTok->isBinaryOperator() &&
349                !NextTok->isOneOf(tok::plus, tok::minus))) {
350             Tok->BlockKind = BK_BracedInit;
351             LBraceStack.back()->BlockKind = BK_BracedInit;
352           } else {
353             Tok->BlockKind = BK_Block;
354             LBraceStack.back()->BlockKind = BK_Block;
355           }
356         }
357         LBraceStack.pop_back();
358       }
359       break;
360     case tok::at:
361     case tok::semi:
362     case tok::kw_if:
363     case tok::kw_while:
364     case tok::kw_for:
365     case tok::kw_switch:
366     case tok::kw_try:
367       if (!LBraceStack.empty())
368         LBraceStack.back()->BlockKind = BK_Block;
369       break;
370     default:
371       break;
372     }
373     Tok = NextTok;
374     Position += ReadTokens;
375   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
376   // Assume other blocks for all unclosed opening braces.
377   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
378     if (LBraceStack[i]->BlockKind == BK_Unknown)
379       LBraceStack[i]->BlockKind = BK_Block;
380   }
381 
382   FormatTok = Tokens->setPosition(StoredPosition);
383 }
384 
385 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
386                                      bool MunchSemi) {
387   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
388   unsigned InitialLevel = Line->Level;
389   nextToken();
390 
391   addUnwrappedLine();
392 
393   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
394                                           MustBeDeclaration);
395   if (AddLevel)
396     ++Line->Level;
397   parseLevel(/*HasOpeningBrace=*/true);
398 
399   if (!FormatTok->Tok.is(tok::r_brace)) {
400     Line->Level = InitialLevel;
401     StructuralError = true;
402     return;
403   }
404 
405   nextToken(); // Munch the closing brace.
406   if (MunchSemi && FormatTok->Tok.is(tok::semi))
407     nextToken();
408   Line->Level = InitialLevel;
409 }
410 
411 void UnwrappedLineParser::parseChildBlock() {
412   FormatTok->BlockKind = BK_Block;
413   nextToken();
414   {
415     ScopedLineState LineState(*this);
416     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
417                                             /*MustBeDeclaration=*/false);
418     Line->Level += 1;
419     parseLevel(/*HasOpeningBrace=*/true);
420     Line->Level -= 1;
421   }
422   nextToken();
423 }
424 
425 void UnwrappedLineParser::parsePPDirective() {
426   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
427   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
428   nextToken();
429 
430   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
431     parsePPUnknown();
432     return;
433   }
434 
435   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
436   case tok::pp_define:
437     parsePPDefine();
438     return;
439   case tok::pp_if:
440     parsePPIf(/*IfDef=*/false);
441     break;
442   case tok::pp_ifdef:
443   case tok::pp_ifndef:
444     parsePPIf(/*IfDef=*/true);
445     break;
446   case tok::pp_else:
447     parsePPElse();
448     break;
449   case tok::pp_elif:
450     parsePPElIf();
451     break;
452   case tok::pp_endif:
453     parsePPEndIf();
454     break;
455   default:
456     parsePPUnknown();
457     break;
458   }
459 }
460 
461 void UnwrappedLineParser::pushPPConditional() {
462   if (!PPStack.empty() && PPStack.back() == PP_Unreachable)
463     PPStack.push_back(PP_Unreachable);
464   else
465     PPStack.push_back(PP_Conditional);
466 }
467 
468 void UnwrappedLineParser::parsePPIf(bool IfDef) {
469   ++PPBranchLevel;
470   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
471   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
472     PPLevelBranchIndex.push_back(0);
473     PPLevelBranchCount.push_back(0);
474   }
475   PPChainBranchIndex.push(0);
476   nextToken();
477   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
478                          StringRef(FormatTok->Tok.getLiteralData(),
479                                    FormatTok->Tok.getLength()) == "0") ||
480                         FormatTok->Tok.is(tok::kw_false);
481   if ((!IfDef && IsLiteralFalse) || PPLevelBranchIndex[PPBranchLevel] > 0) {
482     PPStack.push_back(PP_Unreachable);
483   } else {
484     pushPPConditional();
485   }
486   parsePPUnknown();
487 }
488 
489 void UnwrappedLineParser::parsePPElse() {
490   if (!PPStack.empty())
491     PPStack.pop_back();
492   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
493   if (!PPChainBranchIndex.empty())
494     ++PPChainBranchIndex.top();
495   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
496       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top()) {
497     PPStack.push_back(PP_Unreachable);
498   } else {
499     pushPPConditional();
500   }
501   parsePPUnknown();
502 }
503 
504 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
505 
506 void UnwrappedLineParser::parsePPEndIf() {
507   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
508   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
509     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
510       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
511     }
512   }
513   // Guard against #endif's without #if.
514   if (PPBranchLevel > 0)
515     --PPBranchLevel;
516   if (!PPChainBranchIndex.empty())
517     PPChainBranchIndex.pop();
518   if (!PPStack.empty())
519     PPStack.pop_back();
520   parsePPUnknown();
521 }
522 
523 void UnwrappedLineParser::parsePPDefine() {
524   nextToken();
525 
526   if (FormatTok->Tok.getKind() != tok::identifier) {
527     parsePPUnknown();
528     return;
529   }
530   nextToken();
531   if (FormatTok->Tok.getKind() == tok::l_paren &&
532       FormatTok->WhitespaceRange.getBegin() ==
533           FormatTok->WhitespaceRange.getEnd()) {
534     parseParens();
535   }
536   addUnwrappedLine();
537   Line->Level = 1;
538 
539   // Errors during a preprocessor directive can only affect the layout of the
540   // preprocessor directive, and thus we ignore them. An alternative approach
541   // would be to use the same approach we use on the file level (no
542   // re-indentation if there was a structural error) within the macro
543   // definition.
544   parseFile();
545 }
546 
547 void UnwrappedLineParser::parsePPUnknown() {
548   do {
549     nextToken();
550   } while (!eof());
551   addUnwrappedLine();
552 }
553 
554 // Here we blacklist certain tokens that are not usually the first token in an
555 // unwrapped line. This is used in attempt to distinguish macro calls without
556 // trailing semicolons from other constructs split to several lines.
557 bool tokenCanStartNewLine(clang::Token Tok) {
558   // Semicolon can be a null-statement, l_square can be a start of a macro or
559   // a C++11 attribute, but this doesn't seem to be common.
560   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
561          Tok.isNot(tok::l_square) &&
562          // Tokens that can only be used as binary operators and a part of
563          // overloaded operator names.
564          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
565          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
566          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
567          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
568          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
569          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
570          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
571          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
572          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
573          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
574          Tok.isNot(tok::lesslessequal) &&
575          // Colon is used in labels, base class lists, initializer lists,
576          // range-based for loops, ternary operator, but should never be the
577          // first token in an unwrapped line.
578          Tok.isNot(tok::colon);
579 }
580 
581 void UnwrappedLineParser::parseStructuralElement() {
582   assert(!FormatTok->Tok.is(tok::l_brace));
583   switch (FormatTok->Tok.getKind()) {
584   case tok::at:
585     nextToken();
586     if (FormatTok->Tok.is(tok::l_brace)) {
587       parseBracedList();
588       break;
589     }
590     switch (FormatTok->Tok.getObjCKeywordID()) {
591     case tok::objc_public:
592     case tok::objc_protected:
593     case tok::objc_package:
594     case tok::objc_private:
595       return parseAccessSpecifier();
596     case tok::objc_interface:
597     case tok::objc_implementation:
598       return parseObjCInterfaceOrImplementation();
599     case tok::objc_protocol:
600       return parseObjCProtocol();
601     case tok::objc_end:
602       return; // Handled by the caller.
603     case tok::objc_optional:
604     case tok::objc_required:
605       nextToken();
606       addUnwrappedLine();
607       return;
608     default:
609       break;
610     }
611     break;
612   case tok::kw_namespace:
613     parseNamespace();
614     return;
615   case tok::kw_inline:
616     nextToken();
617     if (FormatTok->Tok.is(tok::kw_namespace)) {
618       parseNamespace();
619       return;
620     }
621     break;
622   case tok::kw_public:
623   case tok::kw_protected:
624   case tok::kw_private:
625     parseAccessSpecifier();
626     return;
627   case tok::kw_if:
628     parseIfThenElse();
629     return;
630   case tok::kw_for:
631   case tok::kw_while:
632     parseForOrWhileLoop();
633     return;
634   case tok::kw_do:
635     parseDoWhile();
636     return;
637   case tok::kw_switch:
638     parseSwitch();
639     return;
640   case tok::kw_default:
641     nextToken();
642     parseLabel();
643     return;
644   case tok::kw_case:
645     parseCaseLabel();
646     return;
647   case tok::kw_extern:
648     nextToken();
649     if (FormatTok->Tok.is(tok::string_literal)) {
650       nextToken();
651       if (FormatTok->Tok.is(tok::l_brace)) {
652         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
653         addUnwrappedLine();
654         return;
655       }
656     }
657     break;
658   case tok::identifier:
659     if (FormatTok->IsForEachMacro) {
660       parseForOrWhileLoop();
661       return;
662     }
663     // In all other cases, parse the declaration.
664     break;
665   default:
666     break;
667   }
668   do {
669     switch (FormatTok->Tok.getKind()) {
670     case tok::at:
671       nextToken();
672       if (FormatTok->Tok.is(tok::l_brace))
673         parseBracedList();
674       break;
675     case tok::kw_enum:
676       parseEnum();
677       break;
678     case tok::kw_typedef:
679       nextToken();
680       // FIXME: Use the IdentifierTable instead.
681       if (FormatTok->TokenText == "NS_ENUM")
682         parseEnum();
683       break;
684     case tok::kw_struct:
685     case tok::kw_union:
686     case tok::kw_class:
687       parseRecord();
688       // A record declaration or definition is always the start of a structural
689       // element.
690       break;
691     case tok::semi:
692       nextToken();
693       addUnwrappedLine();
694       return;
695     case tok::r_brace:
696       addUnwrappedLine();
697       return;
698     case tok::l_paren:
699       parseParens();
700       break;
701     case tok::caret:
702       nextToken();
703       if (FormatTok->Tok.isAnyIdentifier() ||
704           FormatTok->isSimpleTypeSpecifier())
705         nextToken();
706       if (FormatTok->is(tok::l_paren))
707         parseParens();
708       if (FormatTok->is(tok::l_brace))
709         parseChildBlock();
710       break;
711     case tok::l_brace:
712       if (!tryToParseBracedList()) {
713         // A block outside of parentheses must be the last part of a
714         // structural element.
715         // FIXME: Figure out cases where this is not true, and add projections
716         // for them (the one we know is missing are lambdas).
717         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
718           addUnwrappedLine();
719         FormatTok->Type = TT_FunctionLBrace;
720         parseBlock(/*MustBeDeclaration=*/false);
721         addUnwrappedLine();
722         return;
723       }
724       // Otherwise this was a braced init list, and the structural
725       // element continues.
726       break;
727     case tok::identifier: {
728       StringRef Text = FormatTok->TokenText;
729       nextToken();
730       if (Line->Tokens.size() == 1) {
731         if (FormatTok->Tok.is(tok::colon)) {
732           parseLabel();
733           return;
734         }
735         // Recognize function-like macro usages without trailing semicolon.
736         if (FormatTok->Tok.is(tok::l_paren)) {
737           parseParens();
738           if (FormatTok->NewlinesBefore > 0 &&
739               tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
740             addUnwrappedLine();
741             return;
742           }
743         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
744                    Text == Text.upper()) {
745           // Recognize free-standing macros like Q_OBJECT.
746           addUnwrappedLine();
747           return;
748         }
749       }
750       break;
751     }
752     case tok::equal:
753       nextToken();
754       if (FormatTok->Tok.is(tok::l_brace)) {
755         parseBracedList();
756       }
757       break;
758     case tok::l_square:
759       parseSquare();
760       break;
761     default:
762       nextToken();
763       break;
764     }
765   } while (!eof());
766 }
767 
768 bool UnwrappedLineParser::tryToParseLambda() {
769   // FIXME: This is a dirty way to access the previous token. Find a better
770   // solution.
771   if (!Line->Tokens.empty() &&
772       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator) ||
773        Line->Tokens.back().Tok->closesScope() ||
774        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
775     nextToken();
776     return false;
777   }
778   assert(FormatTok->is(tok::l_square));
779   FormatToken &LSquare = *FormatTok;
780   if (!tryToParseLambdaIntroducer())
781     return false;
782 
783   while (FormatTok->isNot(tok::l_brace)) {
784     if (FormatTok->isSimpleTypeSpecifier()) {
785       nextToken();
786       continue;
787     }
788     switch (FormatTok->Tok.getKind()) {
789     case tok::l_brace:
790       break;
791     case tok::l_paren:
792       parseParens();
793       break;
794     case tok::less:
795     case tok::greater:
796     case tok::identifier:
797     case tok::coloncolon:
798     case tok::kw_mutable:
799       nextToken();
800       break;
801     case tok::arrow:
802       FormatTok->Type = TT_TrailingReturnArrow;
803       nextToken();
804       break;
805     default:
806       return true;
807     }
808   }
809   LSquare.Type = TT_LambdaLSquare;
810   parseChildBlock();
811   return true;
812 }
813 
814 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
815   nextToken();
816   if (FormatTok->is(tok::equal)) {
817     nextToken();
818     if (FormatTok->is(tok::r_square)) {
819       nextToken();
820       return true;
821     }
822     if (FormatTok->isNot(tok::comma))
823       return false;
824     nextToken();
825   } else if (FormatTok->is(tok::amp)) {
826     nextToken();
827     if (FormatTok->is(tok::r_square)) {
828       nextToken();
829       return true;
830     }
831     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
832       return false;
833     }
834     if (FormatTok->is(tok::comma))
835       nextToken();
836   } else if (FormatTok->is(tok::r_square)) {
837     nextToken();
838     return true;
839   }
840   do {
841     if (FormatTok->is(tok::amp))
842       nextToken();
843     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
844       return false;
845     nextToken();
846     if (FormatTok->is(tok::comma)) {
847       nextToken();
848     } else if (FormatTok->is(tok::r_square)) {
849       nextToken();
850       return true;
851     } else {
852       return false;
853     }
854   } while (!eof());
855   return false;
856 }
857 
858 bool UnwrappedLineParser::tryToParseBracedList() {
859   if (FormatTok->BlockKind == BK_Unknown)
860     calculateBraceTypes();
861   assert(FormatTok->BlockKind != BK_Unknown);
862   if (FormatTok->BlockKind == BK_Block)
863     return false;
864   parseBracedList();
865   return true;
866 }
867 
868 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
869   bool HasError = false;
870   nextToken();
871 
872   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
873   // replace this by using parseAssigmentExpression() inside.
874   do {
875     // FIXME: When we start to support lambdas, we'll want to parse them away
876     // here, otherwise our bail-out scenarios below break. The better solution
877     // might be to just implement a more or less complete expression parser.
878     switch (FormatTok->Tok.getKind()) {
879     case tok::caret:
880       nextToken();
881       if (FormatTok->is(tok::l_brace)) {
882         parseChildBlock();
883       }
884       break;
885     case tok::l_square:
886       tryToParseLambda();
887       break;
888     case tok::l_brace:
889       // Assume there are no blocks inside a braced init list apart
890       // from the ones we explicitly parse out (like lambdas).
891       FormatTok->BlockKind = BK_BracedInit;
892       parseBracedList();
893       break;
894     case tok::r_brace:
895       nextToken();
896       return !HasError;
897     case tok::semi:
898       HasError = true;
899       if (!ContinueOnSemicolons)
900         return !HasError;
901       nextToken();
902       break;
903     case tok::comma:
904       nextToken();
905       break;
906     default:
907       nextToken();
908       break;
909     }
910   } while (!eof());
911   return false;
912 }
913 
914 void UnwrappedLineParser::parseParens() {
915   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
916   nextToken();
917   do {
918     switch (FormatTok->Tok.getKind()) {
919     case tok::l_paren:
920       parseParens();
921       break;
922     case tok::r_paren:
923       nextToken();
924       return;
925     case tok::r_brace:
926       // A "}" inside parenthesis is an error if there wasn't a matching "{".
927       return;
928     case tok::l_square:
929       tryToParseLambda();
930       break;
931     case tok::l_brace: {
932       if (!tryToParseBracedList()) {
933         parseChildBlock();
934       }
935       break;
936     }
937     case tok::at:
938       nextToken();
939       if (FormatTok->Tok.is(tok::l_brace))
940         parseBracedList();
941       break;
942     default:
943       nextToken();
944       break;
945     }
946   } while (!eof());
947 }
948 
949 void UnwrappedLineParser::parseSquare() {
950   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
951   if (tryToParseLambda())
952     return;
953   do {
954     switch (FormatTok->Tok.getKind()) {
955     case tok::l_paren:
956       parseParens();
957       break;
958     case tok::r_square:
959       nextToken();
960       return;
961     case tok::r_brace:
962       // A "}" inside parenthesis is an error if there wasn't a matching "{".
963       return;
964     case tok::l_square:
965       parseSquare();
966       break;
967     case tok::l_brace: {
968       if (!tryToParseBracedList()) {
969         parseChildBlock();
970       }
971       break;
972     }
973     case tok::at:
974       nextToken();
975       if (FormatTok->Tok.is(tok::l_brace))
976         parseBracedList();
977       break;
978     default:
979       nextToken();
980       break;
981     }
982   } while (!eof());
983 }
984 
985 void UnwrappedLineParser::parseIfThenElse() {
986   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
987   nextToken();
988   if (FormatTok->Tok.is(tok::l_paren))
989     parseParens();
990   bool NeedsUnwrappedLine = false;
991   if (FormatTok->Tok.is(tok::l_brace)) {
992     CompoundStatementIndenter Indenter(this, Style, Line->Level);
993     parseBlock(/*MustBeDeclaration=*/false);
994     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
995         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
996       addUnwrappedLine();
997     } else {
998       NeedsUnwrappedLine = true;
999     }
1000   } else {
1001     addUnwrappedLine();
1002     ++Line->Level;
1003     parseStructuralElement();
1004     --Line->Level;
1005   }
1006   if (FormatTok->Tok.is(tok::kw_else)) {
1007     nextToken();
1008     if (FormatTok->Tok.is(tok::l_brace)) {
1009       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1010       parseBlock(/*MustBeDeclaration=*/false);
1011       addUnwrappedLine();
1012     } else if (FormatTok->Tok.is(tok::kw_if)) {
1013       parseIfThenElse();
1014     } else {
1015       addUnwrappedLine();
1016       ++Line->Level;
1017       parseStructuralElement();
1018       --Line->Level;
1019     }
1020   } else if (NeedsUnwrappedLine) {
1021     addUnwrappedLine();
1022   }
1023 }
1024 
1025 void UnwrappedLineParser::parseNamespace() {
1026   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1027   nextToken();
1028   if (FormatTok->Tok.is(tok::identifier))
1029     nextToken();
1030   if (FormatTok->Tok.is(tok::l_brace)) {
1031     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1032         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1033         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1034       addUnwrappedLine();
1035 
1036     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1037                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1038                      DeclarationScopeStack.size() > 1);
1039     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1040     // Munch the semicolon after a namespace. This is more common than one would
1041     // think. Puttin the semicolon into its own line is very ugly.
1042     if (FormatTok->Tok.is(tok::semi))
1043       nextToken();
1044     addUnwrappedLine();
1045   }
1046   // FIXME: Add error handling.
1047 }
1048 
1049 void UnwrappedLineParser::parseForOrWhileLoop() {
1050   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1051           FormatTok->IsForEachMacro) &&
1052          "'for', 'while' or foreach macro expected");
1053   nextToken();
1054   if (FormatTok->Tok.is(tok::l_paren))
1055     parseParens();
1056   if (FormatTok->Tok.is(tok::l_brace)) {
1057     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1058     parseBlock(/*MustBeDeclaration=*/false);
1059     addUnwrappedLine();
1060   } else {
1061     addUnwrappedLine();
1062     ++Line->Level;
1063     parseStructuralElement();
1064     --Line->Level;
1065   }
1066 }
1067 
1068 void UnwrappedLineParser::parseDoWhile() {
1069   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1070   nextToken();
1071   if (FormatTok->Tok.is(tok::l_brace)) {
1072     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1073     parseBlock(/*MustBeDeclaration=*/false);
1074     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1075       addUnwrappedLine();
1076   } else {
1077     addUnwrappedLine();
1078     ++Line->Level;
1079     parseStructuralElement();
1080     --Line->Level;
1081   }
1082 
1083   // FIXME: Add error handling.
1084   if (!FormatTok->Tok.is(tok::kw_while)) {
1085     addUnwrappedLine();
1086     return;
1087   }
1088 
1089   nextToken();
1090   parseStructuralElement();
1091 }
1092 
1093 void UnwrappedLineParser::parseLabel() {
1094   nextToken();
1095   unsigned OldLineLevel = Line->Level;
1096   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1097     --Line->Level;
1098   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1099     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1100     parseBlock(/*MustBeDeclaration=*/false);
1101     if (FormatTok->Tok.is(tok::kw_break)) {
1102       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1103       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1104           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1105         addUnwrappedLine();
1106       }
1107       parseStructuralElement();
1108     }
1109     addUnwrappedLine();
1110   } else {
1111     addUnwrappedLine();
1112   }
1113   Line->Level = OldLineLevel;
1114 }
1115 
1116 void UnwrappedLineParser::parseCaseLabel() {
1117   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1118   // FIXME: fix handling of complex expressions here.
1119   do {
1120     nextToken();
1121   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1122   parseLabel();
1123 }
1124 
1125 void UnwrappedLineParser::parseSwitch() {
1126   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1127   nextToken();
1128   if (FormatTok->Tok.is(tok::l_paren))
1129     parseParens();
1130   if (FormatTok->Tok.is(tok::l_brace)) {
1131     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1132     parseBlock(/*MustBeDeclaration=*/false);
1133     addUnwrappedLine();
1134   } else {
1135     addUnwrappedLine();
1136     ++Line->Level;
1137     parseStructuralElement();
1138     --Line->Level;
1139   }
1140 }
1141 
1142 void UnwrappedLineParser::parseAccessSpecifier() {
1143   nextToken();
1144   // Understand Qt's slots.
1145   if (FormatTok->is(tok::identifier) &&
1146       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1147     nextToken();
1148   // Otherwise, we don't know what it is, and we'd better keep the next token.
1149   if (FormatTok->Tok.is(tok::colon))
1150     nextToken();
1151   addUnwrappedLine();
1152 }
1153 
1154 void UnwrappedLineParser::parseEnum() {
1155   if (FormatTok->Tok.is(tok::kw_enum)) {
1156     // Won't be 'enum' for NS_ENUMs.
1157     nextToken();
1158   }
1159   // Eat up enum class ...
1160   if (FormatTok->Tok.is(tok::kw_class) ||
1161       FormatTok->Tok.is(tok::kw_struct))
1162       nextToken();
1163   while (FormatTok->Tok.getIdentifierInfo() ||
1164          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1165     nextToken();
1166     // We can have macros or attributes in between 'enum' and the enum name.
1167     if (FormatTok->Tok.is(tok::l_paren)) {
1168       parseParens();
1169     }
1170     if (FormatTok->Tok.is(tok::identifier))
1171       nextToken();
1172   }
1173   if (FormatTok->Tok.is(tok::l_brace)) {
1174     FormatTok->BlockKind = BK_Block;
1175     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1176     if (HasError) {
1177       if (FormatTok->is(tok::semi))
1178         nextToken();
1179       addUnwrappedLine();
1180     }
1181   }
1182   // We fall through to parsing a structural element afterwards, so that in
1183   // enum A {} n, m;
1184   // "} n, m;" will end up in one unwrapped line.
1185 }
1186 
1187 void UnwrappedLineParser::parseRecord() {
1188   nextToken();
1189   if (FormatTok->Tok.is(tok::identifier) ||
1190       FormatTok->Tok.is(tok::kw___attribute) ||
1191       FormatTok->Tok.is(tok::kw___declspec) ||
1192       FormatTok->Tok.is(tok::kw_alignas)) {
1193     nextToken();
1194     // We can have macros or attributes in between 'class' and the class name.
1195     if (FormatTok->Tok.is(tok::l_paren)) {
1196       parseParens();
1197     }
1198     // The actual identifier can be a nested name specifier, and in macros
1199     // it is often token-pasted.
1200     while (FormatTok->Tok.is(tok::identifier) ||
1201            FormatTok->Tok.is(tok::coloncolon) ||
1202            FormatTok->Tok.is(tok::hashhash))
1203       nextToken();
1204 
1205     // Note that parsing away template declarations here leads to incorrectly
1206     // accepting function declarations as record declarations.
1207     // In general, we cannot solve this problem. Consider:
1208     // class A<int> B() {}
1209     // which can be a function definition or a class definition when B() is a
1210     // macro. If we find enough real-world cases where this is a problem, we
1211     // can parse for the 'template' keyword in the beginning of the statement,
1212     // and thus rule out the record production in case there is no template
1213     // (this would still leave us with an ambiguity between template function
1214     // and class declarations).
1215     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1216       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1217         if (FormatTok->Tok.is(tok::semi))
1218           return;
1219         nextToken();
1220       }
1221     }
1222   }
1223   if (FormatTok->Tok.is(tok::l_brace)) {
1224     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1225         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1226         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1227       addUnwrappedLine();
1228 
1229     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1230                /*MunchSemi=*/false);
1231   }
1232   // We fall through to parsing a structural element afterwards, so
1233   // class A {} n, m;
1234   // will end up in one unwrapped line.
1235 }
1236 
1237 void UnwrappedLineParser::parseObjCProtocolList() {
1238   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1239   do
1240     nextToken();
1241   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1242   nextToken(); // Skip '>'.
1243 }
1244 
1245 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1246   do {
1247     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1248       nextToken();
1249       addUnwrappedLine();
1250       break;
1251     }
1252     if (FormatTok->is(tok::l_brace)) {
1253       parseBlock(/*MustBeDeclaration=*/false);
1254       // In ObjC interfaces, nothing should be following the "}".
1255       addUnwrappedLine();
1256     } else if (FormatTok->is(tok::r_brace)) {
1257       // Ignore stray "}". parseStructuralElement doesn't consume them.
1258       nextToken();
1259       addUnwrappedLine();
1260     } else {
1261       parseStructuralElement();
1262     }
1263   } while (!eof());
1264 }
1265 
1266 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1267   nextToken();
1268   nextToken(); // interface name
1269 
1270   // @interface can be followed by either a base class, or a category.
1271   if (FormatTok->Tok.is(tok::colon)) {
1272     nextToken();
1273     nextToken(); // base class name
1274   } else if (FormatTok->Tok.is(tok::l_paren))
1275     // Skip category, if present.
1276     parseParens();
1277 
1278   if (FormatTok->Tok.is(tok::less))
1279     parseObjCProtocolList();
1280 
1281   // If instance variables are present, keep the '{' on the first line too.
1282   if (FormatTok->Tok.is(tok::l_brace))
1283     parseBlock(/*MustBeDeclaration=*/true);
1284 
1285   // With instance variables, this puts '}' on its own line.  Without instance
1286   // variables, this ends the @interface line.
1287   addUnwrappedLine();
1288 
1289   parseObjCUntilAtEnd();
1290 }
1291 
1292 void UnwrappedLineParser::parseObjCProtocol() {
1293   nextToken();
1294   nextToken(); // protocol name
1295 
1296   if (FormatTok->Tok.is(tok::less))
1297     parseObjCProtocolList();
1298 
1299   // Check for protocol declaration.
1300   if (FormatTok->Tok.is(tok::semi)) {
1301     nextToken();
1302     return addUnwrappedLine();
1303   }
1304 
1305   addUnwrappedLine();
1306   parseObjCUntilAtEnd();
1307 }
1308 
1309 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1310                                                  StringRef Prefix = "") {
1311   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1312                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1313   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1314                                                     E = Line.Tokens.end();
1315        I != E; ++I) {
1316     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1317   }
1318   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1319                                                     E = Line.Tokens.end();
1320        I != E; ++I) {
1321     const UnwrappedLineNode &Node = *I;
1322     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1323              I = Node.Children.begin(),
1324              E = Node.Children.end();
1325          I != E; ++I) {
1326       printDebugInfo(*I, "\nChild: ");
1327     }
1328   }
1329   llvm::dbgs() << "\n";
1330 }
1331 
1332 void UnwrappedLineParser::addUnwrappedLine() {
1333   if (Line->Tokens.empty())
1334     return;
1335   DEBUG({
1336     if (CurrentLines == &Lines)
1337       printDebugInfo(*Line);
1338   });
1339   CurrentLines->push_back(*Line);
1340   Line->Tokens.clear();
1341   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1342     for (SmallVectorImpl<UnwrappedLine>::iterator
1343              I = PreprocessorDirectives.begin(),
1344              E = PreprocessorDirectives.end();
1345          I != E; ++I) {
1346       CurrentLines->push_back(*I);
1347     }
1348     PreprocessorDirectives.clear();
1349   }
1350 }
1351 
1352 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1353 
1354 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1355   bool JustComments = Line->Tokens.empty();
1356   for (SmallVectorImpl<FormatToken *>::const_iterator
1357            I = CommentsBeforeNextToken.begin(),
1358            E = CommentsBeforeNextToken.end();
1359        I != E; ++I) {
1360     if ((*I)->NewlinesBefore && JustComments) {
1361       addUnwrappedLine();
1362     }
1363     pushToken(*I);
1364   }
1365   if (NewlineBeforeNext && JustComments) {
1366     addUnwrappedLine();
1367   }
1368   CommentsBeforeNextToken.clear();
1369 }
1370 
1371 void UnwrappedLineParser::nextToken() {
1372   if (eof())
1373     return;
1374   flushComments(FormatTok->NewlinesBefore > 0);
1375   pushToken(FormatTok);
1376   readToken();
1377 }
1378 
1379 void UnwrappedLineParser::readToken() {
1380   bool CommentsInCurrentLine = true;
1381   do {
1382     FormatTok = Tokens->getNextToken();
1383     assert(FormatTok);
1384     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1385            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1386       // If there is an unfinished unwrapped line, we flush the preprocessor
1387       // directives only after that unwrapped line was finished later.
1388       bool SwitchToPreprocessorLines =
1389           !Line->Tokens.empty() && CurrentLines == &Lines;
1390       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1391       // Comments stored before the preprocessor directive need to be output
1392       // before the preprocessor directive, at the same level as the
1393       // preprocessor directive, as we consider them to apply to the directive.
1394       flushComments(FormatTok->NewlinesBefore > 0);
1395       parsePPDirective();
1396     }
1397 
1398     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1399         !Line->InPPDirective) {
1400       continue;
1401     }
1402 
1403     if (!FormatTok->Tok.is(tok::comment))
1404       return;
1405     if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
1406       CommentsInCurrentLine = false;
1407     }
1408     if (CommentsInCurrentLine) {
1409       pushToken(FormatTok);
1410     } else {
1411       CommentsBeforeNextToken.push_back(FormatTok);
1412     }
1413   } while (!eof());
1414 }
1415 
1416 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1417   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1418   if (MustBreakBeforeNextToken) {
1419     Line->Tokens.back().Tok->MustBreakBefore = true;
1420     MustBreakBeforeNextToken = false;
1421   }
1422 }
1423 
1424 } // end namespace format
1425 } // end namespace clang
1426