1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "llvm/Support/Debug.h"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.release();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   FormatToken *getNextToken() override {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   unsigned getPosition() override {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   FormatToken *setPosition(unsigned P) override {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           bool ProbablyBracedList = false;
339           if (Style.Language == FormatStyle::LK_Proto) {
340             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
341           } else {
342             // If there is a comma, semicolon or right paren after the closing
343             // brace, we assume this is a braced initializer list.  Note that
344             // regardless how we mark inner braces here, we will overwrite the
345             // BlockKind later if we parse a braced list (where all blocks
346             // inside are by default braced lists), or when we explicitly detect
347             // blocks (for example while parsing lambdas).
348             //
349             // We exclude + and - as they can be ObjC visibility modifiers.
350             ProbablyBracedList =
351                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
352                                  tok::r_paren, tok::r_square, tok::l_brace) ||
353                 (NextTok->isBinaryOperator() &&
354                  !NextTok->isOneOf(tok::plus, tok::minus));
355           }
356           if (ProbablyBracedList) {
357             Tok->BlockKind = BK_BracedInit;
358             LBraceStack.back()->BlockKind = BK_BracedInit;
359           } else {
360             Tok->BlockKind = BK_Block;
361             LBraceStack.back()->BlockKind = BK_Block;
362           }
363         }
364         LBraceStack.pop_back();
365       }
366       break;
367     case tok::at:
368     case tok::semi:
369     case tok::kw_if:
370     case tok::kw_while:
371     case tok::kw_for:
372     case tok::kw_switch:
373     case tok::kw_try:
374       if (!LBraceStack.empty())
375         LBraceStack.back()->BlockKind = BK_Block;
376       break;
377     default:
378       break;
379     }
380     Tok = NextTok;
381     Position += ReadTokens;
382   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
383   // Assume other blocks for all unclosed opening braces.
384   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
385     if (LBraceStack[i]->BlockKind == BK_Unknown)
386       LBraceStack[i]->BlockKind = BK_Block;
387   }
388 
389   FormatTok = Tokens->setPosition(StoredPosition);
390 }
391 
392 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
393                                      bool MunchSemi) {
394   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
395   unsigned InitialLevel = Line->Level;
396   nextToken();
397 
398   addUnwrappedLine();
399 
400   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
401                                           MustBeDeclaration);
402   if (AddLevel)
403     ++Line->Level;
404   parseLevel(/*HasOpeningBrace=*/true);
405 
406   if (!FormatTok->Tok.is(tok::r_brace)) {
407     Line->Level = InitialLevel;
408     StructuralError = true;
409     return;
410   }
411 
412   nextToken(); // Munch the closing brace.
413   if (MunchSemi && FormatTok->Tok.is(tok::semi))
414     nextToken();
415   Line->Level = InitialLevel;
416 }
417 
418 void UnwrappedLineParser::parseChildBlock() {
419   FormatTok->BlockKind = BK_Block;
420   nextToken();
421   {
422     ScopedLineState LineState(*this);
423     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
424                                             /*MustBeDeclaration=*/false);
425     Line->Level += 1;
426     parseLevel(/*HasOpeningBrace=*/true);
427     Line->Level -= 1;
428   }
429   nextToken();
430 }
431 
432 void UnwrappedLineParser::parsePPDirective() {
433   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
434   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
435   nextToken();
436 
437   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
438     parsePPUnknown();
439     return;
440   }
441 
442   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
443   case tok::pp_define:
444     parsePPDefine();
445     return;
446   case tok::pp_if:
447     parsePPIf(/*IfDef=*/false);
448     break;
449   case tok::pp_ifdef:
450   case tok::pp_ifndef:
451     parsePPIf(/*IfDef=*/true);
452     break;
453   case tok::pp_else:
454     parsePPElse();
455     break;
456   case tok::pp_elif:
457     parsePPElIf();
458     break;
459   case tok::pp_endif:
460     parsePPEndIf();
461     break;
462   default:
463     parsePPUnknown();
464     break;
465   }
466 }
467 
468 void UnwrappedLineParser::pushPPConditional() {
469   if (!PPStack.empty() && PPStack.back() == PP_Unreachable)
470     PPStack.push_back(PP_Unreachable);
471   else
472     PPStack.push_back(PP_Conditional);
473 }
474 
475 void UnwrappedLineParser::parsePPIf(bool IfDef) {
476   ++PPBranchLevel;
477   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
478   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
479     PPLevelBranchIndex.push_back(0);
480     PPLevelBranchCount.push_back(0);
481   }
482   PPChainBranchIndex.push(0);
483   nextToken();
484   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
485                          StringRef(FormatTok->Tok.getLiteralData(),
486                                    FormatTok->Tok.getLength()) == "0") ||
487                         FormatTok->Tok.is(tok::kw_false);
488   if ((!IfDef && IsLiteralFalse) || PPLevelBranchIndex[PPBranchLevel] > 0) {
489     PPStack.push_back(PP_Unreachable);
490   } else {
491     pushPPConditional();
492   }
493   parsePPUnknown();
494 }
495 
496 void UnwrappedLineParser::parsePPElse() {
497   if (!PPStack.empty())
498     PPStack.pop_back();
499   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
500   if (!PPChainBranchIndex.empty())
501     ++PPChainBranchIndex.top();
502   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
503       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top()) {
504     PPStack.push_back(PP_Unreachable);
505   } else {
506     pushPPConditional();
507   }
508   parsePPUnknown();
509 }
510 
511 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
512 
513 void UnwrappedLineParser::parsePPEndIf() {
514   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
515   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
516     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
517       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
518     }
519   }
520   // Guard against #endif's without #if.
521   if (PPBranchLevel > 0)
522     --PPBranchLevel;
523   if (!PPChainBranchIndex.empty())
524     PPChainBranchIndex.pop();
525   if (!PPStack.empty())
526     PPStack.pop_back();
527   parsePPUnknown();
528 }
529 
530 void UnwrappedLineParser::parsePPDefine() {
531   nextToken();
532 
533   if (FormatTok->Tok.getKind() != tok::identifier) {
534     parsePPUnknown();
535     return;
536   }
537   nextToken();
538   if (FormatTok->Tok.getKind() == tok::l_paren &&
539       FormatTok->WhitespaceRange.getBegin() ==
540           FormatTok->WhitespaceRange.getEnd()) {
541     parseParens();
542   }
543   addUnwrappedLine();
544   Line->Level = 1;
545 
546   // Errors during a preprocessor directive can only affect the layout of the
547   // preprocessor directive, and thus we ignore them. An alternative approach
548   // would be to use the same approach we use on the file level (no
549   // re-indentation if there was a structural error) within the macro
550   // definition.
551   parseFile();
552 }
553 
554 void UnwrappedLineParser::parsePPUnknown() {
555   do {
556     nextToken();
557   } while (!eof());
558   addUnwrappedLine();
559 }
560 
561 // Here we blacklist certain tokens that are not usually the first token in an
562 // unwrapped line. This is used in attempt to distinguish macro calls without
563 // trailing semicolons from other constructs split to several lines.
564 bool tokenCanStartNewLine(clang::Token Tok) {
565   // Semicolon can be a null-statement, l_square can be a start of a macro or
566   // a C++11 attribute, but this doesn't seem to be common.
567   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
568          Tok.isNot(tok::l_square) &&
569          // Tokens that can only be used as binary operators and a part of
570          // overloaded operator names.
571          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
572          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
573          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
574          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
575          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
576          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
577          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
578          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
579          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
580          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
581          Tok.isNot(tok::lesslessequal) &&
582          // Colon is used in labels, base class lists, initializer lists,
583          // range-based for loops, ternary operator, but should never be the
584          // first token in an unwrapped line.
585          Tok.isNot(tok::colon);
586 }
587 
588 void UnwrappedLineParser::parseStructuralElement() {
589   assert(!FormatTok->Tok.is(tok::l_brace));
590   switch (FormatTok->Tok.getKind()) {
591   case tok::at:
592     nextToken();
593     if (FormatTok->Tok.is(tok::l_brace)) {
594       parseBracedList();
595       break;
596     }
597     switch (FormatTok->Tok.getObjCKeywordID()) {
598     case tok::objc_public:
599     case tok::objc_protected:
600     case tok::objc_package:
601     case tok::objc_private:
602       return parseAccessSpecifier();
603     case tok::objc_interface:
604     case tok::objc_implementation:
605       return parseObjCInterfaceOrImplementation();
606     case tok::objc_protocol:
607       return parseObjCProtocol();
608     case tok::objc_end:
609       return; // Handled by the caller.
610     case tok::objc_optional:
611     case tok::objc_required:
612       nextToken();
613       addUnwrappedLine();
614       return;
615     default:
616       break;
617     }
618     break;
619   case tok::kw_namespace:
620     parseNamespace();
621     return;
622   case tok::kw_inline:
623     nextToken();
624     if (FormatTok->Tok.is(tok::kw_namespace)) {
625       parseNamespace();
626       return;
627     }
628     break;
629   case tok::kw_public:
630   case tok::kw_protected:
631   case tok::kw_private:
632     parseAccessSpecifier();
633     return;
634   case tok::kw_if:
635     parseIfThenElse();
636     return;
637   case tok::kw_for:
638   case tok::kw_while:
639     parseForOrWhileLoop();
640     return;
641   case tok::kw_do:
642     parseDoWhile();
643     return;
644   case tok::kw_switch:
645     parseSwitch();
646     return;
647   case tok::kw_default:
648     nextToken();
649     parseLabel();
650     return;
651   case tok::kw_case:
652     parseCaseLabel();
653     return;
654   case tok::kw_extern:
655     nextToken();
656     if (FormatTok->Tok.is(tok::string_literal)) {
657       nextToken();
658       if (FormatTok->Tok.is(tok::l_brace)) {
659         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
660         addUnwrappedLine();
661         return;
662       }
663     }
664     break;
665   case tok::identifier:
666     if (FormatTok->IsForEachMacro) {
667       parseForOrWhileLoop();
668       return;
669     }
670     // In all other cases, parse the declaration.
671     break;
672   default:
673     break;
674   }
675   do {
676     switch (FormatTok->Tok.getKind()) {
677     case tok::at:
678       nextToken();
679       if (FormatTok->Tok.is(tok::l_brace))
680         parseBracedList();
681       break;
682     case tok::kw_enum:
683       parseEnum();
684       break;
685     case tok::kw_typedef:
686       nextToken();
687       // FIXME: Use the IdentifierTable instead.
688       if (FormatTok->TokenText == "NS_ENUM")
689         parseEnum();
690       break;
691     case tok::kw_struct:
692     case tok::kw_union:
693     case tok::kw_class:
694       parseRecord();
695       // A record declaration or definition is always the start of a structural
696       // element.
697       break;
698     case tok::semi:
699       nextToken();
700       addUnwrappedLine();
701       return;
702     case tok::r_brace:
703       addUnwrappedLine();
704       return;
705     case tok::l_paren:
706       parseParens();
707       break;
708     case tok::caret:
709       nextToken();
710       if (FormatTok->Tok.isAnyIdentifier() ||
711           FormatTok->isSimpleTypeSpecifier())
712         nextToken();
713       if (FormatTok->is(tok::l_paren))
714         parseParens();
715       if (FormatTok->is(tok::l_brace))
716         parseChildBlock();
717       break;
718     case tok::l_brace:
719       if (!tryToParseBracedList()) {
720         // A block outside of parentheses must be the last part of a
721         // structural element.
722         // FIXME: Figure out cases where this is not true, and add projections
723         // for them (the one we know is missing are lambdas).
724         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
725           addUnwrappedLine();
726         FormatTok->Type = TT_FunctionLBrace;
727         parseBlock(/*MustBeDeclaration=*/false);
728         addUnwrappedLine();
729         return;
730       }
731       // Otherwise this was a braced init list, and the structural
732       // element continues.
733       break;
734     case tok::identifier: {
735       StringRef Text = FormatTok->TokenText;
736       nextToken();
737       if (Line->Tokens.size() == 1) {
738         if (FormatTok->Tok.is(tok::colon)) {
739           parseLabel();
740           return;
741         }
742         // Recognize function-like macro usages without trailing semicolon.
743         if (FormatTok->Tok.is(tok::l_paren)) {
744           parseParens();
745           if (FormatTok->NewlinesBefore > 0 &&
746               tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
747             addUnwrappedLine();
748             return;
749           }
750         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
751                    Text == Text.upper()) {
752           // Recognize free-standing macros like Q_OBJECT.
753           addUnwrappedLine();
754           return;
755         }
756       }
757       break;
758     }
759     case tok::equal:
760       nextToken();
761       if (FormatTok->Tok.is(tok::l_brace)) {
762         parseBracedList();
763       }
764       break;
765     case tok::l_square:
766       parseSquare();
767       break;
768     default:
769       nextToken();
770       break;
771     }
772   } while (!eof());
773 }
774 
775 bool UnwrappedLineParser::tryToParseLambda() {
776   // FIXME: This is a dirty way to access the previous token. Find a better
777   // solution.
778   if (!Line->Tokens.empty() &&
779       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator) ||
780        Line->Tokens.back().Tok->closesScope() ||
781        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
782     nextToken();
783     return false;
784   }
785   assert(FormatTok->is(tok::l_square));
786   FormatToken &LSquare = *FormatTok;
787   if (!tryToParseLambdaIntroducer())
788     return false;
789 
790   while (FormatTok->isNot(tok::l_brace)) {
791     if (FormatTok->isSimpleTypeSpecifier()) {
792       nextToken();
793       continue;
794     }
795     switch (FormatTok->Tok.getKind()) {
796     case tok::l_brace:
797       break;
798     case tok::l_paren:
799       parseParens();
800       break;
801     case tok::less:
802     case tok::greater:
803     case tok::identifier:
804     case tok::coloncolon:
805     case tok::kw_mutable:
806       nextToken();
807       break;
808     case tok::arrow:
809       FormatTok->Type = TT_TrailingReturnArrow;
810       nextToken();
811       break;
812     default:
813       return true;
814     }
815   }
816   LSquare.Type = TT_LambdaLSquare;
817   parseChildBlock();
818   return true;
819 }
820 
821 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
822   nextToken();
823   if (FormatTok->is(tok::equal)) {
824     nextToken();
825     if (FormatTok->is(tok::r_square)) {
826       nextToken();
827       return true;
828     }
829     if (FormatTok->isNot(tok::comma))
830       return false;
831     nextToken();
832   } else if (FormatTok->is(tok::amp)) {
833     nextToken();
834     if (FormatTok->is(tok::r_square)) {
835       nextToken();
836       return true;
837     }
838     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
839       return false;
840     }
841     if (FormatTok->is(tok::comma))
842       nextToken();
843   } else if (FormatTok->is(tok::r_square)) {
844     nextToken();
845     return true;
846   }
847   do {
848     if (FormatTok->is(tok::amp))
849       nextToken();
850     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
851       return false;
852     nextToken();
853     if (FormatTok->is(tok::comma)) {
854       nextToken();
855     } else if (FormatTok->is(tok::r_square)) {
856       nextToken();
857       return true;
858     } else {
859       return false;
860     }
861   } while (!eof());
862   return false;
863 }
864 
865 bool UnwrappedLineParser::tryToParseBracedList() {
866   if (FormatTok->BlockKind == BK_Unknown)
867     calculateBraceTypes();
868   assert(FormatTok->BlockKind != BK_Unknown);
869   if (FormatTok->BlockKind == BK_Block)
870     return false;
871   parseBracedList();
872   return true;
873 }
874 
875 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
876   bool HasError = false;
877   nextToken();
878 
879   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
880   // replace this by using parseAssigmentExpression() inside.
881   do {
882     // FIXME: When we start to support lambdas, we'll want to parse them away
883     // here, otherwise our bail-out scenarios below break. The better solution
884     // might be to just implement a more or less complete expression parser.
885     switch (FormatTok->Tok.getKind()) {
886     case tok::caret:
887       nextToken();
888       if (FormatTok->is(tok::l_brace)) {
889         parseChildBlock();
890       }
891       break;
892     case tok::l_square:
893       tryToParseLambda();
894       break;
895     case tok::l_brace:
896       // Assume there are no blocks inside a braced init list apart
897       // from the ones we explicitly parse out (like lambdas).
898       FormatTok->BlockKind = BK_BracedInit;
899       parseBracedList();
900       break;
901     case tok::r_brace:
902       nextToken();
903       return !HasError;
904     case tok::semi:
905       HasError = true;
906       if (!ContinueOnSemicolons)
907         return !HasError;
908       nextToken();
909       break;
910     case tok::comma:
911       nextToken();
912       break;
913     default:
914       nextToken();
915       break;
916     }
917   } while (!eof());
918   return false;
919 }
920 
921 void UnwrappedLineParser::parseParens() {
922   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
923   nextToken();
924   do {
925     switch (FormatTok->Tok.getKind()) {
926     case tok::l_paren:
927       parseParens();
928       break;
929     case tok::r_paren:
930       nextToken();
931       return;
932     case tok::r_brace:
933       // A "}" inside parenthesis is an error if there wasn't a matching "{".
934       return;
935     case tok::l_square:
936       tryToParseLambda();
937       break;
938     case tok::l_brace: {
939       if (!tryToParseBracedList()) {
940         parseChildBlock();
941       }
942       break;
943     }
944     case tok::at:
945       nextToken();
946       if (FormatTok->Tok.is(tok::l_brace))
947         parseBracedList();
948       break;
949     default:
950       nextToken();
951       break;
952     }
953   } while (!eof());
954 }
955 
956 void UnwrappedLineParser::parseSquare() {
957   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
958   if (tryToParseLambda())
959     return;
960   do {
961     switch (FormatTok->Tok.getKind()) {
962     case tok::l_paren:
963       parseParens();
964       break;
965     case tok::r_square:
966       nextToken();
967       return;
968     case tok::r_brace:
969       // A "}" inside parenthesis is an error if there wasn't a matching "{".
970       return;
971     case tok::l_square:
972       parseSquare();
973       break;
974     case tok::l_brace: {
975       if (!tryToParseBracedList()) {
976         parseChildBlock();
977       }
978       break;
979     }
980     case tok::at:
981       nextToken();
982       if (FormatTok->Tok.is(tok::l_brace))
983         parseBracedList();
984       break;
985     default:
986       nextToken();
987       break;
988     }
989   } while (!eof());
990 }
991 
992 void UnwrappedLineParser::parseIfThenElse() {
993   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
994   nextToken();
995   if (FormatTok->Tok.is(tok::l_paren))
996     parseParens();
997   bool NeedsUnwrappedLine = false;
998   if (FormatTok->Tok.is(tok::l_brace)) {
999     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1000     parseBlock(/*MustBeDeclaration=*/false);
1001     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1002         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1003       addUnwrappedLine();
1004     } else {
1005       NeedsUnwrappedLine = true;
1006     }
1007   } else {
1008     addUnwrappedLine();
1009     ++Line->Level;
1010     parseStructuralElement();
1011     --Line->Level;
1012   }
1013   if (FormatTok->Tok.is(tok::kw_else)) {
1014     nextToken();
1015     if (FormatTok->Tok.is(tok::l_brace)) {
1016       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1017       parseBlock(/*MustBeDeclaration=*/false);
1018       addUnwrappedLine();
1019     } else if (FormatTok->Tok.is(tok::kw_if)) {
1020       parseIfThenElse();
1021     } else {
1022       addUnwrappedLine();
1023       ++Line->Level;
1024       parseStructuralElement();
1025       --Line->Level;
1026     }
1027   } else if (NeedsUnwrappedLine) {
1028     addUnwrappedLine();
1029   }
1030 }
1031 
1032 void UnwrappedLineParser::parseNamespace() {
1033   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1034   nextToken();
1035   if (FormatTok->Tok.is(tok::identifier))
1036     nextToken();
1037   if (FormatTok->Tok.is(tok::l_brace)) {
1038     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1039         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1040         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1041       addUnwrappedLine();
1042 
1043     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1044                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1045                      DeclarationScopeStack.size() > 1);
1046     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1047     // Munch the semicolon after a namespace. This is more common than one would
1048     // think. Puttin the semicolon into its own line is very ugly.
1049     if (FormatTok->Tok.is(tok::semi))
1050       nextToken();
1051     addUnwrappedLine();
1052   }
1053   // FIXME: Add error handling.
1054 }
1055 
1056 void UnwrappedLineParser::parseForOrWhileLoop() {
1057   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1058           FormatTok->IsForEachMacro) &&
1059          "'for', 'while' or foreach macro expected");
1060   nextToken();
1061   if (FormatTok->Tok.is(tok::l_paren))
1062     parseParens();
1063   if (FormatTok->Tok.is(tok::l_brace)) {
1064     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1065     parseBlock(/*MustBeDeclaration=*/false);
1066     addUnwrappedLine();
1067   } else {
1068     addUnwrappedLine();
1069     ++Line->Level;
1070     parseStructuralElement();
1071     --Line->Level;
1072   }
1073 }
1074 
1075 void UnwrappedLineParser::parseDoWhile() {
1076   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1077   nextToken();
1078   if (FormatTok->Tok.is(tok::l_brace)) {
1079     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1080     parseBlock(/*MustBeDeclaration=*/false);
1081     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1082       addUnwrappedLine();
1083   } else {
1084     addUnwrappedLine();
1085     ++Line->Level;
1086     parseStructuralElement();
1087     --Line->Level;
1088   }
1089 
1090   // FIXME: Add error handling.
1091   if (!FormatTok->Tok.is(tok::kw_while)) {
1092     addUnwrappedLine();
1093     return;
1094   }
1095 
1096   nextToken();
1097   parseStructuralElement();
1098 }
1099 
1100 void UnwrappedLineParser::parseLabel() {
1101   nextToken();
1102   unsigned OldLineLevel = Line->Level;
1103   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1104     --Line->Level;
1105   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1106     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1107     parseBlock(/*MustBeDeclaration=*/false);
1108     if (FormatTok->Tok.is(tok::kw_break)) {
1109       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1110       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1111           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1112         addUnwrappedLine();
1113       }
1114       parseStructuralElement();
1115     }
1116     addUnwrappedLine();
1117   } else {
1118     addUnwrappedLine();
1119   }
1120   Line->Level = OldLineLevel;
1121 }
1122 
1123 void UnwrappedLineParser::parseCaseLabel() {
1124   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1125   // FIXME: fix handling of complex expressions here.
1126   do {
1127     nextToken();
1128   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1129   parseLabel();
1130 }
1131 
1132 void UnwrappedLineParser::parseSwitch() {
1133   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1134   nextToken();
1135   if (FormatTok->Tok.is(tok::l_paren))
1136     parseParens();
1137   if (FormatTok->Tok.is(tok::l_brace)) {
1138     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1139     parseBlock(/*MustBeDeclaration=*/false);
1140     addUnwrappedLine();
1141   } else {
1142     addUnwrappedLine();
1143     ++Line->Level;
1144     parseStructuralElement();
1145     --Line->Level;
1146   }
1147 }
1148 
1149 void UnwrappedLineParser::parseAccessSpecifier() {
1150   nextToken();
1151   // Understand Qt's slots.
1152   if (FormatTok->is(tok::identifier) &&
1153       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1154     nextToken();
1155   // Otherwise, we don't know what it is, and we'd better keep the next token.
1156   if (FormatTok->Tok.is(tok::colon))
1157     nextToken();
1158   addUnwrappedLine();
1159 }
1160 
1161 void UnwrappedLineParser::parseEnum() {
1162   if (FormatTok->Tok.is(tok::kw_enum)) {
1163     // Won't be 'enum' for NS_ENUMs.
1164     nextToken();
1165   }
1166   // Eat up enum class ...
1167   if (FormatTok->Tok.is(tok::kw_class) ||
1168       FormatTok->Tok.is(tok::kw_struct))
1169       nextToken();
1170   while (FormatTok->Tok.getIdentifierInfo() ||
1171          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1172     nextToken();
1173     // We can have macros or attributes in between 'enum' and the enum name.
1174     if (FormatTok->Tok.is(tok::l_paren)) {
1175       parseParens();
1176     }
1177     if (FormatTok->Tok.is(tok::identifier))
1178       nextToken();
1179   }
1180   if (FormatTok->Tok.is(tok::l_brace)) {
1181     FormatTok->BlockKind = BK_Block;
1182     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1183     if (HasError) {
1184       if (FormatTok->is(tok::semi))
1185         nextToken();
1186       addUnwrappedLine();
1187     }
1188   }
1189   // We fall through to parsing a structural element afterwards, so that in
1190   // enum A {} n, m;
1191   // "} n, m;" will end up in one unwrapped line.
1192 }
1193 
1194 void UnwrappedLineParser::parseRecord() {
1195   nextToken();
1196   if (FormatTok->Tok.is(tok::identifier) ||
1197       FormatTok->Tok.is(tok::kw___attribute) ||
1198       FormatTok->Tok.is(tok::kw___declspec) ||
1199       FormatTok->Tok.is(tok::kw_alignas)) {
1200     nextToken();
1201     // We can have macros or attributes in between 'class' and the class name.
1202     if (FormatTok->Tok.is(tok::l_paren)) {
1203       parseParens();
1204     }
1205     // The actual identifier can be a nested name specifier, and in macros
1206     // it is often token-pasted.
1207     while (FormatTok->Tok.is(tok::identifier) ||
1208            FormatTok->Tok.is(tok::coloncolon) ||
1209            FormatTok->Tok.is(tok::hashhash))
1210       nextToken();
1211 
1212     // Note that parsing away template declarations here leads to incorrectly
1213     // accepting function declarations as record declarations.
1214     // In general, we cannot solve this problem. Consider:
1215     // class A<int> B() {}
1216     // which can be a function definition or a class definition when B() is a
1217     // macro. If we find enough real-world cases where this is a problem, we
1218     // can parse for the 'template' keyword in the beginning of the statement,
1219     // and thus rule out the record production in case there is no template
1220     // (this would still leave us with an ambiguity between template function
1221     // and class declarations).
1222     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1223       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1224         if (FormatTok->Tok.is(tok::semi))
1225           return;
1226         nextToken();
1227       }
1228     }
1229   }
1230   if (FormatTok->Tok.is(tok::l_brace)) {
1231     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1232         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1233         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1234       addUnwrappedLine();
1235 
1236     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1237                /*MunchSemi=*/false);
1238   }
1239   // We fall through to parsing a structural element afterwards, so
1240   // class A {} n, m;
1241   // will end up in one unwrapped line.
1242 }
1243 
1244 void UnwrappedLineParser::parseObjCProtocolList() {
1245   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1246   do
1247     nextToken();
1248   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1249   nextToken(); // Skip '>'.
1250 }
1251 
1252 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1253   do {
1254     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1255       nextToken();
1256       addUnwrappedLine();
1257       break;
1258     }
1259     if (FormatTok->is(tok::l_brace)) {
1260       parseBlock(/*MustBeDeclaration=*/false);
1261       // In ObjC interfaces, nothing should be following the "}".
1262       addUnwrappedLine();
1263     } else if (FormatTok->is(tok::r_brace)) {
1264       // Ignore stray "}". parseStructuralElement doesn't consume them.
1265       nextToken();
1266       addUnwrappedLine();
1267     } else {
1268       parseStructuralElement();
1269     }
1270   } while (!eof());
1271 }
1272 
1273 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1274   nextToken();
1275   nextToken(); // interface name
1276 
1277   // @interface can be followed by either a base class, or a category.
1278   if (FormatTok->Tok.is(tok::colon)) {
1279     nextToken();
1280     nextToken(); // base class name
1281   } else if (FormatTok->Tok.is(tok::l_paren))
1282     // Skip category, if present.
1283     parseParens();
1284 
1285   if (FormatTok->Tok.is(tok::less))
1286     parseObjCProtocolList();
1287 
1288   // If instance variables are present, keep the '{' on the first line too.
1289   if (FormatTok->Tok.is(tok::l_brace))
1290     parseBlock(/*MustBeDeclaration=*/true);
1291 
1292   // With instance variables, this puts '}' on its own line.  Without instance
1293   // variables, this ends the @interface line.
1294   addUnwrappedLine();
1295 
1296   parseObjCUntilAtEnd();
1297 }
1298 
1299 void UnwrappedLineParser::parseObjCProtocol() {
1300   nextToken();
1301   nextToken(); // protocol name
1302 
1303   if (FormatTok->Tok.is(tok::less))
1304     parseObjCProtocolList();
1305 
1306   // Check for protocol declaration.
1307   if (FormatTok->Tok.is(tok::semi)) {
1308     nextToken();
1309     return addUnwrappedLine();
1310   }
1311 
1312   addUnwrappedLine();
1313   parseObjCUntilAtEnd();
1314 }
1315 
1316 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1317                                                  StringRef Prefix = "") {
1318   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1319                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1320   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1321                                                     E = Line.Tokens.end();
1322        I != E; ++I) {
1323     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1324   }
1325   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1326                                                     E = Line.Tokens.end();
1327        I != E; ++I) {
1328     const UnwrappedLineNode &Node = *I;
1329     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1330              I = Node.Children.begin(),
1331              E = Node.Children.end();
1332          I != E; ++I) {
1333       printDebugInfo(*I, "\nChild: ");
1334     }
1335   }
1336   llvm::dbgs() << "\n";
1337 }
1338 
1339 void UnwrappedLineParser::addUnwrappedLine() {
1340   if (Line->Tokens.empty())
1341     return;
1342   DEBUG({
1343     if (CurrentLines == &Lines)
1344       printDebugInfo(*Line);
1345   });
1346   CurrentLines->push_back(*Line);
1347   Line->Tokens.clear();
1348   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1349     for (SmallVectorImpl<UnwrappedLine>::iterator
1350              I = PreprocessorDirectives.begin(),
1351              E = PreprocessorDirectives.end();
1352          I != E; ++I) {
1353       CurrentLines->push_back(*I);
1354     }
1355     PreprocessorDirectives.clear();
1356   }
1357 }
1358 
1359 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1360 
1361 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1362   bool JustComments = Line->Tokens.empty();
1363   for (SmallVectorImpl<FormatToken *>::const_iterator
1364            I = CommentsBeforeNextToken.begin(),
1365            E = CommentsBeforeNextToken.end();
1366        I != E; ++I) {
1367     if ((*I)->NewlinesBefore && JustComments) {
1368       addUnwrappedLine();
1369     }
1370     pushToken(*I);
1371   }
1372   if (NewlineBeforeNext && JustComments) {
1373     addUnwrappedLine();
1374   }
1375   CommentsBeforeNextToken.clear();
1376 }
1377 
1378 void UnwrappedLineParser::nextToken() {
1379   if (eof())
1380     return;
1381   flushComments(FormatTok->NewlinesBefore > 0);
1382   pushToken(FormatTok);
1383   readToken();
1384 }
1385 
1386 void UnwrappedLineParser::readToken() {
1387   bool CommentsInCurrentLine = true;
1388   do {
1389     FormatTok = Tokens->getNextToken();
1390     assert(FormatTok);
1391     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1392            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1393       // If there is an unfinished unwrapped line, we flush the preprocessor
1394       // directives only after that unwrapped line was finished later.
1395       bool SwitchToPreprocessorLines =
1396           !Line->Tokens.empty() && CurrentLines == &Lines;
1397       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1398       // Comments stored before the preprocessor directive need to be output
1399       // before the preprocessor directive, at the same level as the
1400       // preprocessor directive, as we consider them to apply to the directive.
1401       flushComments(FormatTok->NewlinesBefore > 0);
1402       parsePPDirective();
1403     }
1404 
1405     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1406         !Line->InPPDirective) {
1407       continue;
1408     }
1409 
1410     if (!FormatTok->Tok.is(tok::comment))
1411       return;
1412     if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
1413       CommentsInCurrentLine = false;
1414     }
1415     if (CommentsInCurrentLine) {
1416       pushToken(FormatTok);
1417     } else {
1418       CommentsBeforeNextToken.push_back(FormatTok);
1419     }
1420   } while (!eof());
1421 }
1422 
1423 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1424   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1425   if (MustBreakBeforeNextToken) {
1426     Line->Tokens.back().Tok->MustBreakBefore = true;
1427     MustBreakBeforeNextToken = false;
1428   }
1429 }
1430 
1431 } // end namespace format
1432 } // end namespace clang
1433