1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "llvm/Support/Debug.h"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   virtual FormatToken *getNextToken() {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   virtual unsigned getPosition() { return PreviousTokenSource->getPosition(); }
88 
89   virtual FormatToken *setPosition(unsigned Position) {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.take();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   virtual FormatToken *getNextToken() {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   virtual unsigned getPosition() {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   virtual FormatToken *setPosition(unsigned P) {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           // If there is a comma, semicolon or right paren after the closing
339           // brace, we assume this is a braced initializer list.  Note that
340           // regardless how we mark inner braces here, we will overwrite the
341           // BlockKind later if we parse a braced list (where all blocks inside
342           // are by default braced lists), or when we explicitly detect blocks
343           // (for example while parsing lambdas).
344           //
345           // We exclude + and - as they can be ObjC visibility modifiers.
346           if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren, tok::period,
347                                tok::r_square, tok::l_brace, tok::colon) ||
348               (NextTok->isBinaryOperator() &&
349                !NextTok->isOneOf(tok::plus, tok::minus))) {
350             Tok->BlockKind = BK_BracedInit;
351             LBraceStack.back()->BlockKind = BK_BracedInit;
352           } else {
353             Tok->BlockKind = BK_Block;
354             LBraceStack.back()->BlockKind = BK_Block;
355           }
356         }
357         LBraceStack.pop_back();
358       }
359       break;
360     case tok::semi:
361     case tok::kw_if:
362     case tok::kw_while:
363     case tok::kw_for:
364     case tok::kw_switch:
365     case tok::kw_try:
366       if (!LBraceStack.empty())
367         LBraceStack.back()->BlockKind = BK_Block;
368       break;
369     default:
370       break;
371     }
372     Tok = NextTok;
373     Position += ReadTokens;
374   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
375   // Assume other blocks for all unclosed opening braces.
376   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
377     if (LBraceStack[i]->BlockKind == BK_Unknown)
378       LBraceStack[i]->BlockKind = BK_Block;
379   }
380 
381   FormatTok = Tokens->setPosition(StoredPosition);
382 }
383 
384 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
385                                      bool MunchSemi) {
386   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
387   unsigned InitialLevel = Line->Level;
388   nextToken();
389 
390   addUnwrappedLine();
391 
392   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
393                                           MustBeDeclaration);
394   if (AddLevel)
395     ++Line->Level;
396   parseLevel(/*HasOpeningBrace=*/true);
397 
398   if (!FormatTok->Tok.is(tok::r_brace)) {
399     Line->Level = InitialLevel;
400     StructuralError = true;
401     return;
402   }
403 
404   nextToken(); // Munch the closing brace.
405   if (MunchSemi && FormatTok->Tok.is(tok::semi))
406     nextToken();
407   Line->Level = InitialLevel;
408 }
409 
410 void UnwrappedLineParser::parseChildBlock() {
411   FormatTok->BlockKind = BK_Block;
412   nextToken();
413   {
414     ScopedLineState LineState(*this);
415     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
416                                             /*MustBeDeclaration=*/false);
417     Line->Level += 1;
418     parseLevel(/*HasOpeningBrace=*/true);
419     Line->Level -= 1;
420   }
421   nextToken();
422 }
423 
424 void UnwrappedLineParser::parsePPDirective() {
425   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
426   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
427   nextToken();
428 
429   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
430     parsePPUnknown();
431     return;
432   }
433 
434   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
435   case tok::pp_define:
436     parsePPDefine();
437     return;
438   case tok::pp_if:
439     parsePPIf(/*IfDef=*/false);
440     break;
441   case tok::pp_ifdef:
442   case tok::pp_ifndef:
443     parsePPIf(/*IfDef=*/true);
444     break;
445   case tok::pp_else:
446     parsePPElse();
447     break;
448   case tok::pp_elif:
449     parsePPElIf();
450     break;
451   case tok::pp_endif:
452     parsePPEndIf();
453     break;
454   default:
455     parsePPUnknown();
456     break;
457   }
458 }
459 
460 void UnwrappedLineParser::pushPPConditional() {
461   if (!PPStack.empty() && PPStack.back() == PP_Unreachable)
462     PPStack.push_back(PP_Unreachable);
463   else
464     PPStack.push_back(PP_Conditional);
465 }
466 
467 void UnwrappedLineParser::parsePPIf(bool IfDef) {
468   ++PPBranchLevel;
469   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
470   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
471     PPLevelBranchIndex.push_back(0);
472     PPLevelBranchCount.push_back(0);
473   }
474   PPChainBranchIndex.push(0);
475   nextToken();
476   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
477                          StringRef(FormatTok->Tok.getLiteralData(),
478                                    FormatTok->Tok.getLength()) == "0") ||
479                         FormatTok->Tok.is(tok::kw_false);
480   if ((!IfDef && IsLiteralFalse) || PPLevelBranchIndex[PPBranchLevel] > 0) {
481     PPStack.push_back(PP_Unreachable);
482   } else {
483     pushPPConditional();
484   }
485   parsePPUnknown();
486 }
487 
488 void UnwrappedLineParser::parsePPElse() {
489   if (!PPStack.empty())
490     PPStack.pop_back();
491   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
492   if (!PPChainBranchIndex.empty())
493     ++PPChainBranchIndex.top();
494   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
495       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top()) {
496     PPStack.push_back(PP_Unreachable);
497   } else {
498     pushPPConditional();
499   }
500   parsePPUnknown();
501 }
502 
503 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
504 
505 void UnwrappedLineParser::parsePPEndIf() {
506   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
507   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
508     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
509       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
510     }
511   }
512   --PPBranchLevel;
513   if (!PPChainBranchIndex.empty())
514     PPChainBranchIndex.pop();
515   if (!PPStack.empty())
516     PPStack.pop_back();
517   parsePPUnknown();
518 }
519 
520 void UnwrappedLineParser::parsePPDefine() {
521   nextToken();
522 
523   if (FormatTok->Tok.getKind() != tok::identifier) {
524     parsePPUnknown();
525     return;
526   }
527   nextToken();
528   if (FormatTok->Tok.getKind() == tok::l_paren &&
529       FormatTok->WhitespaceRange.getBegin() ==
530           FormatTok->WhitespaceRange.getEnd()) {
531     parseParens();
532   }
533   addUnwrappedLine();
534   Line->Level = 1;
535 
536   // Errors during a preprocessor directive can only affect the layout of the
537   // preprocessor directive, and thus we ignore them. An alternative approach
538   // would be to use the same approach we use on the file level (no
539   // re-indentation if there was a structural error) within the macro
540   // definition.
541   parseFile();
542 }
543 
544 void UnwrappedLineParser::parsePPUnknown() {
545   do {
546     nextToken();
547   } while (!eof());
548   addUnwrappedLine();
549 }
550 
551 // Here we blacklist certain tokens that are not usually the first token in an
552 // unwrapped line. This is used in attempt to distinguish macro calls without
553 // trailing semicolons from other constructs split to several lines.
554 bool tokenCanStartNewLine(clang::Token Tok) {
555   // Semicolon can be a null-statement, l_square can be a start of a macro or
556   // a C++11 attribute, but this doesn't seem to be common.
557   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
558          Tok.isNot(tok::l_square) &&
559          // Tokens that can only be used as binary operators and a part of
560          // overloaded operator names.
561          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
562          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
563          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
564          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
565          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
566          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
567          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
568          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
569          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
570          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
571          Tok.isNot(tok::lesslessequal) &&
572          // Colon is used in labels, base class lists, initializer lists,
573          // range-based for loops, ternary operator, but should never be the
574          // first token in an unwrapped line.
575          Tok.isNot(tok::colon);
576 }
577 
578 void UnwrappedLineParser::parseStructuralElement() {
579   assert(!FormatTok->Tok.is(tok::l_brace));
580   switch (FormatTok->Tok.getKind()) {
581   case tok::at:
582     nextToken();
583     if (FormatTok->Tok.is(tok::l_brace)) {
584       parseBracedList();
585       break;
586     }
587     switch (FormatTok->Tok.getObjCKeywordID()) {
588     case tok::objc_public:
589     case tok::objc_protected:
590     case tok::objc_package:
591     case tok::objc_private:
592       return parseAccessSpecifier();
593     case tok::objc_interface:
594     case tok::objc_implementation:
595       return parseObjCInterfaceOrImplementation();
596     case tok::objc_protocol:
597       return parseObjCProtocol();
598     case tok::objc_end:
599       return; // Handled by the caller.
600     case tok::objc_optional:
601     case tok::objc_required:
602       nextToken();
603       addUnwrappedLine();
604       return;
605     default:
606       break;
607     }
608     break;
609   case tok::kw_namespace:
610     parseNamespace();
611     return;
612   case tok::kw_inline:
613     nextToken();
614     if (FormatTok->Tok.is(tok::kw_namespace)) {
615       parseNamespace();
616       return;
617     }
618     break;
619   case tok::kw_public:
620   case tok::kw_protected:
621   case tok::kw_private:
622     parseAccessSpecifier();
623     return;
624   case tok::kw_if:
625     parseIfThenElse();
626     return;
627   case tok::kw_for:
628   case tok::kw_while:
629     parseForOrWhileLoop();
630     return;
631   case tok::kw_do:
632     parseDoWhile();
633     return;
634   case tok::kw_switch:
635     parseSwitch();
636     return;
637   case tok::kw_default:
638     nextToken();
639     parseLabel();
640     return;
641   case tok::kw_case:
642     parseCaseLabel();
643     return;
644   case tok::kw_return:
645     parseReturn();
646     return;
647   case tok::kw_extern:
648     nextToken();
649     if (FormatTok->Tok.is(tok::string_literal)) {
650       nextToken();
651       if (FormatTok->Tok.is(tok::l_brace)) {
652         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
653         addUnwrappedLine();
654         return;
655       }
656     }
657     // In all other cases, parse the declaration.
658     break;
659   default:
660     break;
661   }
662   do {
663     switch (FormatTok->Tok.getKind()) {
664     case tok::at:
665       nextToken();
666       if (FormatTok->Tok.is(tok::l_brace))
667         parseBracedList();
668       break;
669     case tok::kw_enum:
670       parseEnum();
671       break;
672     case tok::kw_struct:
673     case tok::kw_union:
674     case tok::kw_class:
675       parseRecord();
676       // A record declaration or definition is always the start of a structural
677       // element.
678       break;
679     case tok::semi:
680       nextToken();
681       addUnwrappedLine();
682       return;
683     case tok::r_brace:
684       addUnwrappedLine();
685       return;
686     case tok::l_paren:
687       parseParens();
688       break;
689     case tok::caret:
690       nextToken();
691       if (FormatTok->is(tok::l_brace)) {
692         parseChildBlock();
693       }
694       break;
695     case tok::l_brace:
696       if (!tryToParseBracedList()) {
697         // A block outside of parentheses must be the last part of a
698         // structural element.
699         // FIXME: Figure out cases where this is not true, and add projections
700         // for them (the one we know is missing are lambdas).
701         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
702           addUnwrappedLine();
703         FormatTok->Type = TT_FunctionLBrace;
704         parseBlock(/*MustBeDeclaration=*/false);
705         addUnwrappedLine();
706         return;
707       }
708       // Otherwise this was a braced init list, and the structural
709       // element continues.
710       break;
711     case tok::identifier: {
712       StringRef Text = FormatTok->TokenText;
713       nextToken();
714       if (Line->Tokens.size() == 1) {
715         if (FormatTok->Tok.is(tok::colon)) {
716           parseLabel();
717           return;
718         }
719         // Recognize function-like macro usages without trailing semicolon.
720         if (FormatTok->Tok.is(tok::l_paren)) {
721           parseParens();
722           if (FormatTok->NewlinesBefore > 0 &&
723               tokenCanStartNewLine(FormatTok->Tok)) {
724             addUnwrappedLine();
725             return;
726           }
727         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
728                    Text == Text.upper()) {
729           // Recognize free-standing macros like Q_OBJECT.
730           addUnwrappedLine();
731           return;
732         }
733       }
734       break;
735     }
736     case tok::equal:
737       nextToken();
738       if (FormatTok->Tok.is(tok::l_brace)) {
739         parseBracedList();
740       }
741       break;
742     case tok::l_square:
743       parseSquare();
744       break;
745     default:
746       nextToken();
747       break;
748     }
749   } while (!eof());
750 }
751 
752 bool UnwrappedLineParser::tryToParseLambda() {
753   // FIXME: This is a dirty way to access the previous token. Find a better
754   // solution.
755   if (!Line->Tokens.empty() &&
756       Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator)) {
757     nextToken();
758     return false;
759   }
760   assert(FormatTok->is(tok::l_square));
761   FormatToken &LSquare = *FormatTok;
762   if (!tryToParseLambdaIntroducer())
763     return false;
764 
765   while (FormatTok && FormatTok->isNot(tok::l_brace)) {
766     if (FormatTok->isSimpleTypeSpecifier()) {
767       nextToken();
768       continue;
769     }
770     switch (FormatTok->Tok.getKind()) {
771     case tok::l_brace:
772       break;
773     case tok::l_paren:
774       parseParens();
775       break;
776     case tok::less:
777     case tok::greater:
778     case tok::identifier:
779     case tok::kw_mutable:
780     case tok::arrow:
781       nextToken();
782       break;
783     default:
784       return true;
785     }
786   }
787   LSquare.Type = TT_LambdaLSquare;
788   parseChildBlock();
789   return true;
790 }
791 
792 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
793   nextToken();
794   if (FormatTok->is(tok::equal)) {
795     nextToken();
796     if (FormatTok->is(tok::r_square)) {
797       nextToken();
798       return true;
799     }
800     if (FormatTok->isNot(tok::comma))
801       return false;
802     nextToken();
803   } else if (FormatTok->is(tok::amp)) {
804     nextToken();
805     if (FormatTok->is(tok::r_square)) {
806       nextToken();
807       return true;
808     }
809     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
810       return false;
811     }
812     if (FormatTok->is(tok::comma))
813       nextToken();
814   } else if (FormatTok->is(tok::r_square)) {
815     nextToken();
816     return true;
817   }
818   do {
819     if (FormatTok->is(tok::amp))
820       nextToken();
821     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
822       return false;
823     nextToken();
824     if (FormatTok->is(tok::comma)) {
825       nextToken();
826     } else if (FormatTok->is(tok::r_square)) {
827       nextToken();
828       return true;
829     } else {
830       return false;
831     }
832   } while (!eof());
833   return false;
834 }
835 
836 bool UnwrappedLineParser::tryToParseBracedList() {
837   if (FormatTok->BlockKind == BK_Unknown)
838     calculateBraceTypes();
839   assert(FormatTok->BlockKind != BK_Unknown);
840   if (FormatTok->BlockKind == BK_Block)
841     return false;
842   parseBracedList();
843   return true;
844 }
845 
846 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
847   bool HasError = false;
848   nextToken();
849 
850   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
851   // replace this by using parseAssigmentExpression() inside.
852   do {
853     // FIXME: When we start to support lambdas, we'll want to parse them away
854     // here, otherwise our bail-out scenarios below break. The better solution
855     // might be to just implement a more or less complete expression parser.
856     switch (FormatTok->Tok.getKind()) {
857     case tok::caret:
858       nextToken();
859       if (FormatTok->is(tok::l_brace)) {
860         parseChildBlock();
861       }
862       break;
863     case tok::l_square:
864       tryToParseLambda();
865       break;
866     case tok::l_brace:
867       // Assume there are no blocks inside a braced init list apart
868       // from the ones we explicitly parse out (like lambdas).
869       FormatTok->BlockKind = BK_BracedInit;
870       parseBracedList();
871       break;
872     case tok::r_brace:
873       nextToken();
874       return !HasError;
875     case tok::semi:
876       HasError = true;
877       if (!ContinueOnSemicolons)
878         return !HasError;
879       nextToken();
880       break;
881     case tok::comma:
882       nextToken();
883       break;
884     default:
885       nextToken();
886       break;
887     }
888   } while (!eof());
889   return false;
890 }
891 
892 void UnwrappedLineParser::parseReturn() {
893   nextToken();
894 
895   do {
896     switch (FormatTok->Tok.getKind()) {
897     case tok::l_brace:
898       parseBracedList();
899       if (FormatTok->Tok.isNot(tok::semi)) {
900         // Assume missing ';'.
901         addUnwrappedLine();
902         return;
903       }
904       break;
905     case tok::l_paren:
906       parseParens();
907       break;
908     case tok::r_brace:
909       // Assume missing ';'.
910       addUnwrappedLine();
911       return;
912     case tok::semi:
913       nextToken();
914       addUnwrappedLine();
915       return;
916     case tok::l_square:
917       tryToParseLambda();
918       break;
919     default:
920       nextToken();
921       break;
922     }
923   } while (!eof());
924 }
925 
926 void UnwrappedLineParser::parseParens() {
927   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
928   nextToken();
929   do {
930     switch (FormatTok->Tok.getKind()) {
931     case tok::l_paren:
932       parseParens();
933       break;
934     case tok::r_paren:
935       nextToken();
936       return;
937     case tok::r_brace:
938       // A "}" inside parenthesis is an error if there wasn't a matching "{".
939       return;
940     case tok::l_square:
941       tryToParseLambda();
942       break;
943     case tok::l_brace: {
944       if (!tryToParseBracedList()) {
945         parseChildBlock();
946       }
947       break;
948     }
949     case tok::at:
950       nextToken();
951       if (FormatTok->Tok.is(tok::l_brace))
952         parseBracedList();
953       break;
954     default:
955       nextToken();
956       break;
957     }
958   } while (!eof());
959 }
960 
961 void UnwrappedLineParser::parseSquare() {
962   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
963   if (tryToParseLambda())
964     return;
965   do {
966     switch (FormatTok->Tok.getKind()) {
967     case tok::l_paren:
968       parseParens();
969       break;
970     case tok::r_square:
971       nextToken();
972       return;
973     case tok::r_brace:
974       // A "}" inside parenthesis is an error if there wasn't a matching "{".
975       return;
976     case tok::l_square:
977       parseSquare();
978       break;
979     case tok::l_brace: {
980       if (!tryToParseBracedList()) {
981         parseChildBlock();
982       }
983       break;
984     }
985     case tok::at:
986       nextToken();
987       if (FormatTok->Tok.is(tok::l_brace))
988         parseBracedList();
989       break;
990     default:
991       nextToken();
992       break;
993     }
994   } while (!eof());
995 }
996 
997 void UnwrappedLineParser::parseIfThenElse() {
998   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
999   nextToken();
1000   if (FormatTok->Tok.is(tok::l_paren))
1001     parseParens();
1002   bool NeedsUnwrappedLine = false;
1003   if (FormatTok->Tok.is(tok::l_brace)) {
1004     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1005     parseBlock(/*MustBeDeclaration=*/false);
1006     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1007         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1008       addUnwrappedLine();
1009     } else {
1010       NeedsUnwrappedLine = true;
1011     }
1012   } else {
1013     addUnwrappedLine();
1014     ++Line->Level;
1015     parseStructuralElement();
1016     --Line->Level;
1017   }
1018   if (FormatTok->Tok.is(tok::kw_else)) {
1019     nextToken();
1020     if (FormatTok->Tok.is(tok::l_brace)) {
1021       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1022       parseBlock(/*MustBeDeclaration=*/false);
1023       addUnwrappedLine();
1024     } else if (FormatTok->Tok.is(tok::kw_if)) {
1025       parseIfThenElse();
1026     } else {
1027       addUnwrappedLine();
1028       ++Line->Level;
1029       parseStructuralElement();
1030       --Line->Level;
1031     }
1032   } else if (NeedsUnwrappedLine) {
1033     addUnwrappedLine();
1034   }
1035 }
1036 
1037 void UnwrappedLineParser::parseNamespace() {
1038   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1039   nextToken();
1040   if (FormatTok->Tok.is(tok::identifier))
1041     nextToken();
1042   if (FormatTok->Tok.is(tok::l_brace)) {
1043     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1044         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1045         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1046       addUnwrappedLine();
1047 
1048     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1049                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1050                      DeclarationScopeStack.size() > 1);
1051     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1052     // Munch the semicolon after a namespace. This is more common than one would
1053     // think. Puttin the semicolon into its own line is very ugly.
1054     if (FormatTok->Tok.is(tok::semi))
1055       nextToken();
1056     addUnwrappedLine();
1057   }
1058   // FIXME: Add error handling.
1059 }
1060 
1061 void UnwrappedLineParser::parseForOrWhileLoop() {
1062   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while)) &&
1063          "'for' or 'while' expected");
1064   nextToken();
1065   if (FormatTok->Tok.is(tok::l_paren))
1066     parseParens();
1067   if (FormatTok->Tok.is(tok::l_brace)) {
1068     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1069     parseBlock(/*MustBeDeclaration=*/false);
1070     addUnwrappedLine();
1071   } else {
1072     addUnwrappedLine();
1073     ++Line->Level;
1074     parseStructuralElement();
1075     --Line->Level;
1076   }
1077 }
1078 
1079 void UnwrappedLineParser::parseDoWhile() {
1080   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1081   nextToken();
1082   if (FormatTok->Tok.is(tok::l_brace)) {
1083     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1084     parseBlock(/*MustBeDeclaration=*/false);
1085     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1086       addUnwrappedLine();
1087   } else {
1088     addUnwrappedLine();
1089     ++Line->Level;
1090     parseStructuralElement();
1091     --Line->Level;
1092   }
1093 
1094   // FIXME: Add error handling.
1095   if (!FormatTok->Tok.is(tok::kw_while)) {
1096     addUnwrappedLine();
1097     return;
1098   }
1099 
1100   nextToken();
1101   parseStructuralElement();
1102 }
1103 
1104 void UnwrappedLineParser::parseLabel() {
1105   nextToken();
1106   unsigned OldLineLevel = Line->Level;
1107   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1108     --Line->Level;
1109   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1110     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1111     parseBlock(/*MustBeDeclaration=*/false);
1112     if (FormatTok->Tok.is(tok::kw_break)) {
1113       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1114       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1115           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1116         addUnwrappedLine();
1117       }
1118       parseStructuralElement();
1119     }
1120     addUnwrappedLine();
1121   } else {
1122     addUnwrappedLine();
1123   }
1124   Line->Level = OldLineLevel;
1125 }
1126 
1127 void UnwrappedLineParser::parseCaseLabel() {
1128   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1129   // FIXME: fix handling of complex expressions here.
1130   do {
1131     nextToken();
1132   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1133   parseLabel();
1134 }
1135 
1136 void UnwrappedLineParser::parseSwitch() {
1137   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1138   nextToken();
1139   if (FormatTok->Tok.is(tok::l_paren))
1140     parseParens();
1141   if (FormatTok->Tok.is(tok::l_brace)) {
1142     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1143     parseBlock(/*MustBeDeclaration=*/false);
1144     addUnwrappedLine();
1145   } else {
1146     addUnwrappedLine();
1147     ++Line->Level;
1148     parseStructuralElement();
1149     --Line->Level;
1150   }
1151 }
1152 
1153 void UnwrappedLineParser::parseAccessSpecifier() {
1154   nextToken();
1155   // Understand Qt's slots.
1156   if (FormatTok->is(tok::identifier) &&
1157       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1158     nextToken();
1159   // Otherwise, we don't know what it is, and we'd better keep the next token.
1160   if (FormatTok->Tok.is(tok::colon))
1161     nextToken();
1162   addUnwrappedLine();
1163 }
1164 
1165 void UnwrappedLineParser::parseEnum() {
1166   nextToken();
1167   // Eat up enum class ...
1168   if (FormatTok->Tok.is(tok::kw_class) ||
1169       FormatTok->Tok.is(tok::kw_struct))
1170       nextToken();
1171   while (FormatTok->Tok.getIdentifierInfo() ||
1172          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1173     nextToken();
1174     // We can have macros or attributes in between 'enum' and the enum name.
1175     if (FormatTok->Tok.is(tok::l_paren)) {
1176       parseParens();
1177     }
1178     if (FormatTok->Tok.is(tok::identifier))
1179       nextToken();
1180   }
1181   if (FormatTok->Tok.is(tok::l_brace)) {
1182     FormatTok->BlockKind = BK_Block;
1183     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1184     if (HasError) {
1185       if (FormatTok->is(tok::semi))
1186         nextToken();
1187       addUnwrappedLine();
1188     }
1189   }
1190   // We fall through to parsing a structural element afterwards, so that in
1191   // enum A {} n, m;
1192   // "} n, m;" will end up in one unwrapped line.
1193 }
1194 
1195 void UnwrappedLineParser::parseRecord() {
1196   nextToken();
1197   if (FormatTok->Tok.is(tok::identifier) ||
1198       FormatTok->Tok.is(tok::kw___attribute) ||
1199       FormatTok->Tok.is(tok::kw___declspec) ||
1200       FormatTok->Tok.is(tok::kw_alignas)) {
1201     nextToken();
1202     // We can have macros or attributes in between 'class' and the class name.
1203     if (FormatTok->Tok.is(tok::l_paren)) {
1204       parseParens();
1205     }
1206     // The actual identifier can be a nested name specifier, and in macros
1207     // it is often token-pasted.
1208     while (FormatTok->Tok.is(tok::identifier) ||
1209            FormatTok->Tok.is(tok::coloncolon) ||
1210            FormatTok->Tok.is(tok::hashhash))
1211       nextToken();
1212 
1213     // Note that parsing away template declarations here leads to incorrectly
1214     // accepting function declarations as record declarations.
1215     // In general, we cannot solve this problem. Consider:
1216     // class A<int> B() {}
1217     // which can be a function definition or a class definition when B() is a
1218     // macro. If we find enough real-world cases where this is a problem, we
1219     // can parse for the 'template' keyword in the beginning of the statement,
1220     // and thus rule out the record production in case there is no template
1221     // (this would still leave us with an ambiguity between template function
1222     // and class declarations).
1223     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1224       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1225         if (FormatTok->Tok.is(tok::semi))
1226           return;
1227         nextToken();
1228       }
1229     }
1230   }
1231   if (FormatTok->Tok.is(tok::l_brace)) {
1232     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1233         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1234         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1235       addUnwrappedLine();
1236 
1237     parseBlock(/*MustBeDeclaration=*/true, /*Addlevel=*/true,
1238                /*MunchSemi=*/false);
1239   }
1240   // We fall through to parsing a structural element afterwards, so
1241   // class A {} n, m;
1242   // will end up in one unwrapped line.
1243 }
1244 
1245 void UnwrappedLineParser::parseObjCProtocolList() {
1246   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1247   do
1248     nextToken();
1249   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1250   nextToken(); // Skip '>'.
1251 }
1252 
1253 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1254   do {
1255     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1256       nextToken();
1257       addUnwrappedLine();
1258       break;
1259     }
1260     if (FormatTok->is(tok::l_brace)) {
1261       parseBlock(/*MustBeDeclaration=*/false);
1262       // In ObjC interfaces, nothing should be following the "}".
1263       addUnwrappedLine();
1264     } else if (FormatTok->is(tok::r_brace)) {
1265       // Ignore stray "}". parseStructuralElement doesn't consume them.
1266       nextToken();
1267       addUnwrappedLine();
1268     } else {
1269       parseStructuralElement();
1270     }
1271   } while (!eof());
1272 }
1273 
1274 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1275   nextToken();
1276   nextToken(); // interface name
1277 
1278   // @interface can be followed by either a base class, or a category.
1279   if (FormatTok->Tok.is(tok::colon)) {
1280     nextToken();
1281     nextToken(); // base class name
1282   } else if (FormatTok->Tok.is(tok::l_paren))
1283     // Skip category, if present.
1284     parseParens();
1285 
1286   if (FormatTok->Tok.is(tok::less))
1287     parseObjCProtocolList();
1288 
1289   // If instance variables are present, keep the '{' on the first line too.
1290   if (FormatTok->Tok.is(tok::l_brace))
1291     parseBlock(/*MustBeDeclaration=*/true);
1292 
1293   // With instance variables, this puts '}' on its own line.  Without instance
1294   // variables, this ends the @interface line.
1295   addUnwrappedLine();
1296 
1297   parseObjCUntilAtEnd();
1298 }
1299 
1300 void UnwrappedLineParser::parseObjCProtocol() {
1301   nextToken();
1302   nextToken(); // protocol name
1303 
1304   if (FormatTok->Tok.is(tok::less))
1305     parseObjCProtocolList();
1306 
1307   // Check for protocol declaration.
1308   if (FormatTok->Tok.is(tok::semi)) {
1309     nextToken();
1310     return addUnwrappedLine();
1311   }
1312 
1313   addUnwrappedLine();
1314   parseObjCUntilAtEnd();
1315 }
1316 
1317 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1318                                                  StringRef Prefix = "") {
1319   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1320                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1321   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1322                                                     E = Line.Tokens.end();
1323        I != E; ++I) {
1324     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1325   }
1326   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1327                                                     E = Line.Tokens.end();
1328        I != E; ++I) {
1329     const UnwrappedLineNode &Node = *I;
1330     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1331              I = Node.Children.begin(),
1332              E = Node.Children.end();
1333          I != E; ++I) {
1334       printDebugInfo(*I, "\nChild: ");
1335     }
1336   }
1337   llvm::dbgs() << "\n";
1338 }
1339 
1340 void UnwrappedLineParser::addUnwrappedLine() {
1341   if (Line->Tokens.empty())
1342     return;
1343   DEBUG({
1344     if (CurrentLines == &Lines)
1345       printDebugInfo(*Line);
1346   });
1347   CurrentLines->push_back(*Line);
1348   Line->Tokens.clear();
1349   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1350     for (SmallVectorImpl<UnwrappedLine>::iterator
1351              I = PreprocessorDirectives.begin(),
1352              E = PreprocessorDirectives.end();
1353          I != E; ++I) {
1354       CurrentLines->push_back(*I);
1355     }
1356     PreprocessorDirectives.clear();
1357   }
1358 }
1359 
1360 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1361 
1362 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1363   bool JustComments = Line->Tokens.empty();
1364   for (SmallVectorImpl<FormatToken *>::const_iterator
1365            I = CommentsBeforeNextToken.begin(),
1366            E = CommentsBeforeNextToken.end();
1367        I != E; ++I) {
1368     if ((*I)->NewlinesBefore && JustComments) {
1369       addUnwrappedLine();
1370     }
1371     pushToken(*I);
1372   }
1373   if (NewlineBeforeNext && JustComments) {
1374     addUnwrappedLine();
1375   }
1376   CommentsBeforeNextToken.clear();
1377 }
1378 
1379 void UnwrappedLineParser::nextToken() {
1380   if (eof())
1381     return;
1382   flushComments(FormatTok->NewlinesBefore > 0);
1383   pushToken(FormatTok);
1384   readToken();
1385 }
1386 
1387 void UnwrappedLineParser::readToken() {
1388   bool CommentsInCurrentLine = true;
1389   do {
1390     FormatTok = Tokens->getNextToken();
1391     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1392            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1393       // If there is an unfinished unwrapped line, we flush the preprocessor
1394       // directives only after that unwrapped line was finished later.
1395       bool SwitchToPreprocessorLines =
1396           !Line->Tokens.empty() && CurrentLines == &Lines;
1397       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1398       // Comments stored before the preprocessor directive need to be output
1399       // before the preprocessor directive, at the same level as the
1400       // preprocessor directive, as we consider them to apply to the directive.
1401       flushComments(FormatTok->NewlinesBefore > 0);
1402       parsePPDirective();
1403     }
1404 
1405     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1406         !Line->InPPDirective) {
1407       continue;
1408     }
1409 
1410     if (!FormatTok->Tok.is(tok::comment))
1411       return;
1412     if (FormatTok->NewlinesBefore > 0 || FormatTok->IsFirst) {
1413       CommentsInCurrentLine = false;
1414     }
1415     if (CommentsInCurrentLine) {
1416       pushToken(FormatTok);
1417     } else {
1418       CommentsBeforeNextToken.push_back(FormatTok);
1419     }
1420   } while (!eof());
1421 }
1422 
1423 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1424   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1425   if (MustBreakBeforeNextToken) {
1426     Line->Tokens.back().Tok->MustBreakBefore = true;
1427     MustBreakBeforeNextToken = false;
1428   }
1429 }
1430 
1431 } // end namespace format
1432 } // end namespace clang
1433