1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "UnwrappedLineParser.h"
17 #include "llvm/Support/Debug.h"
18 
19 #define DEBUG_TYPE "format-parser"
20 
21 namespace clang {
22 namespace format {
23 
24 class FormatTokenSource {
25 public:
26   virtual ~FormatTokenSource() {}
27   virtual FormatToken *getNextToken() = 0;
28 
29   virtual unsigned getPosition() = 0;
30   virtual FormatToken *setPosition(unsigned Position) = 0;
31 };
32 
33 namespace {
34 
35 class ScopedDeclarationState {
36 public:
37   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
38                          bool MustBeDeclaration)
39       : Line(Line), Stack(Stack) {
40     Line.MustBeDeclaration = MustBeDeclaration;
41     Stack.push_back(MustBeDeclaration);
42   }
43   ~ScopedDeclarationState() {
44     Stack.pop_back();
45     if (!Stack.empty())
46       Line.MustBeDeclaration = Stack.back();
47     else
48       Line.MustBeDeclaration = true;
49   }
50 
51 private:
52   UnwrappedLine &Line;
53   std::vector<bool> &Stack;
54 };
55 
56 class ScopedMacroState : public FormatTokenSource {
57 public:
58   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
59                    FormatToken *&ResetToken, bool &StructuralError)
60       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
61         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
62         StructuralError(StructuralError),
63         PreviousStructuralError(StructuralError), Token(NULL) {
64     TokenSource = this;
65     Line.Level = 0;
66     Line.InPPDirective = true;
67   }
68 
69   ~ScopedMacroState() {
70     TokenSource = PreviousTokenSource;
71     ResetToken = Token;
72     Line.InPPDirective = false;
73     Line.Level = PreviousLineLevel;
74     StructuralError = PreviousStructuralError;
75   }
76 
77   FormatToken *getNextToken() override {
78     // The \c UnwrappedLineParser guards against this by never calling
79     // \c getNextToken() after it has encountered the first eof token.
80     assert(!eof());
81     Token = PreviousTokenSource->getNextToken();
82     if (eof())
83       return getFakeEOF();
84     return Token;
85   }
86 
87   unsigned getPosition() override { return PreviousTokenSource->getPosition(); }
88 
89   FormatToken *setPosition(unsigned Position) override {
90     Token = PreviousTokenSource->setPosition(Position);
91     return Token;
92   }
93 
94 private:
95   bool eof() { return Token && Token->HasUnescapedNewline; }
96 
97   FormatToken *getFakeEOF() {
98     static bool EOFInitialized = false;
99     static FormatToken FormatTok;
100     if (!EOFInitialized) {
101       FormatTok.Tok.startToken();
102       FormatTok.Tok.setKind(tok::eof);
103       EOFInitialized = true;
104     }
105     return &FormatTok;
106   }
107 
108   UnwrappedLine &Line;
109   FormatTokenSource *&TokenSource;
110   FormatToken *&ResetToken;
111   unsigned PreviousLineLevel;
112   FormatTokenSource *PreviousTokenSource;
113   bool &StructuralError;
114   bool PreviousStructuralError;
115 
116   FormatToken *Token;
117 };
118 
119 } // end anonymous namespace
120 
121 class ScopedLineState {
122 public:
123   ScopedLineState(UnwrappedLineParser &Parser,
124                   bool SwitchToPreprocessorLines = false)
125       : Parser(Parser) {
126     OriginalLines = Parser.CurrentLines;
127     if (SwitchToPreprocessorLines)
128       Parser.CurrentLines = &Parser.PreprocessorDirectives;
129     else if (!Parser.Line->Tokens.empty())
130       Parser.CurrentLines = &Parser.Line->Tokens.back().Children;
131     PreBlockLine = Parser.Line.release();
132     Parser.Line.reset(new UnwrappedLine());
133     Parser.Line->Level = PreBlockLine->Level;
134     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
135   }
136 
137   ~ScopedLineState() {
138     if (!Parser.Line->Tokens.empty()) {
139       Parser.addUnwrappedLine();
140     }
141     assert(Parser.Line->Tokens.empty());
142     Parser.Line.reset(PreBlockLine);
143     if (Parser.CurrentLines == &Parser.PreprocessorDirectives)
144       Parser.MustBreakBeforeNextToken = true;
145     Parser.CurrentLines = OriginalLines;
146   }
147 
148 private:
149   UnwrappedLineParser &Parser;
150 
151   UnwrappedLine *PreBlockLine;
152   SmallVectorImpl<UnwrappedLine> *OriginalLines;
153 };
154 
155 class CompoundStatementIndenter {
156 public:
157   CompoundStatementIndenter(UnwrappedLineParser *Parser,
158                             const FormatStyle &Style, unsigned &LineLevel)
159       : LineLevel(LineLevel), OldLineLevel(LineLevel) {
160     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman) {
161       Parser->addUnwrappedLine();
162     } else if (Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
163       Parser->addUnwrappedLine();
164       ++LineLevel;
165     }
166   }
167   ~CompoundStatementIndenter() {
168     LineLevel = OldLineLevel;
169   }
170 
171 private:
172   unsigned &LineLevel;
173   unsigned OldLineLevel;
174 };
175 
176 namespace {
177 
178 class IndexedTokenSource : public FormatTokenSource {
179 public:
180   IndexedTokenSource(ArrayRef<FormatToken *> Tokens)
181       : Tokens(Tokens), Position(-1) {}
182 
183   FormatToken *getNextToken() override {
184     ++Position;
185     return Tokens[Position];
186   }
187 
188   unsigned getPosition() override {
189     assert(Position >= 0);
190     return Position;
191   }
192 
193   FormatToken *setPosition(unsigned P) override {
194     Position = P;
195     return Tokens[Position];
196   }
197 
198   void reset() { Position = -1; }
199 
200 private:
201   ArrayRef<FormatToken *> Tokens;
202   int Position;
203 };
204 
205 } // end anonymous namespace
206 
207 UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
208                                          ArrayRef<FormatToken *> Tokens,
209                                          UnwrappedLineConsumer &Callback)
210     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
211       CurrentLines(&Lines), StructuralError(false), Style(Style), Tokens(NULL),
212       Callback(Callback), AllTokens(Tokens), PPBranchLevel(-1) {}
213 
214 void UnwrappedLineParser::reset() {
215   PPBranchLevel = -1;
216   Line.reset(new UnwrappedLine);
217   CommentsBeforeNextToken.clear();
218   FormatTok = NULL;
219   MustBreakBeforeNextToken = false;
220   PreprocessorDirectives.clear();
221   CurrentLines = &Lines;
222   DeclarationScopeStack.clear();
223   StructuralError = false;
224   PPStack.clear();
225 }
226 
227 bool UnwrappedLineParser::parse() {
228   IndexedTokenSource TokenSource(AllTokens);
229   do {
230     DEBUG(llvm::dbgs() << "----\n");
231     reset();
232     Tokens = &TokenSource;
233     TokenSource.reset();
234 
235     readToken();
236     parseFile();
237     // Create line with eof token.
238     pushToken(FormatTok);
239     addUnwrappedLine();
240 
241     for (SmallVectorImpl<UnwrappedLine>::iterator I = Lines.begin(),
242                                                   E = Lines.end();
243          I != E; ++I) {
244       Callback.consumeUnwrappedLine(*I);
245     }
246     Callback.finishRun();
247     Lines.clear();
248     while (!PPLevelBranchIndex.empty() &&
249            PPLevelBranchIndex.back() + 1 >= PPLevelBranchCount.back()) {
250       PPLevelBranchIndex.resize(PPLevelBranchIndex.size() - 1);
251       PPLevelBranchCount.resize(PPLevelBranchCount.size() - 1);
252     }
253     if (!PPLevelBranchIndex.empty()) {
254       ++PPLevelBranchIndex.back();
255       assert(PPLevelBranchIndex.size() == PPLevelBranchCount.size());
256       assert(PPLevelBranchIndex.back() <= PPLevelBranchCount.back());
257     }
258   } while (!PPLevelBranchIndex.empty());
259 
260   return StructuralError;
261 }
262 
263 void UnwrappedLineParser::parseFile() {
264   ScopedDeclarationState DeclarationState(
265       *Line, DeclarationScopeStack,
266       /*MustBeDeclaration=*/ !Line->InPPDirective);
267   parseLevel(/*HasOpeningBrace=*/false);
268   // Make sure to format the remaining tokens.
269   flushComments(true);
270   addUnwrappedLine();
271 }
272 
273 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
274   bool SwitchLabelEncountered = false;
275   do {
276     switch (FormatTok->Tok.getKind()) {
277     case tok::comment:
278       nextToken();
279       addUnwrappedLine();
280       break;
281     case tok::l_brace:
282       // FIXME: Add parameter whether this can happen - if this happens, we must
283       // be in a non-declaration context.
284       parseBlock(/*MustBeDeclaration=*/false);
285       addUnwrappedLine();
286       break;
287     case tok::r_brace:
288       if (HasOpeningBrace)
289         return;
290       StructuralError = true;
291       nextToken();
292       addUnwrappedLine();
293       break;
294     case tok::kw_default:
295     case tok::kw_case:
296       if (!SwitchLabelEncountered &&
297           (Style.IndentCaseLabels || (Line->InPPDirective && Line->Level == 1)))
298         ++Line->Level;
299       SwitchLabelEncountered = true;
300       parseStructuralElement();
301       break;
302     default:
303       parseStructuralElement();
304       break;
305     }
306   } while (!eof());
307 }
308 
309 void UnwrappedLineParser::calculateBraceTypes() {
310   // We'll parse forward through the tokens until we hit
311   // a closing brace or eof - note that getNextToken() will
312   // parse macros, so this will magically work inside macro
313   // definitions, too.
314   unsigned StoredPosition = Tokens->getPosition();
315   unsigned Position = StoredPosition;
316   FormatToken *Tok = FormatTok;
317   // Keep a stack of positions of lbrace tokens. We will
318   // update information about whether an lbrace starts a
319   // braced init list or a different block during the loop.
320   SmallVector<FormatToken *, 8> LBraceStack;
321   assert(Tok->Tok.is(tok::l_brace));
322   do {
323     // Get next none-comment token.
324     FormatToken *NextTok;
325     unsigned ReadTokens = 0;
326     do {
327       NextTok = Tokens->getNextToken();
328       ++ReadTokens;
329     } while (NextTok->is(tok::comment));
330 
331     switch (Tok->Tok.getKind()) {
332     case tok::l_brace:
333       LBraceStack.push_back(Tok);
334       break;
335     case tok::r_brace:
336       if (!LBraceStack.empty()) {
337         if (LBraceStack.back()->BlockKind == BK_Unknown) {
338           bool ProbablyBracedList = false;
339           if (Style.Language == FormatStyle::LK_Proto) {
340             ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
341           } else {
342             // If there is a comma, semicolon or right paren after the closing
343             // brace, we assume this is a braced initializer list.  Note that
344             // regardless how we mark inner braces here, we will overwrite the
345             // BlockKind later if we parse a braced list (where all blocks
346             // inside are by default braced lists), or when we explicitly detect
347             // blocks (for example while parsing lambdas).
348             //
349             // We exclude + and - as they can be ObjC visibility modifiers.
350             ProbablyBracedList =
351                 NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
352                                  tok::r_paren, tok::r_square, tok::l_brace) ||
353                 (NextTok->isBinaryOperator() &&
354                  !NextTok->isOneOf(tok::plus, tok::minus));
355           }
356           if (ProbablyBracedList) {
357             Tok->BlockKind = BK_BracedInit;
358             LBraceStack.back()->BlockKind = BK_BracedInit;
359           } else {
360             Tok->BlockKind = BK_Block;
361             LBraceStack.back()->BlockKind = BK_Block;
362           }
363         }
364         LBraceStack.pop_back();
365       }
366       break;
367     case tok::at:
368     case tok::semi:
369     case tok::kw_if:
370     case tok::kw_while:
371     case tok::kw_for:
372     case tok::kw_switch:
373     case tok::kw_try:
374       if (!LBraceStack.empty())
375         LBraceStack.back()->BlockKind = BK_Block;
376       break;
377     default:
378       break;
379     }
380     Tok = NextTok;
381     Position += ReadTokens;
382   } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
383   // Assume other blocks for all unclosed opening braces.
384   for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
385     if (LBraceStack[i]->BlockKind == BK_Unknown)
386       LBraceStack[i]->BlockKind = BK_Block;
387   }
388 
389   FormatTok = Tokens->setPosition(StoredPosition);
390 }
391 
392 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
393                                      bool MunchSemi) {
394   assert(FormatTok->Tok.is(tok::l_brace) && "'{' expected");
395   unsigned InitialLevel = Line->Level;
396   nextToken();
397 
398   addUnwrappedLine();
399 
400   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
401                                           MustBeDeclaration);
402   if (AddLevel)
403     ++Line->Level;
404   parseLevel(/*HasOpeningBrace=*/true);
405 
406   if (!FormatTok->Tok.is(tok::r_brace)) {
407     Line->Level = InitialLevel;
408     StructuralError = true;
409     return;
410   }
411 
412   nextToken(); // Munch the closing brace.
413   if (MunchSemi && FormatTok->Tok.is(tok::semi))
414     nextToken();
415   Line->Level = InitialLevel;
416 }
417 
418 void UnwrappedLineParser::parseChildBlock() {
419   FormatTok->BlockKind = BK_Block;
420   nextToken();
421   {
422     ScopedLineState LineState(*this);
423     ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
424                                             /*MustBeDeclaration=*/false);
425     Line->Level += 1;
426     parseLevel(/*HasOpeningBrace=*/true);
427     Line->Level -= 1;
428   }
429   nextToken();
430 }
431 
432 void UnwrappedLineParser::parsePPDirective() {
433   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
434   ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
435   nextToken();
436 
437   if (FormatTok->Tok.getIdentifierInfo() == NULL) {
438     parsePPUnknown();
439     return;
440   }
441 
442   switch (FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
443   case tok::pp_define:
444     parsePPDefine();
445     return;
446   case tok::pp_if:
447     parsePPIf(/*IfDef=*/false);
448     break;
449   case tok::pp_ifdef:
450   case tok::pp_ifndef:
451     parsePPIf(/*IfDef=*/true);
452     break;
453   case tok::pp_else:
454     parsePPElse();
455     break;
456   case tok::pp_elif:
457     parsePPElIf();
458     break;
459   case tok::pp_endif:
460     parsePPEndIf();
461     break;
462   default:
463     parsePPUnknown();
464     break;
465   }
466 }
467 
468 void UnwrappedLineParser::conditionalCompilationCondition(bool Unreachable) {
469   if (Unreachable || (!PPStack.empty() && PPStack.back() == PP_Unreachable))
470     PPStack.push_back(PP_Unreachable);
471   else
472     PPStack.push_back(PP_Conditional);
473 }
474 
475 void UnwrappedLineParser::conditionalCompilationStart(bool Unreachable) {
476   ++PPBranchLevel;
477   assert(PPBranchLevel >= 0 && PPBranchLevel <= (int)PPLevelBranchIndex.size());
478   if (PPBranchLevel == (int)PPLevelBranchIndex.size()) {
479     PPLevelBranchIndex.push_back(0);
480     PPLevelBranchCount.push_back(0);
481   }
482   PPChainBranchIndex.push(0);
483   bool Skip = PPLevelBranchIndex[PPBranchLevel] > 0;
484   conditionalCompilationCondition(Unreachable || Skip);
485 }
486 
487 void UnwrappedLineParser::conditionalCompilationAlternative() {
488   if (!PPStack.empty())
489     PPStack.pop_back();
490   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
491   if (!PPChainBranchIndex.empty())
492     ++PPChainBranchIndex.top();
493   conditionalCompilationCondition(
494       PPBranchLevel >= 0 && !PPChainBranchIndex.empty() &&
495       PPLevelBranchIndex[PPBranchLevel] != PPChainBranchIndex.top());
496 }
497 
498 void UnwrappedLineParser::conditionalCompilationEnd() {
499   assert(PPBranchLevel < (int)PPLevelBranchIndex.size());
500   if (PPBranchLevel >= 0 && !PPChainBranchIndex.empty()) {
501     if (PPChainBranchIndex.top() + 1 > PPLevelBranchCount[PPBranchLevel]) {
502       PPLevelBranchCount[PPBranchLevel] = PPChainBranchIndex.top() + 1;
503     }
504   }
505   // Guard against #endif's without #if.
506   if (PPBranchLevel > 0)
507     --PPBranchLevel;
508   if (!PPChainBranchIndex.empty())
509     PPChainBranchIndex.pop();
510   if (!PPStack.empty())
511     PPStack.pop_back();
512 }
513 
514 void UnwrappedLineParser::parsePPIf(bool IfDef) {
515   nextToken();
516   bool IsLiteralFalse = (FormatTok->Tok.isLiteral() &&
517                          StringRef(FormatTok->Tok.getLiteralData(),
518                                    FormatTok->Tok.getLength()) == "0") ||
519                         FormatTok->Tok.is(tok::kw_false);
520   conditionalCompilationStart(!IfDef && IsLiteralFalse);
521   parsePPUnknown();
522 }
523 
524 void UnwrappedLineParser::parsePPElse() {
525   conditionalCompilationAlternative();
526   parsePPUnknown();
527 }
528 
529 void UnwrappedLineParser::parsePPElIf() { parsePPElse(); }
530 
531 void UnwrappedLineParser::parsePPEndIf() {
532   conditionalCompilationEnd();
533   parsePPUnknown();
534 }
535 
536 void UnwrappedLineParser::parsePPDefine() {
537   nextToken();
538 
539   if (FormatTok->Tok.getKind() != tok::identifier) {
540     parsePPUnknown();
541     return;
542   }
543   nextToken();
544   if (FormatTok->Tok.getKind() == tok::l_paren &&
545       FormatTok->WhitespaceRange.getBegin() ==
546           FormatTok->WhitespaceRange.getEnd()) {
547     parseParens();
548   }
549   addUnwrappedLine();
550   Line->Level = 1;
551 
552   // Errors during a preprocessor directive can only affect the layout of the
553   // preprocessor directive, and thus we ignore them. An alternative approach
554   // would be to use the same approach we use on the file level (no
555   // re-indentation if there was a structural error) within the macro
556   // definition.
557   parseFile();
558 }
559 
560 void UnwrappedLineParser::parsePPUnknown() {
561   do {
562     nextToken();
563   } while (!eof());
564   addUnwrappedLine();
565 }
566 
567 // Here we blacklist certain tokens that are not usually the first token in an
568 // unwrapped line. This is used in attempt to distinguish macro calls without
569 // trailing semicolons from other constructs split to several lines.
570 bool tokenCanStartNewLine(clang::Token Tok) {
571   // Semicolon can be a null-statement, l_square can be a start of a macro or
572   // a C++11 attribute, but this doesn't seem to be common.
573   return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
574          Tok.isNot(tok::l_square) &&
575          // Tokens that can only be used as binary operators and a part of
576          // overloaded operator names.
577          Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
578          Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
579          Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
580          Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
581          Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
582          Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
583          Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
584          Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
585          Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
586          Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
587          Tok.isNot(tok::lesslessequal) &&
588          // Colon is used in labels, base class lists, initializer lists,
589          // range-based for loops, ternary operator, but should never be the
590          // first token in an unwrapped line.
591          Tok.isNot(tok::colon);
592 }
593 
594 void UnwrappedLineParser::parseStructuralElement() {
595   assert(!FormatTok->Tok.is(tok::l_brace));
596   switch (FormatTok->Tok.getKind()) {
597   case tok::at:
598     nextToken();
599     if (FormatTok->Tok.is(tok::l_brace)) {
600       parseBracedList();
601       break;
602     }
603     switch (FormatTok->Tok.getObjCKeywordID()) {
604     case tok::objc_public:
605     case tok::objc_protected:
606     case tok::objc_package:
607     case tok::objc_private:
608       return parseAccessSpecifier();
609     case tok::objc_interface:
610     case tok::objc_implementation:
611       return parseObjCInterfaceOrImplementation();
612     case tok::objc_protocol:
613       return parseObjCProtocol();
614     case tok::objc_end:
615       return; // Handled by the caller.
616     case tok::objc_optional:
617     case tok::objc_required:
618       nextToken();
619       addUnwrappedLine();
620       return;
621     default:
622       break;
623     }
624     break;
625   case tok::kw_namespace:
626     parseNamespace();
627     return;
628   case tok::kw_inline:
629     nextToken();
630     if (FormatTok->Tok.is(tok::kw_namespace)) {
631       parseNamespace();
632       return;
633     }
634     break;
635   case tok::kw_public:
636   case tok::kw_protected:
637   case tok::kw_private:
638     parseAccessSpecifier();
639     return;
640   case tok::kw_if:
641     parseIfThenElse();
642     return;
643   case tok::kw_for:
644   case tok::kw_while:
645     parseForOrWhileLoop();
646     return;
647   case tok::kw_do:
648     parseDoWhile();
649     return;
650   case tok::kw_switch:
651     parseSwitch();
652     return;
653   case tok::kw_default:
654     nextToken();
655     parseLabel();
656     return;
657   case tok::kw_case:
658     parseCaseLabel();
659     return;
660   case tok::kw_extern:
661     nextToken();
662     if (FormatTok->Tok.is(tok::string_literal)) {
663       nextToken();
664       if (FormatTok->Tok.is(tok::l_brace)) {
665         parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/false);
666         addUnwrappedLine();
667         return;
668       }
669     }
670     break;
671   case tok::identifier:
672     if (FormatTok->IsForEachMacro) {
673       parseForOrWhileLoop();
674       return;
675     }
676     // In all other cases, parse the declaration.
677     break;
678   default:
679     break;
680   }
681   do {
682     switch (FormatTok->Tok.getKind()) {
683     case tok::at:
684       nextToken();
685       if (FormatTok->Tok.is(tok::l_brace))
686         parseBracedList();
687       break;
688     case tok::kw_enum:
689       parseEnum();
690       break;
691     case tok::kw_typedef:
692       nextToken();
693       // FIXME: Use the IdentifierTable instead.
694       if (FormatTok->TokenText == "NS_ENUM")
695         parseEnum();
696       break;
697     case tok::kw_struct:
698     case tok::kw_union:
699     case tok::kw_class:
700       parseRecord();
701       // A record declaration or definition is always the start of a structural
702       // element.
703       break;
704     case tok::semi:
705       nextToken();
706       addUnwrappedLine();
707       return;
708     case tok::r_brace:
709       addUnwrappedLine();
710       return;
711     case tok::l_paren:
712       parseParens();
713       break;
714     case tok::caret:
715       nextToken();
716       if (FormatTok->Tok.isAnyIdentifier() ||
717           FormatTok->isSimpleTypeSpecifier())
718         nextToken();
719       if (FormatTok->is(tok::l_paren))
720         parseParens();
721       if (FormatTok->is(tok::l_brace))
722         parseChildBlock();
723       break;
724     case tok::l_brace:
725       if (!tryToParseBracedList()) {
726         // A block outside of parentheses must be the last part of a
727         // structural element.
728         // FIXME: Figure out cases where this is not true, and add projections
729         // for them (the one we know is missing are lambdas).
730         if (Style.BreakBeforeBraces != FormatStyle::BS_Attach)
731           addUnwrappedLine();
732         FormatTok->Type = TT_FunctionLBrace;
733         parseBlock(/*MustBeDeclaration=*/false);
734         addUnwrappedLine();
735         return;
736       }
737       // Otherwise this was a braced init list, and the structural
738       // element continues.
739       break;
740     case tok::identifier: {
741       StringRef Text = FormatTok->TokenText;
742       nextToken();
743       if (Line->Tokens.size() == 1) {
744         if (FormatTok->Tok.is(tok::colon)) {
745           parseLabel();
746           return;
747         }
748         // Recognize function-like macro usages without trailing semicolon.
749         if (FormatTok->Tok.is(tok::l_paren)) {
750           parseParens();
751           if (FormatTok->NewlinesBefore > 0 &&
752               tokenCanStartNewLine(FormatTok->Tok) && Text == Text.upper()) {
753             addUnwrappedLine();
754             return;
755           }
756         } else if (FormatTok->HasUnescapedNewline && Text.size() >= 5 &&
757                    Text == Text.upper()) {
758           // Recognize free-standing macros like Q_OBJECT.
759           addUnwrappedLine();
760           return;
761         }
762       }
763       break;
764     }
765     case tok::equal:
766       nextToken();
767       if (FormatTok->Tok.is(tok::l_brace)) {
768         parseBracedList();
769       }
770       break;
771     case tok::l_square:
772       parseSquare();
773       break;
774     default:
775       nextToken();
776       break;
777     }
778   } while (!eof());
779 }
780 
781 bool UnwrappedLineParser::tryToParseLambda() {
782   // FIXME: This is a dirty way to access the previous token. Find a better
783   // solution.
784   if (!Line->Tokens.empty() &&
785       (Line->Tokens.back().Tok->isOneOf(tok::identifier, tok::kw_operator) ||
786        Line->Tokens.back().Tok->closesScope() ||
787        Line->Tokens.back().Tok->isSimpleTypeSpecifier())) {
788     nextToken();
789     return false;
790   }
791   assert(FormatTok->is(tok::l_square));
792   FormatToken &LSquare = *FormatTok;
793   if (!tryToParseLambdaIntroducer())
794     return false;
795 
796   while (FormatTok->isNot(tok::l_brace)) {
797     if (FormatTok->isSimpleTypeSpecifier()) {
798       nextToken();
799       continue;
800     }
801     switch (FormatTok->Tok.getKind()) {
802     case tok::l_brace:
803       break;
804     case tok::l_paren:
805       parseParens();
806       break;
807     case tok::less:
808     case tok::greater:
809     case tok::identifier:
810     case tok::coloncolon:
811     case tok::kw_mutable:
812       nextToken();
813       break;
814     case tok::arrow:
815       FormatTok->Type = TT_TrailingReturnArrow;
816       nextToken();
817       break;
818     default:
819       return true;
820     }
821   }
822   LSquare.Type = TT_LambdaLSquare;
823   parseChildBlock();
824   return true;
825 }
826 
827 bool UnwrappedLineParser::tryToParseLambdaIntroducer() {
828   nextToken();
829   if (FormatTok->is(tok::equal)) {
830     nextToken();
831     if (FormatTok->is(tok::r_square)) {
832       nextToken();
833       return true;
834     }
835     if (FormatTok->isNot(tok::comma))
836       return false;
837     nextToken();
838   } else if (FormatTok->is(tok::amp)) {
839     nextToken();
840     if (FormatTok->is(tok::r_square)) {
841       nextToken();
842       return true;
843     }
844     if (!FormatTok->isOneOf(tok::comma, tok::identifier)) {
845       return false;
846     }
847     if (FormatTok->is(tok::comma))
848       nextToken();
849   } else if (FormatTok->is(tok::r_square)) {
850     nextToken();
851     return true;
852   }
853   do {
854     if (FormatTok->is(tok::amp))
855       nextToken();
856     if (!FormatTok->isOneOf(tok::identifier, tok::kw_this))
857       return false;
858     nextToken();
859     if (FormatTok->is(tok::comma)) {
860       nextToken();
861     } else if (FormatTok->is(tok::r_square)) {
862       nextToken();
863       return true;
864     } else {
865       return false;
866     }
867   } while (!eof());
868   return false;
869 }
870 
871 bool UnwrappedLineParser::tryToParseBracedList() {
872   if (FormatTok->BlockKind == BK_Unknown)
873     calculateBraceTypes();
874   assert(FormatTok->BlockKind != BK_Unknown);
875   if (FormatTok->BlockKind == BK_Block)
876     return false;
877   parseBracedList();
878   return true;
879 }
880 
881 bool UnwrappedLineParser::parseBracedList(bool ContinueOnSemicolons) {
882   bool HasError = false;
883   nextToken();
884 
885   // FIXME: Once we have an expression parser in the UnwrappedLineParser,
886   // replace this by using parseAssigmentExpression() inside.
887   do {
888     // FIXME: When we start to support lambdas, we'll want to parse them away
889     // here, otherwise our bail-out scenarios below break. The better solution
890     // might be to just implement a more or less complete expression parser.
891     switch (FormatTok->Tok.getKind()) {
892     case tok::caret:
893       nextToken();
894       if (FormatTok->is(tok::l_brace)) {
895         parseChildBlock();
896       }
897       break;
898     case tok::l_square:
899       tryToParseLambda();
900       break;
901     case tok::l_brace:
902       // Assume there are no blocks inside a braced init list apart
903       // from the ones we explicitly parse out (like lambdas).
904       FormatTok->BlockKind = BK_BracedInit;
905       parseBracedList();
906       break;
907     case tok::r_brace:
908       nextToken();
909       return !HasError;
910     case tok::semi:
911       HasError = true;
912       if (!ContinueOnSemicolons)
913         return !HasError;
914       nextToken();
915       break;
916     case tok::comma:
917       nextToken();
918       break;
919     default:
920       nextToken();
921       break;
922     }
923   } while (!eof());
924   return false;
925 }
926 
927 void UnwrappedLineParser::parseParens() {
928   assert(FormatTok->Tok.is(tok::l_paren) && "'(' expected.");
929   nextToken();
930   do {
931     switch (FormatTok->Tok.getKind()) {
932     case tok::l_paren:
933       parseParens();
934       break;
935     case tok::r_paren:
936       nextToken();
937       return;
938     case tok::r_brace:
939       // A "}" inside parenthesis is an error if there wasn't a matching "{".
940       return;
941     case tok::l_square:
942       tryToParseLambda();
943       break;
944     case tok::l_brace: {
945       if (!tryToParseBracedList()) {
946         parseChildBlock();
947       }
948       break;
949     }
950     case tok::at:
951       nextToken();
952       if (FormatTok->Tok.is(tok::l_brace))
953         parseBracedList();
954       break;
955     default:
956       nextToken();
957       break;
958     }
959   } while (!eof());
960 }
961 
962 void UnwrappedLineParser::parseSquare() {
963   assert(FormatTok->Tok.is(tok::l_square) && "'[' expected.");
964   if (tryToParseLambda())
965     return;
966   do {
967     switch (FormatTok->Tok.getKind()) {
968     case tok::l_paren:
969       parseParens();
970       break;
971     case tok::r_square:
972       nextToken();
973       return;
974     case tok::r_brace:
975       // A "}" inside parenthesis is an error if there wasn't a matching "{".
976       return;
977     case tok::l_square:
978       parseSquare();
979       break;
980     case tok::l_brace: {
981       if (!tryToParseBracedList()) {
982         parseChildBlock();
983       }
984       break;
985     }
986     case tok::at:
987       nextToken();
988       if (FormatTok->Tok.is(tok::l_brace))
989         parseBracedList();
990       break;
991     default:
992       nextToken();
993       break;
994     }
995   } while (!eof());
996 }
997 
998 void UnwrappedLineParser::parseIfThenElse() {
999   assert(FormatTok->Tok.is(tok::kw_if) && "'if' expected");
1000   nextToken();
1001   if (FormatTok->Tok.is(tok::l_paren))
1002     parseParens();
1003   bool NeedsUnwrappedLine = false;
1004   if (FormatTok->Tok.is(tok::l_brace)) {
1005     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1006     parseBlock(/*MustBeDeclaration=*/false);
1007     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1008         Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1009       addUnwrappedLine();
1010     } else {
1011       NeedsUnwrappedLine = true;
1012     }
1013   } else {
1014     addUnwrappedLine();
1015     ++Line->Level;
1016     parseStructuralElement();
1017     --Line->Level;
1018   }
1019   if (FormatTok->Tok.is(tok::kw_else)) {
1020     nextToken();
1021     if (FormatTok->Tok.is(tok::l_brace)) {
1022       CompoundStatementIndenter Indenter(this, Style, Line->Level);
1023       parseBlock(/*MustBeDeclaration=*/false);
1024       addUnwrappedLine();
1025     } else if (FormatTok->Tok.is(tok::kw_if)) {
1026       parseIfThenElse();
1027     } else {
1028       addUnwrappedLine();
1029       ++Line->Level;
1030       parseStructuralElement();
1031       --Line->Level;
1032     }
1033   } else if (NeedsUnwrappedLine) {
1034     addUnwrappedLine();
1035   }
1036 }
1037 
1038 void UnwrappedLineParser::parseNamespace() {
1039   assert(FormatTok->Tok.is(tok::kw_namespace) && "'namespace' expected");
1040   nextToken();
1041   if (FormatTok->Tok.is(tok::identifier))
1042     nextToken();
1043   if (FormatTok->Tok.is(tok::l_brace)) {
1044     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1045         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1046         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1047       addUnwrappedLine();
1048 
1049     bool AddLevel = Style.NamespaceIndentation == FormatStyle::NI_All ||
1050                     (Style.NamespaceIndentation == FormatStyle::NI_Inner &&
1051                      DeclarationScopeStack.size() > 1);
1052     parseBlock(/*MustBeDeclaration=*/true, AddLevel);
1053     // Munch the semicolon after a namespace. This is more common than one would
1054     // think. Puttin the semicolon into its own line is very ugly.
1055     if (FormatTok->Tok.is(tok::semi))
1056       nextToken();
1057     addUnwrappedLine();
1058   }
1059   // FIXME: Add error handling.
1060 }
1061 
1062 void UnwrappedLineParser::parseForOrWhileLoop() {
1063   assert((FormatTok->Tok.is(tok::kw_for) || FormatTok->Tok.is(tok::kw_while) ||
1064           FormatTok->IsForEachMacro) &&
1065          "'for', 'while' or foreach macro expected");
1066   nextToken();
1067   if (FormatTok->Tok.is(tok::l_paren))
1068     parseParens();
1069   if (FormatTok->Tok.is(tok::l_brace)) {
1070     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1071     parseBlock(/*MustBeDeclaration=*/false);
1072     addUnwrappedLine();
1073   } else {
1074     addUnwrappedLine();
1075     ++Line->Level;
1076     parseStructuralElement();
1077     --Line->Level;
1078   }
1079 }
1080 
1081 void UnwrappedLineParser::parseDoWhile() {
1082   assert(FormatTok->Tok.is(tok::kw_do) && "'do' expected");
1083   nextToken();
1084   if (FormatTok->Tok.is(tok::l_brace)) {
1085     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1086     parseBlock(/*MustBeDeclaration=*/false);
1087     if (Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1088       addUnwrappedLine();
1089   } else {
1090     addUnwrappedLine();
1091     ++Line->Level;
1092     parseStructuralElement();
1093     --Line->Level;
1094   }
1095 
1096   // FIXME: Add error handling.
1097   if (!FormatTok->Tok.is(tok::kw_while)) {
1098     addUnwrappedLine();
1099     return;
1100   }
1101 
1102   nextToken();
1103   parseStructuralElement();
1104 }
1105 
1106 void UnwrappedLineParser::parseLabel() {
1107   nextToken();
1108   unsigned OldLineLevel = Line->Level;
1109   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
1110     --Line->Level;
1111   if (CommentsBeforeNextToken.empty() && FormatTok->Tok.is(tok::l_brace)) {
1112     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1113     parseBlock(/*MustBeDeclaration=*/false);
1114     if (FormatTok->Tok.is(tok::kw_break)) {
1115       // "break;" after "}" on its own line only for BS_Allman and BS_GNU
1116       if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1117           Style.BreakBeforeBraces == FormatStyle::BS_GNU) {
1118         addUnwrappedLine();
1119       }
1120       parseStructuralElement();
1121     }
1122     addUnwrappedLine();
1123   } else {
1124     addUnwrappedLine();
1125   }
1126   Line->Level = OldLineLevel;
1127 }
1128 
1129 void UnwrappedLineParser::parseCaseLabel() {
1130   assert(FormatTok->Tok.is(tok::kw_case) && "'case' expected");
1131   // FIXME: fix handling of complex expressions here.
1132   do {
1133     nextToken();
1134   } while (!eof() && !FormatTok->Tok.is(tok::colon));
1135   parseLabel();
1136 }
1137 
1138 void UnwrappedLineParser::parseSwitch() {
1139   assert(FormatTok->Tok.is(tok::kw_switch) && "'switch' expected");
1140   nextToken();
1141   if (FormatTok->Tok.is(tok::l_paren))
1142     parseParens();
1143   if (FormatTok->Tok.is(tok::l_brace)) {
1144     CompoundStatementIndenter Indenter(this, Style, Line->Level);
1145     parseBlock(/*MustBeDeclaration=*/false);
1146     addUnwrappedLine();
1147   } else {
1148     addUnwrappedLine();
1149     ++Line->Level;
1150     parseStructuralElement();
1151     --Line->Level;
1152   }
1153 }
1154 
1155 void UnwrappedLineParser::parseAccessSpecifier() {
1156   nextToken();
1157   // Understand Qt's slots.
1158   if (FormatTok->is(tok::identifier) &&
1159       (FormatTok->TokenText == "slots" || FormatTok->TokenText == "Q_SLOTS"))
1160     nextToken();
1161   // Otherwise, we don't know what it is, and we'd better keep the next token.
1162   if (FormatTok->Tok.is(tok::colon))
1163     nextToken();
1164   addUnwrappedLine();
1165 }
1166 
1167 void UnwrappedLineParser::parseEnum() {
1168   if (FormatTok->Tok.is(tok::kw_enum)) {
1169     // Won't be 'enum' for NS_ENUMs.
1170     nextToken();
1171   }
1172   // Eat up enum class ...
1173   if (FormatTok->Tok.is(tok::kw_class) ||
1174       FormatTok->Tok.is(tok::kw_struct))
1175       nextToken();
1176   while (FormatTok->Tok.getIdentifierInfo() ||
1177          FormatTok->isOneOf(tok::colon, tok::coloncolon)) {
1178     nextToken();
1179     // We can have macros or attributes in between 'enum' and the enum name.
1180     if (FormatTok->Tok.is(tok::l_paren)) {
1181       parseParens();
1182     }
1183     if (FormatTok->Tok.is(tok::identifier))
1184       nextToken();
1185   }
1186   if (FormatTok->Tok.is(tok::l_brace)) {
1187     FormatTok->BlockKind = BK_Block;
1188     bool HasError = !parseBracedList(/*ContinueOnSemicolons=*/true);
1189     if (HasError) {
1190       if (FormatTok->is(tok::semi))
1191         nextToken();
1192       addUnwrappedLine();
1193     }
1194   }
1195   // We fall through to parsing a structural element afterwards, so that in
1196   // enum A {} n, m;
1197   // "} n, m;" will end up in one unwrapped line.
1198 }
1199 
1200 void UnwrappedLineParser::parseRecord() {
1201   nextToken();
1202   if (FormatTok->Tok.is(tok::identifier) ||
1203       FormatTok->Tok.is(tok::kw___attribute) ||
1204       FormatTok->Tok.is(tok::kw___declspec) ||
1205       FormatTok->Tok.is(tok::kw_alignas)) {
1206     nextToken();
1207     // We can have macros or attributes in between 'class' and the class name.
1208     if (FormatTok->Tok.is(tok::l_paren)) {
1209       parseParens();
1210     }
1211     // The actual identifier can be a nested name specifier, and in macros
1212     // it is often token-pasted.
1213     while (FormatTok->Tok.is(tok::identifier) ||
1214            FormatTok->Tok.is(tok::coloncolon) ||
1215            FormatTok->Tok.is(tok::hashhash))
1216       nextToken();
1217 
1218     // Note that parsing away template declarations here leads to incorrectly
1219     // accepting function declarations as record declarations.
1220     // In general, we cannot solve this problem. Consider:
1221     // class A<int> B() {}
1222     // which can be a function definition or a class definition when B() is a
1223     // macro. If we find enough real-world cases where this is a problem, we
1224     // can parse for the 'template' keyword in the beginning of the statement,
1225     // and thus rule out the record production in case there is no template
1226     // (this would still leave us with an ambiguity between template function
1227     // and class declarations).
1228     if (FormatTok->Tok.is(tok::colon) || FormatTok->Tok.is(tok::less)) {
1229       while (!eof() && FormatTok->Tok.isNot(tok::l_brace)) {
1230         if (FormatTok->Tok.is(tok::semi))
1231           return;
1232         nextToken();
1233       }
1234     }
1235   }
1236   if (FormatTok->Tok.is(tok::l_brace)) {
1237     if (Style.BreakBeforeBraces == FormatStyle::BS_Linux ||
1238         Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1239         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1240       addUnwrappedLine();
1241 
1242     parseBlock(/*MustBeDeclaration=*/true, /*AddLevel=*/true,
1243                /*MunchSemi=*/false);
1244   }
1245   // We fall through to parsing a structural element afterwards, so
1246   // class A {} n, m;
1247   // will end up in one unwrapped line.
1248 }
1249 
1250 void UnwrappedLineParser::parseObjCProtocolList() {
1251   assert(FormatTok->Tok.is(tok::less) && "'<' expected.");
1252   do
1253     nextToken();
1254   while (!eof() && FormatTok->Tok.isNot(tok::greater));
1255   nextToken(); // Skip '>'.
1256 }
1257 
1258 void UnwrappedLineParser::parseObjCUntilAtEnd() {
1259   do {
1260     if (FormatTok->Tok.isObjCAtKeyword(tok::objc_end)) {
1261       nextToken();
1262       addUnwrappedLine();
1263       break;
1264     }
1265     if (FormatTok->is(tok::l_brace)) {
1266       parseBlock(/*MustBeDeclaration=*/false);
1267       // In ObjC interfaces, nothing should be following the "}".
1268       addUnwrappedLine();
1269     } else if (FormatTok->is(tok::r_brace)) {
1270       // Ignore stray "}". parseStructuralElement doesn't consume them.
1271       nextToken();
1272       addUnwrappedLine();
1273     } else {
1274       parseStructuralElement();
1275     }
1276   } while (!eof());
1277 }
1278 
1279 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
1280   nextToken();
1281   nextToken(); // interface name
1282 
1283   // @interface can be followed by either a base class, or a category.
1284   if (FormatTok->Tok.is(tok::colon)) {
1285     nextToken();
1286     nextToken(); // base class name
1287   } else if (FormatTok->Tok.is(tok::l_paren))
1288     // Skip category, if present.
1289     parseParens();
1290 
1291   if (FormatTok->Tok.is(tok::less))
1292     parseObjCProtocolList();
1293 
1294   if (FormatTok->Tok.is(tok::l_brace)) {
1295     if (Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
1296         Style.BreakBeforeBraces == FormatStyle::BS_GNU)
1297       addUnwrappedLine();
1298     parseBlock(/*MustBeDeclaration=*/true);
1299   }
1300 
1301   // With instance variables, this puts '}' on its own line.  Without instance
1302   // variables, this ends the @interface line.
1303   addUnwrappedLine();
1304 
1305   parseObjCUntilAtEnd();
1306 }
1307 
1308 void UnwrappedLineParser::parseObjCProtocol() {
1309   nextToken();
1310   nextToken(); // protocol name
1311 
1312   if (FormatTok->Tok.is(tok::less))
1313     parseObjCProtocolList();
1314 
1315   // Check for protocol declaration.
1316   if (FormatTok->Tok.is(tok::semi)) {
1317     nextToken();
1318     return addUnwrappedLine();
1319   }
1320 
1321   addUnwrappedLine();
1322   parseObjCUntilAtEnd();
1323 }
1324 
1325 LLVM_ATTRIBUTE_UNUSED static void printDebugInfo(const UnwrappedLine &Line,
1326                                                  StringRef Prefix = "") {
1327   llvm::dbgs() << Prefix << "Line(" << Line.Level << ")"
1328                << (Line.InPPDirective ? " MACRO" : "") << ": ";
1329   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1330                                                     E = Line.Tokens.end();
1331        I != E; ++I) {
1332     llvm::dbgs() << I->Tok->Tok.getName() << "[" << I->Tok->Type << "] ";
1333   }
1334   for (std::list<UnwrappedLineNode>::const_iterator I = Line.Tokens.begin(),
1335                                                     E = Line.Tokens.end();
1336        I != E; ++I) {
1337     const UnwrappedLineNode &Node = *I;
1338     for (SmallVectorImpl<UnwrappedLine>::const_iterator
1339              I = Node.Children.begin(),
1340              E = Node.Children.end();
1341          I != E; ++I) {
1342       printDebugInfo(*I, "\nChild: ");
1343     }
1344   }
1345   llvm::dbgs() << "\n";
1346 }
1347 
1348 void UnwrappedLineParser::addUnwrappedLine() {
1349   if (Line->Tokens.empty())
1350     return;
1351   DEBUG({
1352     if (CurrentLines == &Lines)
1353       printDebugInfo(*Line);
1354   });
1355   CurrentLines->push_back(*Line);
1356   Line->Tokens.clear();
1357   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
1358     for (SmallVectorImpl<UnwrappedLine>::iterator
1359              I = PreprocessorDirectives.begin(),
1360              E = PreprocessorDirectives.end();
1361          I != E; ++I) {
1362       CurrentLines->push_back(*I);
1363     }
1364     PreprocessorDirectives.clear();
1365   }
1366 }
1367 
1368 bool UnwrappedLineParser::eof() const { return FormatTok->Tok.is(tok::eof); }
1369 
1370 bool UnwrappedLineParser::isOnNewLine(const FormatToken& FormatTok) {
1371   return (Line->InPPDirective || FormatTok.HasUnescapedNewline) &&
1372          FormatTok.NewlinesBefore > 0;
1373 }
1374 
1375 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
1376   bool JustComments = Line->Tokens.empty();
1377   for (SmallVectorImpl<FormatToken *>::const_iterator
1378            I = CommentsBeforeNextToken.begin(),
1379            E = CommentsBeforeNextToken.end();
1380        I != E; ++I) {
1381     if (isOnNewLine(**I) && JustComments) {
1382       addUnwrappedLine();
1383     }
1384     pushToken(*I);
1385   }
1386   if (NewlineBeforeNext && JustComments) {
1387     addUnwrappedLine();
1388   }
1389   CommentsBeforeNextToken.clear();
1390 }
1391 
1392 void UnwrappedLineParser::nextToken() {
1393   if (eof())
1394     return;
1395   flushComments(isOnNewLine(*FormatTok));
1396   pushToken(FormatTok);
1397   readToken();
1398 }
1399 
1400 void UnwrappedLineParser::readToken() {
1401   bool CommentsInCurrentLine = true;
1402   do {
1403     FormatTok = Tokens->getNextToken();
1404     assert(FormatTok);
1405     while (!Line->InPPDirective && FormatTok->Tok.is(tok::hash) &&
1406            (FormatTok->HasUnescapedNewline || FormatTok->IsFirst)) {
1407       // If there is an unfinished unwrapped line, we flush the preprocessor
1408       // directives only after that unwrapped line was finished later.
1409       bool SwitchToPreprocessorLines =
1410           !Line->Tokens.empty() && CurrentLines == &Lines;
1411       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
1412       // Comments stored before the preprocessor directive need to be output
1413       // before the preprocessor directive, at the same level as the
1414       // preprocessor directive, as we consider them to apply to the directive.
1415       flushComments(isOnNewLine(*FormatTok));
1416       parsePPDirective();
1417     }
1418     while (FormatTok->Type == TT_ConflictStart ||
1419            FormatTok->Type == TT_ConflictEnd ||
1420            FormatTok->Type == TT_ConflictAlternative) {
1421       if (FormatTok->Type == TT_ConflictStart) {
1422         conditionalCompilationStart(/*Unreachable=*/false);
1423       } else if (FormatTok->Type == TT_ConflictAlternative) {
1424         conditionalCompilationAlternative();
1425       } else if(FormatTok->Type == TT_ConflictEnd) {
1426         conditionalCompilationEnd();
1427       }
1428       FormatTok = Tokens->getNextToken();
1429       FormatTok->MustBreakBefore = true;
1430     }
1431 
1432     if (!PPStack.empty() && (PPStack.back() == PP_Unreachable) &&
1433         !Line->InPPDirective) {
1434       continue;
1435     }
1436 
1437     if (!FormatTok->Tok.is(tok::comment))
1438       return;
1439     if (isOnNewLine(*FormatTok) || FormatTok->IsFirst) {
1440       CommentsInCurrentLine = false;
1441     }
1442     if (CommentsInCurrentLine) {
1443       pushToken(FormatTok);
1444     } else {
1445       CommentsBeforeNextToken.push_back(FormatTok);
1446     }
1447   } while (!eof());
1448 }
1449 
1450 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
1451   Line->Tokens.push_back(UnwrappedLineNode(Tok));
1452   if (MustBreakBeforeNextToken) {
1453     Line->Tokens.back().Tok->MustBreakBefore = true;
1454     MustBreakBeforeNextToken = false;
1455   }
1456 }
1457 
1458 } // end namespace format
1459 } // end namespace clang
1460