1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "llvm/Support/Debug.h"
21 
22 // Uncomment to get debug output from tests:
23 // #define DEBUG_WITH_TYPE(T, X) do { X; } while(0)
24 
25 namespace clang {
26 namespace format {
27 
28 class ScopedDeclarationState {
29 public:
30   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
31                          bool MustBeDeclaration)
32       : Line(Line), Stack(Stack) {
33     Line.MustBeDeclaration = MustBeDeclaration;
34     Stack.push_back(MustBeDeclaration);
35   }
36   ~ScopedDeclarationState() {
37     Stack.pop_back();
38     if (!Stack.empty())
39       Line.MustBeDeclaration = Stack.back();
40     else
41       Line.MustBeDeclaration = true;
42   }
43 private:
44   UnwrappedLine &Line;
45   std::vector<bool> &Stack;
46 };
47 
48 class ScopedMacroState : public FormatTokenSource {
49 public:
50   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
51                    FormatToken &ResetToken)
52       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
53         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
54     TokenSource = this;
55     Line.Level = 0;
56     Line.InPPDirective = true;
57   }
58 
59   ~ScopedMacroState() {
60     TokenSource = PreviousTokenSource;
61     ResetToken = Token;
62     Line.InPPDirective = false;
63     Line.Level = PreviousLineLevel;
64   }
65 
66   virtual FormatToken getNextToken() {
67     // The \c UnwrappedLineParser guards against this by never calling
68     // \c getNextToken() after it has encountered the first eof token.
69     assert(!eof());
70     Token = PreviousTokenSource->getNextToken();
71     if (eof())
72       return createEOF();
73     return Token;
74   }
75 
76 private:
77   bool eof() {
78     return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
79   }
80 
81   FormatToken createEOF() {
82     FormatToken FormatTok;
83     FormatTok.Tok.startToken();
84     FormatTok.Tok.setKind(tok::eof);
85     return FormatTok;
86   }
87 
88   UnwrappedLine &Line;
89   FormatTokenSource *&TokenSource;
90   FormatToken &ResetToken;
91   unsigned PreviousLineLevel;
92   FormatTokenSource *PreviousTokenSource;
93 
94   FormatToken Token;
95 };
96 
97 class ScopedLineState {
98 public:
99   ScopedLineState(UnwrappedLineParser &Parser,
100                   bool SwitchToPreprocessorLines = false)
101       : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
102     if (SwitchToPreprocessorLines)
103       Parser.CurrentLines = &Parser.PreprocessorDirectives;
104     PreBlockLine = Parser.Line.take();
105     Parser.Line.reset(new UnwrappedLine());
106     Parser.Line->Level = PreBlockLine->Level;
107     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
108   }
109 
110   ~ScopedLineState() {
111     if (!Parser.Line->Tokens.empty()) {
112       Parser.addUnwrappedLine();
113     }
114     assert(Parser.Line->Tokens.empty());
115     Parser.Line.reset(PreBlockLine);
116     Parser.MustBreakBeforeNextToken = true;
117     if (SwitchToPreprocessorLines)
118       Parser.CurrentLines = &Parser.Lines;
119   }
120 
121 private:
122   UnwrappedLineParser &Parser;
123   const bool SwitchToPreprocessorLines;
124 
125   UnwrappedLine *PreBlockLine;
126 };
127 
128 UnwrappedLineParser::UnwrappedLineParser(
129     clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
130     FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
131     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
132       CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
133       Callback(Callback) {}
134 
135 bool UnwrappedLineParser::parse() {
136   DEBUG(llvm::dbgs() << "----\n");
137   readToken();
138   bool Error = parseFile();
139   for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
140                                             E = Lines.end();
141        I != E; ++I) {
142     Callback.consumeUnwrappedLine(*I);
143   }
144   return Error;
145 }
146 
147 bool UnwrappedLineParser::parseFile() {
148   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
149                                           /*MustBeDeclaration=*/ true);
150   bool Error = parseLevel(/*HasOpeningBrace=*/false);
151   // Make sure to format the remaining tokens.
152   flushComments(true);
153   addUnwrappedLine();
154   return Error;
155 }
156 
157 bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
158   bool Error = false;
159   do {
160     switch (FormatTok.Tok.getKind()) {
161     case tok::comment:
162       nextToken();
163       addUnwrappedLine();
164       break;
165     case tok::l_brace:
166       // FIXME: Add parameter whether this can happen - if this happens, we must
167       // be in a non-declaration context.
168       Error |= parseBlock(/*MustBeDeclaration=*/ false);
169       addUnwrappedLine();
170       break;
171     case tok::r_brace:
172       if (HasOpeningBrace) {
173         return false;
174       } else {
175         Diag.Report(FormatTok.Tok.getLocation(),
176                     Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
177                                          "unexpected '}'"));
178         Error = true;
179         nextToken();
180         addUnwrappedLine();
181       }
182       break;
183     default:
184       parseStructuralElement();
185       break;
186     }
187   } while (!eof());
188   return Error;
189 }
190 
191 bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration, unsigned AddLevels) {
192   assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
193   nextToken();
194 
195   addUnwrappedLine();
196 
197   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
198                                           MustBeDeclaration);
199   Line->Level += AddLevels;
200   parseLevel(/*HasOpeningBrace=*/true);
201 
202   if (!FormatTok.Tok.is(tok::r_brace)) {
203     Line->Level -= AddLevels;
204     return true;
205   }
206 
207   nextToken();  // Munch the closing brace.
208   Line->Level -= AddLevels;
209   return false;
210 }
211 
212 void UnwrappedLineParser::parsePPDirective() {
213   assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
214   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
215   nextToken();
216 
217   if (FormatTok.Tok.getIdentifierInfo() == NULL) {
218     parsePPUnknown();
219     return;
220   }
221 
222   switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
223   case tok::pp_define:
224     parsePPDefine();
225     break;
226   default:
227     parsePPUnknown();
228     break;
229   }
230 }
231 
232 void UnwrappedLineParser::parsePPDefine() {
233   nextToken();
234 
235   if (FormatTok.Tok.getKind() != tok::identifier) {
236     parsePPUnknown();
237     return;
238   }
239   nextToken();
240   if (FormatTok.Tok.getKind() == tok::l_paren &&
241       FormatTok.WhiteSpaceLength == 0) {
242     parseParens();
243   }
244   addUnwrappedLine();
245   Line->Level = 1;
246 
247   // Errors during a preprocessor directive can only affect the layout of the
248   // preprocessor directive, and thus we ignore them. An alternative approach
249   // would be to use the same approach we use on the file level (no
250   // re-indentation if there was a structural error) within the macro
251   // definition.
252   parseFile();
253 }
254 
255 void UnwrappedLineParser::parsePPUnknown() {
256   do {
257     nextToken();
258   } while (!eof());
259   addUnwrappedLine();
260 }
261 
262 void UnwrappedLineParser::parseStructuralElement() {
263   assert(!FormatTok.Tok.is(tok::l_brace));
264   int TokenNumber = 0;
265   switch (FormatTok.Tok.getKind()) {
266   case tok::at:
267     nextToken();
268     switch (FormatTok.Tok.getObjCKeywordID()) {
269     case tok::objc_public:
270     case tok::objc_protected:
271     case tok::objc_package:
272     case tok::objc_private:
273       return parseAccessSpecifier();
274     case tok::objc_interface:
275     case tok::objc_implementation:
276       return parseObjCInterfaceOrImplementation();
277     case tok::objc_protocol:
278       return parseObjCProtocol();
279     case tok::objc_end:
280       return; // Handled by the caller.
281     case tok::objc_optional:
282     case tok::objc_required:
283       nextToken();
284       addUnwrappedLine();
285       return;
286     default:
287       break;
288     }
289     break;
290   case tok::kw_namespace:
291     parseNamespace();
292     return;
293   case tok::kw_inline:
294     nextToken();
295     TokenNumber++;
296     if (FormatTok.Tok.is(tok::kw_namespace)) {
297       parseNamespace();
298       return;
299     }
300     break;
301   case tok::kw_public:
302   case tok::kw_protected:
303   case tok::kw_private:
304     parseAccessSpecifier();
305     return;
306   case tok::kw_if:
307     parseIfThenElse();
308     return;
309   case tok::kw_for:
310   case tok::kw_while:
311     parseForOrWhileLoop();
312     return;
313   case tok::kw_do:
314     parseDoWhile();
315     return;
316   case tok::kw_switch:
317     parseSwitch();
318     return;
319   case tok::kw_default:
320     nextToken();
321     parseLabel();
322     return;
323   case tok::kw_case:
324     parseCaseLabel();
325     return;
326   case tok::kw_return:
327     parseReturn();
328     return;
329   case tok::kw_extern:
330     nextToken();
331     if (FormatTok.Tok.is(tok::string_literal)) {
332       nextToken();
333       if (FormatTok.Tok.is(tok::l_brace)) {
334         parseBlock(/*MustBeDeclaration=*/ true, 0);
335         addUnwrappedLine();
336         return;
337       }
338     }
339     // In all other cases, parse the declaration.
340     break;
341   default:
342     break;
343   }
344   do {
345     ++TokenNumber;
346     switch (FormatTok.Tok.getKind()) {
347     case tok::kw_enum:
348       parseEnum();
349       break;
350     case tok::kw_struct:
351     case tok::kw_union:
352     case tok::kw_class:
353       parseRecord();
354       // A record declaration or definition is always the start of a structural
355       // element.
356       break;
357     case tok::semi:
358       nextToken();
359       addUnwrappedLine();
360       return;
361     case tok::r_brace:
362       addUnwrappedLine();
363       return;
364     case tok::l_paren:
365       parseParens();
366       break;
367     case tok::l_brace:
368       // A block outside of parentheses must be the last part of a
369       // structural element.
370       // FIXME: Figure out cases where this is not true, and add projections for
371       // them (the one we know is missing are lambdas).
372       parseBlock(/*MustBeDeclaration=*/ false);
373       addUnwrappedLine();
374       return;
375     case tok::identifier:
376       nextToken();
377       if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
378         parseLabel();
379         return;
380       }
381       break;
382     case tok::equal:
383       nextToken();
384       if (FormatTok.Tok.is(tok::l_brace)) {
385         parseBracedList();
386       }
387       break;
388     default:
389       nextToken();
390       break;
391     }
392   } while (!eof());
393 }
394 
395 void UnwrappedLineParser::parseBracedList() {
396   nextToken();
397 
398   do {
399     switch (FormatTok.Tok.getKind()) {
400     case tok::l_brace:
401       parseBracedList();
402       break;
403     case tok::r_brace:
404       nextToken();
405       return;
406     default:
407       nextToken();
408       break;
409     }
410   } while (!eof());
411 }
412 
413 void UnwrappedLineParser::parseReturn() {
414   nextToken();
415 
416   do {
417     switch (FormatTok.Tok.getKind()) {
418     case tok::l_brace:
419       parseBracedList();
420       break;
421     case tok::l_paren:
422       parseParens();
423       break;
424     case tok::r_brace:
425       // Assume missing ';'.
426       addUnwrappedLine();
427       return;
428     case tok::semi:
429       nextToken();
430       addUnwrappedLine();
431       return;
432     default:
433       nextToken();
434       break;
435     }
436   } while (!eof());
437 }
438 
439 void UnwrappedLineParser::parseParens() {
440   assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
441   nextToken();
442   do {
443     switch (FormatTok.Tok.getKind()) {
444     case tok::l_paren:
445       parseParens();
446       break;
447     case tok::r_paren:
448       nextToken();
449       return;
450     case tok::l_brace:
451       {
452         nextToken();
453         ScopedLineState LineState(*this);
454         ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
455                                                 /*MustBeDeclaration=*/ false);
456         Line->Level += 1;
457         parseLevel(/*HasOpeningBrace=*/ true);
458         Line->Level -= 1;
459       }
460       break;
461     default:
462       nextToken();
463       break;
464     }
465   } while (!eof());
466 }
467 
468 void UnwrappedLineParser::parseIfThenElse() {
469   assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
470   nextToken();
471   if (FormatTok.Tok.is(tok::l_paren))
472     parseParens();
473   bool NeedsUnwrappedLine = false;
474   if (FormatTok.Tok.is(tok::l_brace)) {
475     parseBlock(/*MustBeDeclaration=*/ false);
476     NeedsUnwrappedLine = true;
477   } else {
478     addUnwrappedLine();
479     ++Line->Level;
480     parseStructuralElement();
481     --Line->Level;
482   }
483   if (FormatTok.Tok.is(tok::kw_else)) {
484     nextToken();
485     if (FormatTok.Tok.is(tok::l_brace)) {
486       parseBlock(/*MustBeDeclaration=*/ false);
487       addUnwrappedLine();
488     } else if (FormatTok.Tok.is(tok::kw_if)) {
489       parseIfThenElse();
490     } else {
491       addUnwrappedLine();
492       ++Line->Level;
493       parseStructuralElement();
494       --Line->Level;
495     }
496   } else if (NeedsUnwrappedLine) {
497     addUnwrappedLine();
498   }
499 }
500 
501 void UnwrappedLineParser::parseNamespace() {
502   assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
503   nextToken();
504   if (FormatTok.Tok.is(tok::identifier))
505     nextToken();
506   if (FormatTok.Tok.is(tok::l_brace)) {
507     parseBlock(/*MustBeDeclaration=*/ true, 0);
508     // Munch the semicolon after a namespace. This is more common than one would
509     // think. Puttin the semicolon into its own line is very ugly.
510     if (FormatTok.Tok.is(tok::semi))
511       nextToken();
512     addUnwrappedLine();
513   }
514   // FIXME: Add error handling.
515 }
516 
517 void UnwrappedLineParser::parseForOrWhileLoop() {
518   assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
519          "'for' or 'while' expected");
520   nextToken();
521   if (FormatTok.Tok.is(tok::l_paren))
522     parseParens();
523   if (FormatTok.Tok.is(tok::l_brace)) {
524     parseBlock(/*MustBeDeclaration=*/ false);
525     addUnwrappedLine();
526   } else {
527     addUnwrappedLine();
528     ++Line->Level;
529     parseStructuralElement();
530     --Line->Level;
531   }
532 }
533 
534 void UnwrappedLineParser::parseDoWhile() {
535   assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
536   nextToken();
537   if (FormatTok.Tok.is(tok::l_brace)) {
538     parseBlock(/*MustBeDeclaration=*/ false);
539   } else {
540     addUnwrappedLine();
541     ++Line->Level;
542     parseStructuralElement();
543     --Line->Level;
544   }
545 
546   // FIXME: Add error handling.
547   if (!FormatTok.Tok.is(tok::kw_while)) {
548     addUnwrappedLine();
549     return;
550   }
551 
552   nextToken();
553   parseStructuralElement();
554 }
555 
556 void UnwrappedLineParser::parseLabel() {
557   // FIXME: remove all asserts.
558   assert(FormatTok.Tok.is(tok::colon) && "':' expected");
559   nextToken();
560   unsigned OldLineLevel = Line->Level;
561   if (Line->Level > 0)
562     --Line->Level;
563   if (FormatTok.Tok.is(tok::l_brace)) {
564     parseBlock(/*MustBeDeclaration=*/ false);
565     if (FormatTok.Tok.is(tok::kw_break))
566       parseStructuralElement(); // "break;" after "}" goes on the same line.
567   }
568   addUnwrappedLine();
569   Line->Level = OldLineLevel;
570 }
571 
572 void UnwrappedLineParser::parseCaseLabel() {
573   assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
574   // FIXME: fix handling of complex expressions here.
575   do {
576     nextToken();
577   } while (!eof() && !FormatTok.Tok.is(tok::colon));
578   parseLabel();
579 }
580 
581 void UnwrappedLineParser::parseSwitch() {
582   assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
583   nextToken();
584   if (FormatTok.Tok.is(tok::l_paren))
585     parseParens();
586   if (FormatTok.Tok.is(tok::l_brace)) {
587     parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
588     addUnwrappedLine();
589   } else {
590     addUnwrappedLine();
591     Line->Level += (Style.IndentCaseLabels ? 2 : 1);
592     parseStructuralElement();
593     Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
594   }
595 }
596 
597 void UnwrappedLineParser::parseAccessSpecifier() {
598   nextToken();
599   // Otherwise, we don't know what it is, and we'd better keep the next token.
600   if (FormatTok.Tok.is(tok::colon))
601     nextToken();
602   addUnwrappedLine();
603 }
604 
605 void UnwrappedLineParser::parseEnum() {
606   nextToken();
607   if (FormatTok.Tok.is(tok::identifier) ||
608       FormatTok.Tok.is(tok::kw___attribute) ||
609       FormatTok.Tok.is(tok::kw___declspec)) {
610     nextToken();
611     // We can have macros or attributes in between 'enum' and the enum name.
612     if (FormatTok.Tok.is(tok::l_paren)) {
613       parseParens();
614     }
615     if (FormatTok.Tok.is(tok::identifier))
616       nextToken();
617   }
618   if (FormatTok.Tok.is(tok::l_brace)) {
619     nextToken();
620     addUnwrappedLine();
621     ++Line->Level;
622     do {
623       switch (FormatTok.Tok.getKind()) {
624       case tok::l_paren:
625         parseParens();
626         break;
627       case tok::r_brace:
628         addUnwrappedLine();
629         nextToken();
630         --Line->Level;
631         return;
632       case tok::comma:
633         nextToken();
634         addUnwrappedLine();
635         break;
636       default:
637         nextToken();
638         break;
639       }
640     } while (!eof());
641   }
642   // We fall through to parsing a structural element afterwards, so that in
643   // enum A {} n, m;
644   // "} n, m;" will end up in one unwrapped line.
645 }
646 
647 void UnwrappedLineParser::parseRecord() {
648   nextToken();
649   if (FormatTok.Tok.is(tok::identifier) ||
650       FormatTok.Tok.is(tok::kw___attribute) ||
651       FormatTok.Tok.is(tok::kw___declspec)) {
652     nextToken();
653     // We can have macros or attributes in between 'class' and the class name.
654     if (FormatTok.Tok.is(tok::l_paren)) {
655       parseParens();
656     }
657     // The actual identifier can be a nested name specifier, and in macros
658     // it is often token-pasted.
659     while (FormatTok.Tok.is(tok::identifier) ||
660            FormatTok.Tok.is(tok::coloncolon) ||
661            FormatTok.Tok.is(tok::hashhash))
662       nextToken();
663 
664     // Note that parsing away template declarations here leads to incorrectly
665     // accepting function declarations as record declarations.
666     // In general, we cannot solve this problem. Consider:
667     // class A<int> B() {}
668     // which can be a function definition or a class definition when B() is a
669     // macro. If we find enough real-world cases where this is a problem, we
670     // can parse for the 'template' keyword in the beginning of the statement,
671     // and thus rule out the record production in case there is no template
672     // (this would still leave us with an ambiguity between template function
673     // and class declarations).
674     if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
675       while (FormatTok.Tok.isNot(tok::l_brace)) {
676         if (FormatTok.Tok.is(tok::semi))
677           return;
678         nextToken();
679       }
680     }
681   }
682   if (FormatTok.Tok.is(tok::l_brace))
683     parseBlock(/*MustBeDeclaration=*/ true);
684   // We fall through to parsing a structural element afterwards, so
685   // class A {} n, m;
686   // will end up in one unwrapped line.
687 }
688 
689 void UnwrappedLineParser::parseObjCProtocolList() {
690   assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
691   do
692     nextToken();
693   while (!eof() && FormatTok.Tok.isNot(tok::greater));
694   nextToken(); // Skip '>'.
695 }
696 
697 void UnwrappedLineParser::parseObjCUntilAtEnd() {
698   do {
699     if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
700       nextToken();
701       addUnwrappedLine();
702       break;
703     }
704     parseStructuralElement();
705   } while (!eof());
706 }
707 
708 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
709   nextToken();
710   nextToken();  // interface name
711 
712   // @interface can be followed by either a base class, or a category.
713   if (FormatTok.Tok.is(tok::colon)) {
714     nextToken();
715     nextToken();  // base class name
716   } else if (FormatTok.Tok.is(tok::l_paren))
717     // Skip category, if present.
718     parseParens();
719 
720   if (FormatTok.Tok.is(tok::less))
721     parseObjCProtocolList();
722 
723   // If instance variables are present, keep the '{' on the first line too.
724   if (FormatTok.Tok.is(tok::l_brace))
725     parseBlock(/*MustBeDeclaration=*/ true);
726 
727   // With instance variables, this puts '}' on its own line.  Without instance
728   // variables, this ends the @interface line.
729   addUnwrappedLine();
730 
731   parseObjCUntilAtEnd();
732 }
733 
734 void UnwrappedLineParser::parseObjCProtocol() {
735   nextToken();
736   nextToken();  // protocol name
737 
738   if (FormatTok.Tok.is(tok::less))
739     parseObjCProtocolList();
740 
741   // Check for protocol declaration.
742   if (FormatTok.Tok.is(tok::semi)) {
743     nextToken();
744     return addUnwrappedLine();
745   }
746 
747   addUnwrappedLine();
748   parseObjCUntilAtEnd();
749 }
750 
751 void UnwrappedLineParser::addUnwrappedLine() {
752   if (Line->Tokens.empty())
753     return;
754   DEBUG({
755     llvm::dbgs() << "Line(" << Line->Level << "): ";
756     for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
757                                           E = Line->Tokens.end();
758          I != E; ++I) {
759       llvm::dbgs() << I->Tok.getName() << " ";
760 
761     }
762     llvm::dbgs() << "\n";
763   });
764   CurrentLines->push_back(*Line);
765   Line->Tokens.clear();
766   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
767     for (std::vector<UnwrappedLine>::iterator I = PreprocessorDirectives
768              .begin(), E = PreprocessorDirectives.end();
769          I != E; ++I) {
770       CurrentLines->push_back(*I);
771     }
772     PreprocessorDirectives.clear();
773   }
774 
775 }
776 
777 bool UnwrappedLineParser::eof() const {
778   return FormatTok.Tok.is(tok::eof);
779 }
780 
781 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
782   bool JustComments = Line->Tokens.empty();
783   for (SmallVectorImpl<FormatToken>::const_iterator
784            I = CommentsBeforeNextToken.begin(),
785            E = CommentsBeforeNextToken.end();
786        I != E; ++I) {
787     if (I->NewlinesBefore && JustComments) {
788       addUnwrappedLine();
789     }
790     pushToken(*I);
791   }
792   if (NewlineBeforeNext && JustComments) {
793     addUnwrappedLine();
794   }
795   CommentsBeforeNextToken.clear();
796 }
797 
798 void UnwrappedLineParser::nextToken() {
799   if (eof())
800     return;
801   flushComments(FormatTok.NewlinesBefore > 0);
802   pushToken(FormatTok);
803   readToken();
804 }
805 
806 void UnwrappedLineParser::readToken() {
807   bool CommentsInCurrentLine = true;
808   do {
809     FormatTok = Tokens->getNextToken();
810     while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
811            ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
812             FormatTok.IsFirst)) {
813       // If there is an unfinished unwrapped line, we flush the preprocessor
814       // directives only after that unwrapped line was finished later.
815       bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
816                                        CurrentLines == &Lines;
817       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
818       parsePPDirective();
819     }
820     if (!FormatTok.Tok.is(tok::comment))
821       return;
822     if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
823       CommentsInCurrentLine = false;
824     }
825     if (CommentsInCurrentLine) {
826       pushToken(FormatTok);
827     } else {
828       CommentsBeforeNextToken.push_back(FormatTok);
829     }
830   } while (!eof());
831 }
832 
833 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
834   Line->Tokens.push_back(Tok);
835   if (MustBreakBeforeNextToken) {
836     Line->Tokens.back().MustBreakBefore = true;
837     MustBreakBeforeNextToken = false;
838   }
839 }
840 
841 } // end namespace format
842 } // end namespace clang
843