1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "llvm/Support/Debug.h"
21 
22 namespace clang {
23 namespace format {
24 
25 class ScopedDeclarationState {
26 public:
27   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28                          bool MustBeDeclaration)
29       : Line(Line), Stack(Stack) {
30     Line.MustBeDeclaration = MustBeDeclaration;
31     Stack.push_back(MustBeDeclaration);
32   }
33   ~ScopedDeclarationState() {
34     Stack.pop_back();
35     if (!Stack.empty())
36       Line.MustBeDeclaration = Stack.back();
37     else
38       Line.MustBeDeclaration = true;
39   }
40 private:
41   UnwrappedLine &Line;
42   std::vector<bool> &Stack;
43 };
44 
45 class ScopedMacroState : public FormatTokenSource {
46 public:
47   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48                    FormatToken &ResetToken)
49       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
51     TokenSource = this;
52     Line.Level = 0;
53     Line.InPPDirective = true;
54   }
55 
56   ~ScopedMacroState() {
57     TokenSource = PreviousTokenSource;
58     ResetToken = Token;
59     Line.InPPDirective = false;
60     Line.Level = PreviousLineLevel;
61   }
62 
63   virtual FormatToken getNextToken() {
64     // The \c UnwrappedLineParser guards against this by never calling
65     // \c getNextToken() after it has encountered the first eof token.
66     assert(!eof());
67     Token = PreviousTokenSource->getNextToken();
68     if (eof())
69       return createEOF();
70     return Token;
71   }
72 
73 private:
74   bool eof() {
75     return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
76   }
77 
78   FormatToken createEOF() {
79     FormatToken FormatTok;
80     FormatTok.Tok.startToken();
81     FormatTok.Tok.setKind(tok::eof);
82     return FormatTok;
83   }
84 
85   UnwrappedLine &Line;
86   FormatTokenSource *&TokenSource;
87   FormatToken &ResetToken;
88   unsigned PreviousLineLevel;
89   FormatTokenSource *PreviousTokenSource;
90 
91   FormatToken Token;
92 };
93 
94 class ScopedLineState {
95 public:
96   ScopedLineState(UnwrappedLineParser &Parser,
97                   bool SwitchToPreprocessorLines = false)
98       : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
99     if (SwitchToPreprocessorLines)
100       Parser.CurrentLines = &Parser.PreprocessorDirectives;
101     PreBlockLine = Parser.Line.take();
102     Parser.Line.reset(new UnwrappedLine());
103     Parser.Line->Level = PreBlockLine->Level;
104     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
105   }
106 
107   ~ScopedLineState() {
108     if (!Parser.Line->Tokens.empty()) {
109       Parser.addUnwrappedLine();
110     }
111     assert(Parser.Line->Tokens.empty());
112     Parser.Line.reset(PreBlockLine);
113     Parser.MustBreakBeforeNextToken = true;
114     if (SwitchToPreprocessorLines)
115       Parser.CurrentLines = &Parser.Lines;
116   }
117 
118 private:
119   UnwrappedLineParser &Parser;
120   const bool SwitchToPreprocessorLines;
121 
122   UnwrappedLine *PreBlockLine;
123 };
124 
125 UnwrappedLineParser::UnwrappedLineParser(
126     clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
127     FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
128     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
129       CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
130       Callback(Callback) {}
131 
132 bool UnwrappedLineParser::parse() {
133   DEBUG(llvm::dbgs() << "----\n");
134   readToken();
135   bool Error = parseFile();
136   for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
137                                             E = Lines.end();
138        I != E; ++I) {
139     Callback.consumeUnwrappedLine(*I);
140   }
141   return Error;
142 }
143 
144 bool UnwrappedLineParser::parseFile() {
145   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
146                                           /*MustBeDeclaration=*/ true);
147   bool Error = parseLevel(/*HasOpeningBrace=*/false);
148   // Make sure to format the remaining tokens.
149   flushComments(true);
150   addUnwrappedLine();
151   return Error;
152 }
153 
154 bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
155   bool Error = false;
156   do {
157     switch (FormatTok.Tok.getKind()) {
158     case tok::comment:
159       nextToken();
160       addUnwrappedLine();
161       break;
162     case tok::l_brace:
163       // FIXME: Add parameter whether this can happen - if this happens, we must
164       // be in a non-declaration context.
165       Error |= parseBlock(/*MustBeDeclaration=*/ false);
166       addUnwrappedLine();
167       break;
168     case tok::r_brace:
169       if (HasOpeningBrace) {
170         return false;
171       } else {
172         Diag.Report(FormatTok.Tok.getLocation(),
173                     Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
174                                          "unexpected '}'"));
175         Error = true;
176         nextToken();
177         addUnwrappedLine();
178       }
179       break;
180     default:
181       parseStructuralElement();
182       break;
183     }
184   } while (!eof());
185   return Error;
186 }
187 
188 bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
189                                      unsigned AddLevels) {
190   assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
191   nextToken();
192 
193   addUnwrappedLine();
194 
195   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
196                                           MustBeDeclaration);
197   Line->Level += AddLevels;
198   parseLevel(/*HasOpeningBrace=*/true);
199 
200   if (!FormatTok.Tok.is(tok::r_brace)) {
201     Line->Level -= AddLevels;
202     return true;
203   }
204 
205   nextToken();  // Munch the closing brace.
206   Line->Level -= AddLevels;
207   return false;
208 }
209 
210 void UnwrappedLineParser::parsePPDirective() {
211   assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
212   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
213   nextToken();
214 
215   if (FormatTok.Tok.getIdentifierInfo() == NULL) {
216     parsePPUnknown();
217     return;
218   }
219 
220   switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
221   case tok::pp_define:
222     parsePPDefine();
223     break;
224   default:
225     parsePPUnknown();
226     break;
227   }
228 }
229 
230 void UnwrappedLineParser::parsePPDefine() {
231   nextToken();
232 
233   if (FormatTok.Tok.getKind() != tok::identifier) {
234     parsePPUnknown();
235     return;
236   }
237   nextToken();
238   if (FormatTok.Tok.getKind() == tok::l_paren &&
239       FormatTok.WhiteSpaceLength == 0) {
240     parseParens();
241   }
242   addUnwrappedLine();
243   Line->Level = 1;
244 
245   // Errors during a preprocessor directive can only affect the layout of the
246   // preprocessor directive, and thus we ignore them. An alternative approach
247   // would be to use the same approach we use on the file level (no
248   // re-indentation if there was a structural error) within the macro
249   // definition.
250   parseFile();
251 }
252 
253 void UnwrappedLineParser::parsePPUnknown() {
254   do {
255     nextToken();
256   } while (!eof());
257   addUnwrappedLine();
258 }
259 
260 void UnwrappedLineParser::parseStructuralElement() {
261   assert(!FormatTok.Tok.is(tok::l_brace));
262   int TokenNumber = 0;
263   switch (FormatTok.Tok.getKind()) {
264   case tok::at:
265     nextToken();
266     if (FormatTok.Tok.is(tok::l_brace)) {
267       parseBracedList();
268       break;
269     }
270     switch (FormatTok.Tok.getObjCKeywordID()) {
271     case tok::objc_public:
272     case tok::objc_protected:
273     case tok::objc_package:
274     case tok::objc_private:
275       return parseAccessSpecifier();
276     case tok::objc_interface:
277     case tok::objc_implementation:
278       return parseObjCInterfaceOrImplementation();
279     case tok::objc_protocol:
280       return parseObjCProtocol();
281     case tok::objc_end:
282       return; // Handled by the caller.
283     case tok::objc_optional:
284     case tok::objc_required:
285       nextToken();
286       addUnwrappedLine();
287       return;
288     default:
289       break;
290     }
291     break;
292   case tok::kw_namespace:
293     parseNamespace();
294     return;
295   case tok::kw_inline:
296     nextToken();
297     TokenNumber++;
298     if (FormatTok.Tok.is(tok::kw_namespace)) {
299       parseNamespace();
300       return;
301     }
302     break;
303   case tok::kw_public:
304   case tok::kw_protected:
305   case tok::kw_private:
306     parseAccessSpecifier();
307     return;
308   case tok::kw_if:
309     parseIfThenElse();
310     return;
311   case tok::kw_for:
312   case tok::kw_while:
313     parseForOrWhileLoop();
314     return;
315   case tok::kw_do:
316     parseDoWhile();
317     return;
318   case tok::kw_switch:
319     parseSwitch();
320     return;
321   case tok::kw_default:
322     nextToken();
323     parseLabel();
324     return;
325   case tok::kw_case:
326     parseCaseLabel();
327     return;
328   case tok::kw_return:
329     parseReturn();
330     return;
331   case tok::kw_extern:
332     nextToken();
333     if (FormatTok.Tok.is(tok::string_literal)) {
334       nextToken();
335       if (FormatTok.Tok.is(tok::l_brace)) {
336         parseBlock(/*MustBeDeclaration=*/ true, 0);
337         addUnwrappedLine();
338         return;
339       }
340     }
341     // In all other cases, parse the declaration.
342     break;
343   default:
344     break;
345   }
346   do {
347     ++TokenNumber;
348     switch (FormatTok.Tok.getKind()) {
349     case tok::at:
350       nextToken();
351       if (FormatTok.Tok.is(tok::l_brace))
352         parseBracedList();
353       break;
354     case tok::kw_enum:
355       parseEnum();
356       break;
357     case tok::kw_struct:
358     case tok::kw_union:
359     case tok::kw_class:
360       parseRecord();
361       // A record declaration or definition is always the start of a structural
362       // element.
363       break;
364     case tok::semi:
365       nextToken();
366       addUnwrappedLine();
367       return;
368     case tok::r_brace:
369       addUnwrappedLine();
370       return;
371     case tok::l_paren:
372       parseParens();
373       break;
374     case tok::l_brace:
375       // A block outside of parentheses must be the last part of a
376       // structural element.
377       // FIXME: Figure out cases where this is not true, and add projections for
378       // them (the one we know is missing are lambdas).
379       parseBlock(/*MustBeDeclaration=*/ false);
380       addUnwrappedLine();
381       return;
382     case tok::identifier:
383       nextToken();
384       if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
385         parseLabel();
386         return;
387       }
388       break;
389     case tok::equal:
390       nextToken();
391       if (FormatTok.Tok.is(tok::l_brace)) {
392         parseBracedList();
393       }
394       break;
395     default:
396       nextToken();
397       break;
398     }
399   } while (!eof());
400 }
401 
402 void UnwrappedLineParser::parseBracedList() {
403   nextToken();
404 
405   do {
406     switch (FormatTok.Tok.getKind()) {
407     case tok::l_brace:
408       parseBracedList();
409       break;
410     case tok::r_brace:
411       nextToken();
412       return;
413     default:
414       nextToken();
415       break;
416     }
417   } while (!eof());
418 }
419 
420 void UnwrappedLineParser::parseReturn() {
421   nextToken();
422 
423   do {
424     switch (FormatTok.Tok.getKind()) {
425     case tok::l_brace:
426       parseBracedList();
427       break;
428     case tok::l_paren:
429       parseParens();
430       break;
431     case tok::r_brace:
432       // Assume missing ';'.
433       addUnwrappedLine();
434       return;
435     case tok::semi:
436       nextToken();
437       addUnwrappedLine();
438       return;
439     default:
440       nextToken();
441       break;
442     }
443   } while (!eof());
444 }
445 
446 void UnwrappedLineParser::parseParens() {
447   assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
448   nextToken();
449   do {
450     switch (FormatTok.Tok.getKind()) {
451     case tok::l_paren:
452       parseParens();
453       break;
454     case tok::r_paren:
455       nextToken();
456       return;
457     case tok::l_brace: {
458       nextToken();
459       ScopedLineState LineState(*this);
460       ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
461                                               /*MustBeDeclaration=*/ false);
462       Line->Level += 1;
463       parseLevel(/*HasOpeningBrace=*/ true);
464       Line->Level -= 1;
465       break;
466     }
467     case tok::at:
468       nextToken();
469       if (FormatTok.Tok.is(tok::l_brace))
470         parseBracedList();
471       break;
472     default:
473       nextToken();
474       break;
475     }
476   } while (!eof());
477 }
478 
479 void UnwrappedLineParser::parseIfThenElse() {
480   assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
481   nextToken();
482   if (FormatTok.Tok.is(tok::l_paren))
483     parseParens();
484   bool NeedsUnwrappedLine = false;
485   if (FormatTok.Tok.is(tok::l_brace)) {
486     parseBlock(/*MustBeDeclaration=*/ false);
487     NeedsUnwrappedLine = true;
488   } else {
489     addUnwrappedLine();
490     ++Line->Level;
491     parseStructuralElement();
492     --Line->Level;
493   }
494   if (FormatTok.Tok.is(tok::kw_else)) {
495     nextToken();
496     if (FormatTok.Tok.is(tok::l_brace)) {
497       parseBlock(/*MustBeDeclaration=*/ false);
498       addUnwrappedLine();
499     } else if (FormatTok.Tok.is(tok::kw_if)) {
500       parseIfThenElse();
501     } else {
502       addUnwrappedLine();
503       ++Line->Level;
504       parseStructuralElement();
505       --Line->Level;
506     }
507   } else if (NeedsUnwrappedLine) {
508     addUnwrappedLine();
509   }
510 }
511 
512 void UnwrappedLineParser::parseNamespace() {
513   assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
514   nextToken();
515   if (FormatTok.Tok.is(tok::identifier))
516     nextToken();
517   if (FormatTok.Tok.is(tok::l_brace)) {
518     parseBlock(/*MustBeDeclaration=*/ true, 0);
519     // Munch the semicolon after a namespace. This is more common than one would
520     // think. Puttin the semicolon into its own line is very ugly.
521     if (FormatTok.Tok.is(tok::semi))
522       nextToken();
523     addUnwrappedLine();
524   }
525   // FIXME: Add error handling.
526 }
527 
528 void UnwrappedLineParser::parseForOrWhileLoop() {
529   assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
530          "'for' or 'while' expected");
531   nextToken();
532   if (FormatTok.Tok.is(tok::l_paren))
533     parseParens();
534   if (FormatTok.Tok.is(tok::l_brace)) {
535     parseBlock(/*MustBeDeclaration=*/ false);
536     addUnwrappedLine();
537   } else {
538     addUnwrappedLine();
539     ++Line->Level;
540     parseStructuralElement();
541     --Line->Level;
542   }
543 }
544 
545 void UnwrappedLineParser::parseDoWhile() {
546   assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
547   nextToken();
548   if (FormatTok.Tok.is(tok::l_brace)) {
549     parseBlock(/*MustBeDeclaration=*/ false);
550   } else {
551     addUnwrappedLine();
552     ++Line->Level;
553     parseStructuralElement();
554     --Line->Level;
555   }
556 
557   // FIXME: Add error handling.
558   if (!FormatTok.Tok.is(tok::kw_while)) {
559     addUnwrappedLine();
560     return;
561   }
562 
563   nextToken();
564   parseStructuralElement();
565 }
566 
567 void UnwrappedLineParser::parseLabel() {
568   if (FormatTok.Tok.isNot(tok::colon))
569     return;
570   nextToken();
571   unsigned OldLineLevel = Line->Level;
572   if (Line->Level > 0)
573     --Line->Level;
574   if (FormatTok.Tok.is(tok::l_brace)) {
575     parseBlock(/*MustBeDeclaration=*/ false);
576     if (FormatTok.Tok.is(tok::kw_break))
577       parseStructuralElement(); // "break;" after "}" goes on the same line.
578   }
579   addUnwrappedLine();
580   Line->Level = OldLineLevel;
581 }
582 
583 void UnwrappedLineParser::parseCaseLabel() {
584   assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
585   // FIXME: fix handling of complex expressions here.
586   do {
587     nextToken();
588   } while (!eof() && !FormatTok.Tok.is(tok::colon));
589   parseLabel();
590 }
591 
592 void UnwrappedLineParser::parseSwitch() {
593   assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
594   nextToken();
595   if (FormatTok.Tok.is(tok::l_paren))
596     parseParens();
597   if (FormatTok.Tok.is(tok::l_brace)) {
598     parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
599     addUnwrappedLine();
600   } else {
601     addUnwrappedLine();
602     Line->Level += (Style.IndentCaseLabels ? 2 : 1);
603     parseStructuralElement();
604     Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
605   }
606 }
607 
608 void UnwrappedLineParser::parseAccessSpecifier() {
609   nextToken();
610   // Otherwise, we don't know what it is, and we'd better keep the next token.
611   if (FormatTok.Tok.is(tok::colon))
612     nextToken();
613   addUnwrappedLine();
614 }
615 
616 void UnwrappedLineParser::parseEnum() {
617   nextToken();
618   if (FormatTok.Tok.is(tok::identifier) ||
619       FormatTok.Tok.is(tok::kw___attribute) ||
620       FormatTok.Tok.is(tok::kw___declspec)) {
621     nextToken();
622     // We can have macros or attributes in between 'enum' and the enum name.
623     if (FormatTok.Tok.is(tok::l_paren)) {
624       parseParens();
625     }
626     if (FormatTok.Tok.is(tok::identifier))
627       nextToken();
628   }
629   if (FormatTok.Tok.is(tok::l_brace)) {
630     nextToken();
631     addUnwrappedLine();
632     ++Line->Level;
633     do {
634       switch (FormatTok.Tok.getKind()) {
635       case tok::l_paren:
636         parseParens();
637         break;
638       case tok::r_brace:
639         addUnwrappedLine();
640         nextToken();
641         --Line->Level;
642         return;
643       case tok::comma:
644         nextToken();
645         addUnwrappedLine();
646         break;
647       default:
648         nextToken();
649         break;
650       }
651     } while (!eof());
652   }
653   // We fall through to parsing a structural element afterwards, so that in
654   // enum A {} n, m;
655   // "} n, m;" will end up in one unwrapped line.
656 }
657 
658 void UnwrappedLineParser::parseRecord() {
659   nextToken();
660   if (FormatTok.Tok.is(tok::identifier) ||
661       FormatTok.Tok.is(tok::kw___attribute) ||
662       FormatTok.Tok.is(tok::kw___declspec)) {
663     nextToken();
664     // We can have macros or attributes in between 'class' and the class name.
665     if (FormatTok.Tok.is(tok::l_paren)) {
666       parseParens();
667     }
668     // The actual identifier can be a nested name specifier, and in macros
669     // it is often token-pasted.
670     while (FormatTok.Tok.is(tok::identifier) ||
671            FormatTok.Tok.is(tok::coloncolon) ||
672            FormatTok.Tok.is(tok::hashhash))
673       nextToken();
674 
675     // Note that parsing away template declarations here leads to incorrectly
676     // accepting function declarations as record declarations.
677     // In general, we cannot solve this problem. Consider:
678     // class A<int> B() {}
679     // which can be a function definition or a class definition when B() is a
680     // macro. If we find enough real-world cases where this is a problem, we
681     // can parse for the 'template' keyword in the beginning of the statement,
682     // and thus rule out the record production in case there is no template
683     // (this would still leave us with an ambiguity between template function
684     // and class declarations).
685     if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
686       while (FormatTok.Tok.isNot(tok::l_brace)) {
687         if (FormatTok.Tok.is(tok::semi))
688           return;
689         nextToken();
690       }
691     }
692   }
693   if (FormatTok.Tok.is(tok::l_brace))
694     parseBlock(/*MustBeDeclaration=*/ true);
695   // We fall through to parsing a structural element afterwards, so
696   // class A {} n, m;
697   // will end up in one unwrapped line.
698 }
699 
700 void UnwrappedLineParser::parseObjCProtocolList() {
701   assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
702   do
703     nextToken();
704   while (!eof() && FormatTok.Tok.isNot(tok::greater));
705   nextToken(); // Skip '>'.
706 }
707 
708 void UnwrappedLineParser::parseObjCUntilAtEnd() {
709   do {
710     if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
711       nextToken();
712       addUnwrappedLine();
713       break;
714     }
715     parseStructuralElement();
716   } while (!eof());
717 }
718 
719 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
720   nextToken();
721   nextToken();  // interface name
722 
723   // @interface can be followed by either a base class, or a category.
724   if (FormatTok.Tok.is(tok::colon)) {
725     nextToken();
726     nextToken();  // base class name
727   } else if (FormatTok.Tok.is(tok::l_paren))
728     // Skip category, if present.
729     parseParens();
730 
731   if (FormatTok.Tok.is(tok::less))
732     parseObjCProtocolList();
733 
734   // If instance variables are present, keep the '{' on the first line too.
735   if (FormatTok.Tok.is(tok::l_brace))
736     parseBlock(/*MustBeDeclaration=*/ true);
737 
738   // With instance variables, this puts '}' on its own line.  Without instance
739   // variables, this ends the @interface line.
740   addUnwrappedLine();
741 
742   parseObjCUntilAtEnd();
743 }
744 
745 void UnwrappedLineParser::parseObjCProtocol() {
746   nextToken();
747   nextToken();  // protocol name
748 
749   if (FormatTok.Tok.is(tok::less))
750     parseObjCProtocolList();
751 
752   // Check for protocol declaration.
753   if (FormatTok.Tok.is(tok::semi)) {
754     nextToken();
755     return addUnwrappedLine();
756   }
757 
758   addUnwrappedLine();
759   parseObjCUntilAtEnd();
760 }
761 
762 void UnwrappedLineParser::addUnwrappedLine() {
763   if (Line->Tokens.empty())
764     return;
765   DEBUG({
766     llvm::dbgs() << "Line(" << Line->Level << ")"
767                  << (Line->InPPDirective ? " MACRO" : "") << ": ";
768     for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
769                                           E = Line->Tokens.end();
770          I != E; ++I) {
771       llvm::dbgs() << I->Tok.getName() << " ";
772 
773     }
774     llvm::dbgs() << "\n";
775   });
776   CurrentLines->push_back(*Line);
777   Line->Tokens.clear();
778   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
779     for (std::vector<UnwrappedLine>::iterator I = PreprocessorDirectives
780              .begin(), E = PreprocessorDirectives.end();
781          I != E; ++I) {
782       CurrentLines->push_back(*I);
783     }
784     PreprocessorDirectives.clear();
785   }
786 
787 }
788 
789 bool UnwrappedLineParser::eof() const {
790   return FormatTok.Tok.is(tok::eof);
791 }
792 
793 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
794   bool JustComments = Line->Tokens.empty();
795   for (SmallVectorImpl<FormatToken>::const_iterator
796            I = CommentsBeforeNextToken.begin(),
797            E = CommentsBeforeNextToken.end();
798        I != E; ++I) {
799     if (I->NewlinesBefore && JustComments) {
800       addUnwrappedLine();
801     }
802     pushToken(*I);
803   }
804   if (NewlineBeforeNext && JustComments) {
805     addUnwrappedLine();
806   }
807   CommentsBeforeNextToken.clear();
808 }
809 
810 void UnwrappedLineParser::nextToken() {
811   if (eof())
812     return;
813   flushComments(FormatTok.NewlinesBefore > 0);
814   pushToken(FormatTok);
815   readToken();
816 }
817 
818 void UnwrappedLineParser::readToken() {
819   bool CommentsInCurrentLine = true;
820   do {
821     FormatTok = Tokens->getNextToken();
822     while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
823            ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
824             FormatTok.IsFirst)) {
825       // If there is an unfinished unwrapped line, we flush the preprocessor
826       // directives only after that unwrapped line was finished later.
827       bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
828                                        CurrentLines == &Lines;
829       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
830       parsePPDirective();
831     }
832     if (!FormatTok.Tok.is(tok::comment))
833       return;
834     if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
835       CommentsInCurrentLine = false;
836     }
837     if (CommentsInCurrentLine) {
838       pushToken(FormatTok);
839     } else {
840       CommentsBeforeNextToken.push_back(FormatTok);
841     }
842   } while (!eof());
843 }
844 
845 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
846   Line->Tokens.push_back(Tok);
847   if (MustBreakBeforeNextToken) {
848     Line->Tokens.back().MustBreakBefore = true;
849     MustBreakBeforeNextToken = false;
850   }
851 }
852 
853 } // end namespace format
854 } // end namespace clang
855