1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "llvm/Support/Debug.h"
21 
22 namespace clang {
23 namespace format {
24 
25 class ScopedDeclarationState {
26 public:
27   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28                          bool MustBeDeclaration)
29       : Line(Line), Stack(Stack) {
30     Line.MustBeDeclaration = MustBeDeclaration;
31     Stack.push_back(MustBeDeclaration);
32   }
33   ~ScopedDeclarationState() {
34     Stack.pop_back();
35     if (!Stack.empty())
36       Line.MustBeDeclaration = Stack.back();
37     else
38       Line.MustBeDeclaration = true;
39   }
40 private:
41   UnwrappedLine &Line;
42   std::vector<bool> &Stack;
43 };
44 
45 class ScopedMacroState : public FormatTokenSource {
46 public:
47   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48                    FormatToken &ResetToken)
49       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
51     TokenSource = this;
52     Line.Level = 0;
53     Line.InPPDirective = true;
54   }
55 
56   ~ScopedMacroState() {
57     TokenSource = PreviousTokenSource;
58     ResetToken = Token;
59     Line.InPPDirective = false;
60     Line.Level = PreviousLineLevel;
61   }
62 
63   virtual FormatToken getNextToken() {
64     // The \c UnwrappedLineParser guards against this by never calling
65     // \c getNextToken() after it has encountered the first eof token.
66     assert(!eof());
67     Token = PreviousTokenSource->getNextToken();
68     if (eof())
69       return createEOF();
70     return Token;
71   }
72 
73 private:
74   bool eof() {
75     return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
76   }
77 
78   FormatToken createEOF() {
79     FormatToken FormatTok;
80     FormatTok.Tok.startToken();
81     FormatTok.Tok.setKind(tok::eof);
82     return FormatTok;
83   }
84 
85   UnwrappedLine &Line;
86   FormatTokenSource *&TokenSource;
87   FormatToken &ResetToken;
88   unsigned PreviousLineLevel;
89   FormatTokenSource *PreviousTokenSource;
90 
91   FormatToken Token;
92 };
93 
94 class ScopedLineState {
95 public:
96   ScopedLineState(UnwrappedLineParser &Parser,
97                   bool SwitchToPreprocessorLines = false)
98       : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
99     if (SwitchToPreprocessorLines)
100       Parser.CurrentLines = &Parser.PreprocessorDirectives;
101     PreBlockLine = Parser.Line.take();
102     Parser.Line.reset(new UnwrappedLine());
103     Parser.Line->Level = PreBlockLine->Level;
104     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
105   }
106 
107   ~ScopedLineState() {
108     if (!Parser.Line->Tokens.empty()) {
109       Parser.addUnwrappedLine();
110     }
111     assert(Parser.Line->Tokens.empty());
112     Parser.Line.reset(PreBlockLine);
113     Parser.MustBreakBeforeNextToken = true;
114     if (SwitchToPreprocessorLines)
115       Parser.CurrentLines = &Parser.Lines;
116   }
117 
118 private:
119   UnwrappedLineParser &Parser;
120   const bool SwitchToPreprocessorLines;
121 
122   UnwrappedLine *PreBlockLine;
123 };
124 
125 UnwrappedLineParser::UnwrappedLineParser(
126     clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
127     FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
128     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
129       CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
130       Callback(Callback) {}
131 
132 bool UnwrappedLineParser::parse() {
133   DEBUG(llvm::dbgs() << "----\n");
134   readToken();
135   bool Error = parseFile();
136   for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
137                                             E = Lines.end();
138        I != E; ++I) {
139     Callback.consumeUnwrappedLine(*I);
140   }
141 
142   // Create line with eof token.
143   pushToken(FormatTok);
144   Callback.consumeUnwrappedLine(*Line);
145 
146   return Error;
147 }
148 
149 bool UnwrappedLineParser::parseFile() {
150   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
151                                           /*MustBeDeclaration=*/ true);
152   bool Error = parseLevel(/*HasOpeningBrace=*/false);
153   // Make sure to format the remaining tokens.
154   flushComments(true);
155   addUnwrappedLine();
156   return Error;
157 }
158 
159 bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
160   bool Error = false;
161   do {
162     switch (FormatTok.Tok.getKind()) {
163     case tok::comment:
164       nextToken();
165       addUnwrappedLine();
166       break;
167     case tok::l_brace:
168       // FIXME: Add parameter whether this can happen - if this happens, we must
169       // be in a non-declaration context.
170       Error |= parseBlock(/*MustBeDeclaration=*/ false);
171       addUnwrappedLine();
172       break;
173     case tok::r_brace:
174       if (HasOpeningBrace) {
175         return false;
176       } else {
177         Diag.Report(FormatTok.Tok.getLocation(),
178                     Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
179                                          "unexpected '}'"));
180         Error = true;
181         nextToken();
182         addUnwrappedLine();
183       }
184       break;
185     default:
186       parseStructuralElement();
187       break;
188     }
189   } while (!eof());
190   return Error;
191 }
192 
193 bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
194                                      unsigned AddLevels) {
195   assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
196   nextToken();
197 
198   addUnwrappedLine();
199 
200   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
201                                           MustBeDeclaration);
202   Line->Level += AddLevels;
203   parseLevel(/*HasOpeningBrace=*/true);
204 
205   if (!FormatTok.Tok.is(tok::r_brace)) {
206     Line->Level -= AddLevels;
207     return true;
208   }
209 
210   nextToken();  // Munch the closing brace.
211   Line->Level -= AddLevels;
212   return false;
213 }
214 
215 void UnwrappedLineParser::parsePPDirective() {
216   assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
217   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
218   nextToken();
219 
220   if (FormatTok.Tok.getIdentifierInfo() == NULL) {
221     parsePPUnknown();
222     return;
223   }
224 
225   switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
226   case tok::pp_define:
227     parsePPDefine();
228     break;
229   default:
230     parsePPUnknown();
231     break;
232   }
233 }
234 
235 void UnwrappedLineParser::parsePPDefine() {
236   nextToken();
237 
238   if (FormatTok.Tok.getKind() != tok::identifier) {
239     parsePPUnknown();
240     return;
241   }
242   nextToken();
243   if (FormatTok.Tok.getKind() == tok::l_paren &&
244       FormatTok.WhiteSpaceLength == 0) {
245     parseParens();
246   }
247   addUnwrappedLine();
248   Line->Level = 1;
249 
250   // Errors during a preprocessor directive can only affect the layout of the
251   // preprocessor directive, and thus we ignore them. An alternative approach
252   // would be to use the same approach we use on the file level (no
253   // re-indentation if there was a structural error) within the macro
254   // definition.
255   parseFile();
256 }
257 
258 void UnwrappedLineParser::parsePPUnknown() {
259   do {
260     nextToken();
261   } while (!eof());
262   addUnwrappedLine();
263 }
264 
265 void UnwrappedLineParser::parseStructuralElement() {
266   assert(!FormatTok.Tok.is(tok::l_brace));
267   int TokenNumber = 0;
268   switch (FormatTok.Tok.getKind()) {
269   case tok::at:
270     nextToken();
271     if (FormatTok.Tok.is(tok::l_brace)) {
272       parseBracedList();
273       break;
274     }
275     switch (FormatTok.Tok.getObjCKeywordID()) {
276     case tok::objc_public:
277     case tok::objc_protected:
278     case tok::objc_package:
279     case tok::objc_private:
280       return parseAccessSpecifier();
281     case tok::objc_interface:
282     case tok::objc_implementation:
283       return parseObjCInterfaceOrImplementation();
284     case tok::objc_protocol:
285       return parseObjCProtocol();
286     case tok::objc_end:
287       return; // Handled by the caller.
288     case tok::objc_optional:
289     case tok::objc_required:
290       nextToken();
291       addUnwrappedLine();
292       return;
293     default:
294       break;
295     }
296     break;
297   case tok::kw_namespace:
298     parseNamespace();
299     return;
300   case tok::kw_inline:
301     nextToken();
302     TokenNumber++;
303     if (FormatTok.Tok.is(tok::kw_namespace)) {
304       parseNamespace();
305       return;
306     }
307     break;
308   case tok::kw_public:
309   case tok::kw_protected:
310   case tok::kw_private:
311     parseAccessSpecifier();
312     return;
313   case tok::kw_if:
314     parseIfThenElse();
315     return;
316   case tok::kw_for:
317   case tok::kw_while:
318     parseForOrWhileLoop();
319     return;
320   case tok::kw_do:
321     parseDoWhile();
322     return;
323   case tok::kw_switch:
324     parseSwitch();
325     return;
326   case tok::kw_default:
327     nextToken();
328     parseLabel();
329     return;
330   case tok::kw_case:
331     parseCaseLabel();
332     return;
333   case tok::kw_return:
334     parseReturn();
335     return;
336   case tok::kw_extern:
337     nextToken();
338     if (FormatTok.Tok.is(tok::string_literal)) {
339       nextToken();
340       if (FormatTok.Tok.is(tok::l_brace)) {
341         parseBlock(/*MustBeDeclaration=*/ true, 0);
342         addUnwrappedLine();
343         return;
344       }
345     }
346     // In all other cases, parse the declaration.
347     break;
348   default:
349     break;
350   }
351   do {
352     ++TokenNumber;
353     switch (FormatTok.Tok.getKind()) {
354     case tok::at:
355       nextToken();
356       if (FormatTok.Tok.is(tok::l_brace))
357         parseBracedList();
358       break;
359     case tok::kw_enum:
360       parseEnum();
361       break;
362     case tok::kw_struct:
363     case tok::kw_union:
364     case tok::kw_class:
365       parseRecord();
366       // A record declaration or definition is always the start of a structural
367       // element.
368       break;
369     case tok::semi:
370       nextToken();
371       addUnwrappedLine();
372       return;
373     case tok::r_brace:
374       addUnwrappedLine();
375       return;
376     case tok::l_paren:
377       parseParens();
378       break;
379     case tok::l_brace:
380       // A block outside of parentheses must be the last part of a
381       // structural element.
382       // FIXME: Figure out cases where this is not true, and add projections for
383       // them (the one we know is missing are lambdas).
384       parseBlock(/*MustBeDeclaration=*/ false);
385       addUnwrappedLine();
386       return;
387     case tok::identifier:
388       nextToken();
389       if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
390         parseLabel();
391         return;
392       }
393       break;
394     case tok::equal:
395       nextToken();
396       if (FormatTok.Tok.is(tok::l_brace)) {
397         parseBracedList();
398       }
399       break;
400     default:
401       nextToken();
402       break;
403     }
404   } while (!eof());
405 }
406 
407 void UnwrappedLineParser::parseBracedList() {
408   nextToken();
409 
410   do {
411     switch (FormatTok.Tok.getKind()) {
412     case tok::l_brace:
413       parseBracedList();
414       break;
415     case tok::r_brace:
416       nextToken();
417       return;
418     default:
419       nextToken();
420       break;
421     }
422   } while (!eof());
423 }
424 
425 void UnwrappedLineParser::parseReturn() {
426   nextToken();
427 
428   do {
429     switch (FormatTok.Tok.getKind()) {
430     case tok::l_brace:
431       parseBracedList();
432       break;
433     case tok::l_paren:
434       parseParens();
435       break;
436     case tok::r_brace:
437       // Assume missing ';'.
438       addUnwrappedLine();
439       return;
440     case tok::semi:
441       nextToken();
442       addUnwrappedLine();
443       return;
444     default:
445       nextToken();
446       break;
447     }
448   } while (!eof());
449 }
450 
451 void UnwrappedLineParser::parseParens() {
452   assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
453   nextToken();
454   do {
455     switch (FormatTok.Tok.getKind()) {
456     case tok::l_paren:
457       parseParens();
458       break;
459     case tok::r_paren:
460       nextToken();
461       return;
462     case tok::l_brace: {
463       nextToken();
464       ScopedLineState LineState(*this);
465       ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
466                                               /*MustBeDeclaration=*/ false);
467       Line->Level += 1;
468       parseLevel(/*HasOpeningBrace=*/ true);
469       Line->Level -= 1;
470       break;
471     }
472     case tok::at:
473       nextToken();
474       if (FormatTok.Tok.is(tok::l_brace))
475         parseBracedList();
476       break;
477     default:
478       nextToken();
479       break;
480     }
481   } while (!eof());
482 }
483 
484 void UnwrappedLineParser::parseIfThenElse() {
485   assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
486   nextToken();
487   if (FormatTok.Tok.is(tok::l_paren))
488     parseParens();
489   bool NeedsUnwrappedLine = false;
490   if (FormatTok.Tok.is(tok::l_brace)) {
491     parseBlock(/*MustBeDeclaration=*/ false);
492     NeedsUnwrappedLine = true;
493   } else {
494     addUnwrappedLine();
495     ++Line->Level;
496     parseStructuralElement();
497     --Line->Level;
498   }
499   if (FormatTok.Tok.is(tok::kw_else)) {
500     nextToken();
501     if (FormatTok.Tok.is(tok::l_brace)) {
502       parseBlock(/*MustBeDeclaration=*/ false);
503       addUnwrappedLine();
504     } else if (FormatTok.Tok.is(tok::kw_if)) {
505       parseIfThenElse();
506     } else {
507       addUnwrappedLine();
508       ++Line->Level;
509       parseStructuralElement();
510       --Line->Level;
511     }
512   } else if (NeedsUnwrappedLine) {
513     addUnwrappedLine();
514   }
515 }
516 
517 void UnwrappedLineParser::parseNamespace() {
518   assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
519   nextToken();
520   if (FormatTok.Tok.is(tok::identifier))
521     nextToken();
522   if (FormatTok.Tok.is(tok::l_brace)) {
523     parseBlock(/*MustBeDeclaration=*/ true, 0);
524     // Munch the semicolon after a namespace. This is more common than one would
525     // think. Puttin the semicolon into its own line is very ugly.
526     if (FormatTok.Tok.is(tok::semi))
527       nextToken();
528     addUnwrappedLine();
529   }
530   // FIXME: Add error handling.
531 }
532 
533 void UnwrappedLineParser::parseForOrWhileLoop() {
534   assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
535          "'for' or 'while' expected");
536   nextToken();
537   if (FormatTok.Tok.is(tok::l_paren))
538     parseParens();
539   if (FormatTok.Tok.is(tok::l_brace)) {
540     parseBlock(/*MustBeDeclaration=*/ false);
541     addUnwrappedLine();
542   } else {
543     addUnwrappedLine();
544     ++Line->Level;
545     parseStructuralElement();
546     --Line->Level;
547   }
548 }
549 
550 void UnwrappedLineParser::parseDoWhile() {
551   assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
552   nextToken();
553   if (FormatTok.Tok.is(tok::l_brace)) {
554     parseBlock(/*MustBeDeclaration=*/ false);
555   } else {
556     addUnwrappedLine();
557     ++Line->Level;
558     parseStructuralElement();
559     --Line->Level;
560   }
561 
562   // FIXME: Add error handling.
563   if (!FormatTok.Tok.is(tok::kw_while)) {
564     addUnwrappedLine();
565     return;
566   }
567 
568   nextToken();
569   parseStructuralElement();
570 }
571 
572 void UnwrappedLineParser::parseLabel() {
573   if (FormatTok.Tok.isNot(tok::colon))
574     return;
575   nextToken();
576   unsigned OldLineLevel = Line->Level;
577   if (Line->Level > 0)
578     --Line->Level;
579   if (FormatTok.Tok.is(tok::l_brace)) {
580     parseBlock(/*MustBeDeclaration=*/ false);
581     if (FormatTok.Tok.is(tok::kw_break))
582       parseStructuralElement(); // "break;" after "}" goes on the same line.
583   }
584   addUnwrappedLine();
585   Line->Level = OldLineLevel;
586 }
587 
588 void UnwrappedLineParser::parseCaseLabel() {
589   assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
590   // FIXME: fix handling of complex expressions here.
591   do {
592     nextToken();
593   } while (!eof() && !FormatTok.Tok.is(tok::colon));
594   parseLabel();
595 }
596 
597 void UnwrappedLineParser::parseSwitch() {
598   assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
599   nextToken();
600   if (FormatTok.Tok.is(tok::l_paren))
601     parseParens();
602   if (FormatTok.Tok.is(tok::l_brace)) {
603     parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
604     addUnwrappedLine();
605   } else {
606     addUnwrappedLine();
607     Line->Level += (Style.IndentCaseLabels ? 2 : 1);
608     parseStructuralElement();
609     Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
610   }
611 }
612 
613 void UnwrappedLineParser::parseAccessSpecifier() {
614   nextToken();
615   // Otherwise, we don't know what it is, and we'd better keep the next token.
616   if (FormatTok.Tok.is(tok::colon))
617     nextToken();
618   addUnwrappedLine();
619 }
620 
621 void UnwrappedLineParser::parseEnum() {
622   nextToken();
623   if (FormatTok.Tok.is(tok::identifier) ||
624       FormatTok.Tok.is(tok::kw___attribute) ||
625       FormatTok.Tok.is(tok::kw___declspec)) {
626     nextToken();
627     // We can have macros or attributes in between 'enum' and the enum name.
628     if (FormatTok.Tok.is(tok::l_paren)) {
629       parseParens();
630     }
631     if (FormatTok.Tok.is(tok::identifier))
632       nextToken();
633   }
634   if (FormatTok.Tok.is(tok::l_brace)) {
635     nextToken();
636     addUnwrappedLine();
637     ++Line->Level;
638     do {
639       switch (FormatTok.Tok.getKind()) {
640       case tok::l_paren:
641         parseParens();
642         break;
643       case tok::r_brace:
644         addUnwrappedLine();
645         nextToken();
646         --Line->Level;
647         return;
648       case tok::comma:
649         nextToken();
650         addUnwrappedLine();
651         break;
652       default:
653         nextToken();
654         break;
655       }
656     } while (!eof());
657   }
658   // We fall through to parsing a structural element afterwards, so that in
659   // enum A {} n, m;
660   // "} n, m;" will end up in one unwrapped line.
661 }
662 
663 void UnwrappedLineParser::parseRecord() {
664   nextToken();
665   if (FormatTok.Tok.is(tok::identifier) ||
666       FormatTok.Tok.is(tok::kw___attribute) ||
667       FormatTok.Tok.is(tok::kw___declspec)) {
668     nextToken();
669     // We can have macros or attributes in between 'class' and the class name.
670     if (FormatTok.Tok.is(tok::l_paren)) {
671       parseParens();
672     }
673     // The actual identifier can be a nested name specifier, and in macros
674     // it is often token-pasted.
675     while (FormatTok.Tok.is(tok::identifier) ||
676            FormatTok.Tok.is(tok::coloncolon) ||
677            FormatTok.Tok.is(tok::hashhash))
678       nextToken();
679 
680     // Note that parsing away template declarations here leads to incorrectly
681     // accepting function declarations as record declarations.
682     // In general, we cannot solve this problem. Consider:
683     // class A<int> B() {}
684     // which can be a function definition or a class definition when B() is a
685     // macro. If we find enough real-world cases where this is a problem, we
686     // can parse for the 'template' keyword in the beginning of the statement,
687     // and thus rule out the record production in case there is no template
688     // (this would still leave us with an ambiguity between template function
689     // and class declarations).
690     if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
691       while (FormatTok.Tok.isNot(tok::l_brace)) {
692         if (FormatTok.Tok.is(tok::semi))
693           return;
694         nextToken();
695       }
696     }
697   }
698   if (FormatTok.Tok.is(tok::l_brace))
699     parseBlock(/*MustBeDeclaration=*/ true);
700   // We fall through to parsing a structural element afterwards, so
701   // class A {} n, m;
702   // will end up in one unwrapped line.
703 }
704 
705 void UnwrappedLineParser::parseObjCProtocolList() {
706   assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
707   do
708     nextToken();
709   while (!eof() && FormatTok.Tok.isNot(tok::greater));
710   nextToken(); // Skip '>'.
711 }
712 
713 void UnwrappedLineParser::parseObjCUntilAtEnd() {
714   do {
715     if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
716       nextToken();
717       addUnwrappedLine();
718       break;
719     }
720     parseStructuralElement();
721   } while (!eof());
722 }
723 
724 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
725   nextToken();
726   nextToken();  // interface name
727 
728   // @interface can be followed by either a base class, or a category.
729   if (FormatTok.Tok.is(tok::colon)) {
730     nextToken();
731     nextToken();  // base class name
732   } else if (FormatTok.Tok.is(tok::l_paren))
733     // Skip category, if present.
734     parseParens();
735 
736   if (FormatTok.Tok.is(tok::less))
737     parseObjCProtocolList();
738 
739   // If instance variables are present, keep the '{' on the first line too.
740   if (FormatTok.Tok.is(tok::l_brace))
741     parseBlock(/*MustBeDeclaration=*/ true);
742 
743   // With instance variables, this puts '}' on its own line.  Without instance
744   // variables, this ends the @interface line.
745   addUnwrappedLine();
746 
747   parseObjCUntilAtEnd();
748 }
749 
750 void UnwrappedLineParser::parseObjCProtocol() {
751   nextToken();
752   nextToken();  // protocol name
753 
754   if (FormatTok.Tok.is(tok::less))
755     parseObjCProtocolList();
756 
757   // Check for protocol declaration.
758   if (FormatTok.Tok.is(tok::semi)) {
759     nextToken();
760     return addUnwrappedLine();
761   }
762 
763   addUnwrappedLine();
764   parseObjCUntilAtEnd();
765 }
766 
767 void UnwrappedLineParser::addUnwrappedLine() {
768   if (Line->Tokens.empty())
769     return;
770   DEBUG({
771     llvm::dbgs() << "Line(" << Line->Level << ")"
772                  << (Line->InPPDirective ? " MACRO" : "") << ": ";
773     for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
774                                           E = Line->Tokens.end();
775          I != E; ++I) {
776       llvm::dbgs() << I->Tok.getName() << " ";
777 
778     }
779     llvm::dbgs() << "\n";
780   });
781   CurrentLines->push_back(*Line);
782   Line->Tokens.clear();
783   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
784     for (std::vector<UnwrappedLine>::iterator
785              I = PreprocessorDirectives.begin(),
786              E = PreprocessorDirectives.end();
787          I != E; ++I) {
788       CurrentLines->push_back(*I);
789     }
790     PreprocessorDirectives.clear();
791   }
792 }
793 
794 bool UnwrappedLineParser::eof() const {
795   return FormatTok.Tok.is(tok::eof);
796 }
797 
798 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
799   bool JustComments = Line->Tokens.empty();
800   for (SmallVectorImpl<FormatToken>::const_iterator
801            I = CommentsBeforeNextToken.begin(),
802            E = CommentsBeforeNextToken.end();
803        I != E; ++I) {
804     if (I->NewlinesBefore && JustComments) {
805       addUnwrappedLine();
806     }
807     pushToken(*I);
808   }
809   if (NewlineBeforeNext && JustComments) {
810     addUnwrappedLine();
811   }
812   CommentsBeforeNextToken.clear();
813 }
814 
815 void UnwrappedLineParser::nextToken() {
816   if (eof())
817     return;
818   flushComments(FormatTok.NewlinesBefore > 0);
819   pushToken(FormatTok);
820   readToken();
821 }
822 
823 void UnwrappedLineParser::readToken() {
824   bool CommentsInCurrentLine = true;
825   do {
826     FormatTok = Tokens->getNextToken();
827     while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
828            ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
829             FormatTok.IsFirst)) {
830       // If there is an unfinished unwrapped line, we flush the preprocessor
831       // directives only after that unwrapped line was finished later.
832       bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
833                                        CurrentLines == &Lines;
834       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
835       parsePPDirective();
836     }
837     if (!FormatTok.Tok.is(tok::comment))
838       return;
839     if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
840       CommentsInCurrentLine = false;
841     }
842     if (CommentsInCurrentLine) {
843       pushToken(FormatTok);
844     } else {
845       CommentsBeforeNextToken.push_back(FormatTok);
846     }
847   } while (!eof());
848 }
849 
850 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
851   Line->Tokens.push_back(Tok);
852   if (MustBreakBeforeNextToken) {
853     Line->Tokens.back().MustBreakBefore = true;
854     MustBreakBeforeNextToken = false;
855   }
856 }
857 
858 } // end namespace format
859 } // end namespace clang
860