1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 /// This is EXPERIMENTAL code under heavy development. It is not in a state yet,
15 /// where it can be used to format real code.
16 ///
17 //===----------------------------------------------------------------------===//
18 
19 #define DEBUG_TYPE "format-parser"
20 
21 #include "UnwrappedLineParser.h"
22 #include "clang/Basic/Diagnostic.h"
23 #include "llvm/Support/Debug.h"
24 
25 // Uncomment to get debug output from tests:
26 // #define DEBUG_WITH_TYPE(T, X) do { X; } while(0)
27 
28 namespace clang {
29 namespace format {
30 
31 class ScopedDeclarationState {
32 public:
33   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
34                          bool MustBeDeclaration)
35       : Line(Line), Stack(Stack) {
36     Line.MustBeDeclaration = MustBeDeclaration;
37     Stack.push_back(MustBeDeclaration);
38   }
39   ~ScopedDeclarationState() {
40     Stack.pop_back();
41     if (!Stack.empty())
42       Line.MustBeDeclaration = Stack.back();
43     else
44       Line.MustBeDeclaration = true;
45   }
46 private:
47   UnwrappedLine &Line;
48   std::vector<bool> &Stack;
49 };
50 
51 class ScopedMacroState : public FormatTokenSource {
52 public:
53   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
54                    FormatToken &ResetToken)
55       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
56         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
57     TokenSource = this;
58     Line.Level = 0;
59     Line.InPPDirective = true;
60   }
61 
62   ~ScopedMacroState() {
63     TokenSource = PreviousTokenSource;
64     ResetToken = Token;
65     Line.InPPDirective = false;
66     Line.Level = PreviousLineLevel;
67   }
68 
69   virtual FormatToken getNextToken() {
70     // The \c UnwrappedLineParser guards against this by never calling
71     // \c getNextToken() after it has encountered the first eof token.
72     assert(!eof());
73     Token = PreviousTokenSource->getNextToken();
74     if (eof())
75       return createEOF();
76     return Token;
77   }
78 
79 private:
80   bool eof() {
81     return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
82   }
83 
84   FormatToken createEOF() {
85     FormatToken FormatTok;
86     FormatTok.Tok.startToken();
87     FormatTok.Tok.setKind(tok::eof);
88     return FormatTok;
89   }
90 
91   UnwrappedLine &Line;
92   FormatTokenSource *&TokenSource;
93   FormatToken &ResetToken;
94   unsigned PreviousLineLevel;
95   FormatTokenSource *PreviousTokenSource;
96 
97   FormatToken Token;
98 };
99 
100 class ScopedLineState {
101 public:
102   ScopedLineState(UnwrappedLineParser &Parser,
103                   bool SwitchToPreprocessorLines = false)
104       : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
105     if (SwitchToPreprocessorLines)
106       Parser.CurrentLines = &Parser.PreprocessorDirectives;
107     PreBlockLine = Parser.Line.take();
108     Parser.Line.reset(new UnwrappedLine());
109     Parser.Line->Level = PreBlockLine->Level;
110     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
111   }
112 
113   ~ScopedLineState() {
114     if (!Parser.Line->Tokens.empty()) {
115       Parser.addUnwrappedLine();
116     }
117     assert(Parser.Line->Tokens.empty());
118     Parser.Line.reset(PreBlockLine);
119     Parser.MustBreakBeforeNextToken = true;
120     if (SwitchToPreprocessorLines)
121       Parser.CurrentLines = &Parser.Lines;
122   }
123 
124 private:
125   UnwrappedLineParser &Parser;
126   const bool SwitchToPreprocessorLines;
127 
128   UnwrappedLine *PreBlockLine;
129 };
130 
131 UnwrappedLineParser::UnwrappedLineParser(
132     clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
133     FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
134     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
135       CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
136       Callback(Callback) {}
137 
138 bool UnwrappedLineParser::parse() {
139   DEBUG(llvm::dbgs() << "----\n");
140   readToken();
141   bool Error = parseFile();
142   for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
143                                             E = Lines.end();
144        I != E; ++I) {
145     Callback.consumeUnwrappedLine(*I);
146   }
147   return Error;
148 }
149 
150 bool UnwrappedLineParser::parseFile() {
151   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
152                                           /*MustBeDeclaration=*/ true);
153   bool Error = parseLevel(/*HasOpeningBrace=*/false);
154   // Make sure to format the remaining tokens.
155   flushComments(true);
156   addUnwrappedLine();
157   return Error;
158 }
159 
160 bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
161   bool Error = false;
162   do {
163     switch (FormatTok.Tok.getKind()) {
164     case tok::comment:
165       nextToken();
166       addUnwrappedLine();
167       break;
168     case tok::l_brace:
169       // FIXME: Add parameter whether this can happen - if this happens, we must
170       // be in a non-declaration context.
171       Error |= parseBlock(/*MustBeDeclaration=*/ false);
172       addUnwrappedLine();
173       break;
174     case tok::r_brace:
175       if (HasOpeningBrace) {
176         return false;
177       } else {
178         Diag.Report(FormatTok.Tok.getLocation(),
179                     Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
180                                          "unexpected '}'"));
181         Error = true;
182         nextToken();
183         addUnwrappedLine();
184       }
185       break;
186     default:
187       parseStructuralElement();
188       break;
189     }
190   } while (!eof());
191   return Error;
192 }
193 
194 bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration, unsigned AddLevels) {
195   assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
196   nextToken();
197 
198   addUnwrappedLine();
199 
200   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
201                                           MustBeDeclaration);
202   Line->Level += AddLevels;
203   parseLevel(/*HasOpeningBrace=*/true);
204 
205   if (!FormatTok.Tok.is(tok::r_brace)) {
206     Line->Level -= AddLevels;
207     return true;
208   }
209 
210   nextToken();  // Munch the closing brace.
211   Line->Level -= AddLevels;
212   return false;
213 }
214 
215 void UnwrappedLineParser::parsePPDirective() {
216   assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
217   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
218   nextToken();
219 
220   if (FormatTok.Tok.getIdentifierInfo() == NULL) {
221     addUnwrappedLine();
222     return;
223   }
224 
225   switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
226   case tok::pp_define:
227     parsePPDefine();
228     break;
229   default:
230     parsePPUnknown();
231     break;
232   }
233 }
234 
235 void UnwrappedLineParser::parsePPDefine() {
236   nextToken();
237 
238   if (FormatTok.Tok.getKind() != tok::identifier) {
239     parsePPUnknown();
240     return;
241   }
242   nextToken();
243   if (FormatTok.Tok.getKind() == tok::l_paren &&
244       FormatTok.WhiteSpaceLength == 0) {
245     parseParens();
246   }
247   addUnwrappedLine();
248   Line->Level = 1;
249 
250   // Errors during a preprocessor directive can only affect the layout of the
251   // preprocessor directive, and thus we ignore them. An alternative approach
252   // would be to use the same approach we use on the file level (no
253   // re-indentation if there was a structural error) within the macro
254   // definition.
255   parseFile();
256 }
257 
258 void UnwrappedLineParser::parsePPUnknown() {
259   do {
260     nextToken();
261   } while (!eof());
262   addUnwrappedLine();
263 }
264 
265 void UnwrappedLineParser::parseStructuralElement() {
266   assert(!FormatTok.Tok.is(tok::l_brace));
267   int TokenNumber = 0;
268   switch (FormatTok.Tok.getKind()) {
269   case tok::at:
270     nextToken();
271     switch (FormatTok.Tok.getObjCKeywordID()) {
272     case tok::objc_public:
273     case tok::objc_protected:
274     case tok::objc_package:
275     case tok::objc_private:
276       return parseAccessSpecifier();
277     case tok::objc_interface:
278     case tok::objc_implementation:
279       return parseObjCInterfaceOrImplementation();
280     case tok::objc_protocol:
281       return parseObjCProtocol();
282     case tok::objc_end:
283       return; // Handled by the caller.
284     case tok::objc_optional:
285     case tok::objc_required:
286       nextToken();
287       addUnwrappedLine();
288       return;
289     default:
290       break;
291     }
292     break;
293   case tok::kw_namespace:
294     parseNamespace();
295     return;
296   case tok::kw_inline:
297     nextToken();
298     TokenNumber++;
299     if (FormatTok.Tok.is(tok::kw_namespace)) {
300       parseNamespace();
301       return;
302     }
303     break;
304   case tok::kw_public:
305   case tok::kw_protected:
306   case tok::kw_private:
307     parseAccessSpecifier();
308     return;
309   case tok::kw_if:
310     parseIfThenElse();
311     return;
312   case tok::kw_for:
313   case tok::kw_while:
314     parseForOrWhileLoop();
315     return;
316   case tok::kw_do:
317     parseDoWhile();
318     return;
319   case tok::kw_switch:
320     parseSwitch();
321     return;
322   case tok::kw_default:
323     nextToken();
324     parseLabel();
325     return;
326   case tok::kw_case:
327     parseCaseLabel();
328     return;
329   case tok::kw_return:
330     parseReturn();
331     return;
332   case tok::kw_extern:
333     nextToken();
334     if (FormatTok.Tok.is(tok::string_literal)) {
335       nextToken();
336       if (FormatTok.Tok.is(tok::l_brace)) {
337         parseBlock(/*MustBeDeclaration=*/ true, 0);
338         addUnwrappedLine();
339         return;
340       }
341     }
342     // In all other cases, parse the declaration.
343     break;
344   default:
345     break;
346   }
347   do {
348     ++TokenNumber;
349     switch (FormatTok.Tok.getKind()) {
350     case tok::kw_enum:
351       parseEnum();
352       break;
353     case tok::kw_struct:
354     case tok::kw_union:
355     case tok::kw_class:
356       parseRecord();
357       // A record declaration or definition is always the start of a structural
358       // element.
359       break;
360     case tok::semi:
361       nextToken();
362       addUnwrappedLine();
363       return;
364     case tok::r_brace:
365       addUnwrappedLine();
366       return;
367     case tok::l_paren:
368       parseParens();
369       break;
370     case tok::l_brace:
371       // A block outside of parentheses must be the last part of a
372       // structural element.
373       // FIXME: Figure out cases where this is not true, and add projections for
374       // them (the one we know is missing are lambdas).
375       parseBlock(/*MustBeDeclaration=*/ false);
376       addUnwrappedLine();
377       return;
378     case tok::identifier:
379       nextToken();
380       if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
381         parseLabel();
382         return;
383       }
384       break;
385     case tok::equal:
386       nextToken();
387       if (FormatTok.Tok.is(tok::l_brace)) {
388         parseBracedList();
389       }
390       break;
391     default:
392       nextToken();
393       break;
394     }
395   } while (!eof());
396 }
397 
398 void UnwrappedLineParser::parseBracedList() {
399   nextToken();
400 
401   do {
402     switch (FormatTok.Tok.getKind()) {
403     case tok::l_brace:
404       parseBracedList();
405       break;
406     case tok::r_brace:
407       nextToken();
408       return;
409     default:
410       nextToken();
411       break;
412     }
413   } while (!eof());
414 }
415 
416 void UnwrappedLineParser::parseReturn() {
417   nextToken();
418 
419   do {
420     switch (FormatTok.Tok.getKind()) {
421     case tok::l_brace:
422       parseBracedList();
423       break;
424     case tok::l_paren:
425       parseParens();
426       break;
427     case tok::r_brace:
428       // Assume missing ';'.
429       addUnwrappedLine();
430       return;
431     case tok::semi:
432       nextToken();
433       addUnwrappedLine();
434       return;
435     default:
436       nextToken();
437       break;
438     }
439   } while (!eof());
440 }
441 
442 void UnwrappedLineParser::parseParens() {
443   assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
444   nextToken();
445   do {
446     switch (FormatTok.Tok.getKind()) {
447     case tok::l_paren:
448       parseParens();
449       break;
450     case tok::r_paren:
451       nextToken();
452       return;
453     case tok::l_brace:
454       {
455         nextToken();
456         ScopedLineState LineState(*this);
457         ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
458                                                 /*MustBeDeclaration=*/ false);
459         Line->Level += 1;
460         parseLevel(/*HasOpeningBrace=*/ true);
461         Line->Level -= 1;
462       }
463       break;
464     default:
465       nextToken();
466       break;
467     }
468   } while (!eof());
469 }
470 
471 void UnwrappedLineParser::parseIfThenElse() {
472   assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
473   nextToken();
474   if (FormatTok.Tok.is(tok::l_paren))
475     parseParens();
476   bool NeedsUnwrappedLine = false;
477   if (FormatTok.Tok.is(tok::l_brace)) {
478     parseBlock(/*MustBeDeclaration=*/ false);
479     NeedsUnwrappedLine = true;
480   } else {
481     addUnwrappedLine();
482     ++Line->Level;
483     parseStructuralElement();
484     --Line->Level;
485   }
486   if (FormatTok.Tok.is(tok::kw_else)) {
487     nextToken();
488     if (FormatTok.Tok.is(tok::l_brace)) {
489       parseBlock(/*MustBeDeclaration=*/ false);
490       addUnwrappedLine();
491     } else if (FormatTok.Tok.is(tok::kw_if)) {
492       parseIfThenElse();
493     } else {
494       addUnwrappedLine();
495       ++Line->Level;
496       parseStructuralElement();
497       --Line->Level;
498     }
499   } else if (NeedsUnwrappedLine) {
500     addUnwrappedLine();
501   }
502 }
503 
504 void UnwrappedLineParser::parseNamespace() {
505   assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
506   nextToken();
507   if (FormatTok.Tok.is(tok::identifier))
508     nextToken();
509   if (FormatTok.Tok.is(tok::l_brace)) {
510     parseBlock(/*MustBeDeclaration=*/ true, 0);
511     addUnwrappedLine();
512   }
513   // FIXME: Add error handling.
514 }
515 
516 void UnwrappedLineParser::parseForOrWhileLoop() {
517   assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
518          "'for' or 'while' expected");
519   nextToken();
520   if (FormatTok.Tok.is(tok::l_paren))
521     parseParens();
522   if (FormatTok.Tok.is(tok::l_brace)) {
523     parseBlock(/*MustBeDeclaration=*/ false);
524     addUnwrappedLine();
525   } else {
526     addUnwrappedLine();
527     ++Line->Level;
528     parseStructuralElement();
529     --Line->Level;
530   }
531 }
532 
533 void UnwrappedLineParser::parseDoWhile() {
534   assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
535   nextToken();
536   if (FormatTok.Tok.is(tok::l_brace)) {
537     parseBlock(/*MustBeDeclaration=*/ false);
538   } else {
539     addUnwrappedLine();
540     ++Line->Level;
541     parseStructuralElement();
542     --Line->Level;
543   }
544 
545   // FIXME: Add error handling.
546   if (!FormatTok.Tok.is(tok::kw_while)) {
547     addUnwrappedLine();
548     return;
549   }
550 
551   nextToken();
552   parseStructuralElement();
553 }
554 
555 void UnwrappedLineParser::parseLabel() {
556   // FIXME: remove all asserts.
557   assert(FormatTok.Tok.is(tok::colon) && "':' expected");
558   nextToken();
559   unsigned OldLineLevel = Line->Level;
560   if (Line->Level > 0)
561     --Line->Level;
562   if (FormatTok.Tok.is(tok::l_brace)) {
563     parseBlock(/*MustBeDeclaration=*/ false);
564     if (FormatTok.Tok.is(tok::kw_break))
565       parseStructuralElement(); // "break;" after "}" goes on the same line.
566   }
567   addUnwrappedLine();
568   Line->Level = OldLineLevel;
569 }
570 
571 void UnwrappedLineParser::parseCaseLabel() {
572   assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
573   // FIXME: fix handling of complex expressions here.
574   do {
575     nextToken();
576   } while (!eof() && !FormatTok.Tok.is(tok::colon));
577   parseLabel();
578 }
579 
580 void UnwrappedLineParser::parseSwitch() {
581   assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
582   nextToken();
583   if (FormatTok.Tok.is(tok::l_paren))
584     parseParens();
585   if (FormatTok.Tok.is(tok::l_brace)) {
586     parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
587     addUnwrappedLine();
588   } else {
589     addUnwrappedLine();
590     Line->Level += (Style.IndentCaseLabels ? 2 : 1);
591     parseStructuralElement();
592     Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
593   }
594 }
595 
596 void UnwrappedLineParser::parseAccessSpecifier() {
597   nextToken();
598   // Otherwise, we don't know what it is, and we'd better keep the next token.
599   if (FormatTok.Tok.is(tok::colon))
600     nextToken();
601   addUnwrappedLine();
602 }
603 
604 void UnwrappedLineParser::parseEnum() {
605   nextToken();
606   if (FormatTok.Tok.is(tok::identifier) ||
607       FormatTok.Tok.is(tok::kw___attribute) ||
608       FormatTok.Tok.is(tok::kw___declspec)) {
609     nextToken();
610     // We can have macros or attributes in between 'enum' and the enum name.
611     if (FormatTok.Tok.is(tok::l_paren)) {
612       parseParens();
613     }
614     if (FormatTok.Tok.is(tok::identifier))
615       nextToken();
616   }
617   if (FormatTok.Tok.is(tok::l_brace)) {
618     nextToken();
619     addUnwrappedLine();
620     ++Line->Level;
621     do {
622       switch (FormatTok.Tok.getKind()) {
623       case tok::l_paren:
624         parseParens();
625         break;
626       case tok::r_brace:
627         addUnwrappedLine();
628         nextToken();
629         --Line->Level;
630         return;
631       case tok::comma:
632         nextToken();
633         addUnwrappedLine();
634         break;
635       default:
636         nextToken();
637         break;
638       }
639     } while (!eof());
640   }
641   // We fall through to parsing a structural element afterwards, so that in
642   // enum A {} n, m;
643   // "} n, m;" will end up in one unwrapped line.
644 }
645 
646 void UnwrappedLineParser::parseRecord() {
647   nextToken();
648   if (FormatTok.Tok.is(tok::identifier) ||
649       FormatTok.Tok.is(tok::kw___attribute) ||
650       FormatTok.Tok.is(tok::kw___declspec)) {
651     nextToken();
652     // We can have macros or attributes in between 'class' and the class name.
653     if (FormatTok.Tok.is(tok::l_paren)) {
654       parseParens();
655     }
656     // The actual identifier can be a nested name specifier.
657     while (FormatTok.Tok.is(tok::identifier) ||
658            FormatTok.Tok.is(tok::coloncolon))
659       nextToken();
660 
661     // Note that parsing away template declarations here leads to incorrectly
662     // accepting function declarations as record declarations.
663     // In general, we cannot solve this problem. Consider:
664     // class A<int> B() {}
665     // which can be a function definition or a class definition when B() is a
666     // macro. If we find enough real-world cases where this is a problem, we
667     // can parse for the 'template' keyword in the beginning of the statement,
668     // and thus rule out the record production in case there is no template
669     // (this would still leave us with an ambiguity between template function
670     // and class declarations).
671     if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
672       while (FormatTok.Tok.isNot(tok::l_brace)) {
673         if (FormatTok.Tok.is(tok::semi))
674           return;
675         nextToken();
676       }
677     }
678   }
679   if (FormatTok.Tok.is(tok::l_brace))
680     parseBlock(/*MustBeDeclaration=*/ true);
681   // We fall through to parsing a structural element afterwards, so
682   // class A {} n, m;
683   // will end up in one unwrapped line.
684 }
685 
686 void UnwrappedLineParser::parseObjCProtocolList() {
687   assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
688   do
689     nextToken();
690   while (!eof() && FormatTok.Tok.isNot(tok::greater));
691   nextToken(); // Skip '>'.
692 }
693 
694 void UnwrappedLineParser::parseObjCUntilAtEnd() {
695   do {
696     if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
697       nextToken();
698       addUnwrappedLine();
699       break;
700     }
701     parseStructuralElement();
702   } while (!eof());
703 }
704 
705 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
706   nextToken();
707   nextToken();  // interface name
708 
709   // @interface can be followed by either a base class, or a category.
710   if (FormatTok.Tok.is(tok::colon)) {
711     nextToken();
712     nextToken();  // base class name
713   } else if (FormatTok.Tok.is(tok::l_paren))
714     // Skip category, if present.
715     parseParens();
716 
717   if (FormatTok.Tok.is(tok::less))
718     parseObjCProtocolList();
719 
720   // If instance variables are present, keep the '{' on the first line too.
721   if (FormatTok.Tok.is(tok::l_brace))
722     parseBlock(/*MustBeDeclaration=*/ true);
723 
724   // With instance variables, this puts '}' on its own line.  Without instance
725   // variables, this ends the @interface line.
726   addUnwrappedLine();
727 
728   parseObjCUntilAtEnd();
729 }
730 
731 void UnwrappedLineParser::parseObjCProtocol() {
732   nextToken();
733   nextToken();  // protocol name
734 
735   if (FormatTok.Tok.is(tok::less))
736     parseObjCProtocolList();
737 
738   // Check for protocol declaration.
739   if (FormatTok.Tok.is(tok::semi)) {
740     nextToken();
741     return addUnwrappedLine();
742   }
743 
744   addUnwrappedLine();
745   parseObjCUntilAtEnd();
746 }
747 
748 void UnwrappedLineParser::addUnwrappedLine() {
749   if (Line->Tokens.empty())
750     return;
751   DEBUG({
752     llvm::dbgs() << "Line(" << Line->Level << "): ";
753     for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
754                                           E = Line->Tokens.end();
755          I != E; ++I) {
756       llvm::dbgs() << I->Tok.getName() << " ";
757 
758     }
759     llvm::dbgs() << "\n";
760   });
761   CurrentLines->push_back(*Line);
762   Line->Tokens.clear();
763   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
764     for (std::vector<UnwrappedLine>::iterator I = PreprocessorDirectives
765              .begin(), E = PreprocessorDirectives.end();
766          I != E; ++I) {
767       CurrentLines->push_back(*I);
768     }
769     PreprocessorDirectives.clear();
770   }
771 
772 }
773 
774 bool UnwrappedLineParser::eof() const {
775   return FormatTok.Tok.is(tok::eof);
776 }
777 
778 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
779   bool JustComments = Line->Tokens.empty();
780   for (SmallVectorImpl<FormatToken>::const_iterator
781            I = CommentsBeforeNextToken.begin(),
782            E = CommentsBeforeNextToken.end();
783        I != E; ++I) {
784     if (I->HasUnescapedNewline && JustComments) {
785       addUnwrappedLine();
786     }
787     pushToken(*I);
788   }
789   if (NewlineBeforeNext && JustComments) {
790     addUnwrappedLine();
791   }
792   CommentsBeforeNextToken.clear();
793 }
794 
795 void UnwrappedLineParser::nextToken() {
796   if (eof())
797     return;
798   flushComments(FormatTok.HasUnescapedNewline);
799   pushToken(FormatTok);
800   readToken();
801 }
802 
803 void UnwrappedLineParser::readToken() {
804   bool CommentsInCurrentLine = true;
805   do {
806     FormatTok = Tokens->getNextToken();
807     while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
808            ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
809             FormatTok.IsFirst)) {
810       // If there is an unfinished unwrapped line, we flush the preprocessor
811       // directives only after that unwrapped line was finished later.
812       bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
813                                        CurrentLines == &Lines;
814       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
815       parsePPDirective();
816     }
817     if (!FormatTok.Tok.is(tok::comment))
818       return;
819     if (FormatTok.HasUnescapedNewline || FormatTok.IsFirst) {
820       CommentsInCurrentLine = false;
821     }
822     if (CommentsInCurrentLine) {
823       pushToken(FormatTok);
824     } else {
825       CommentsBeforeNextToken.push_back(FormatTok);
826     }
827   } while (!eof());
828 }
829 
830 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
831   Line->Tokens.push_back(Tok);
832   if (MustBreakBeforeNextToken) {
833     Line->Tokens.back().MustBreakBefore = true;
834     MustBreakBeforeNextToken = false;
835   }
836 }
837 
838 } // end namespace format
839 } // end namespace clang
840