1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-parser"
17 
18 #include "UnwrappedLineParser.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "llvm/Support/Debug.h"
21 
22 namespace clang {
23 namespace format {
24 
25 class ScopedDeclarationState {
26 public:
27   ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28                          bool MustBeDeclaration)
29       : Line(Line), Stack(Stack) {
30     Line.MustBeDeclaration = MustBeDeclaration;
31     Stack.push_back(MustBeDeclaration);
32   }
33   ~ScopedDeclarationState() {
34     Stack.pop_back();
35     if (!Stack.empty())
36       Line.MustBeDeclaration = Stack.back();
37     else
38       Line.MustBeDeclaration = true;
39   }
40 private:
41   UnwrappedLine &Line;
42   std::vector<bool> &Stack;
43 };
44 
45 class ScopedMacroState : public FormatTokenSource {
46 public:
47   ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48                    FormatToken &ResetToken)
49       : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50         PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
51     TokenSource = this;
52     Line.Level = 0;
53     Line.InPPDirective = true;
54   }
55 
56   ~ScopedMacroState() {
57     TokenSource = PreviousTokenSource;
58     ResetToken = Token;
59     Line.InPPDirective = false;
60     Line.Level = PreviousLineLevel;
61   }
62 
63   virtual FormatToken getNextToken() {
64     // The \c UnwrappedLineParser guards against this by never calling
65     // \c getNextToken() after it has encountered the first eof token.
66     assert(!eof());
67     Token = PreviousTokenSource->getNextToken();
68     if (eof())
69       return createEOF();
70     return Token;
71   }
72 
73 private:
74   bool eof() { return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline; }
75 
76   FormatToken createEOF() {
77     FormatToken FormatTok;
78     FormatTok.Tok.startToken();
79     FormatTok.Tok.setKind(tok::eof);
80     return FormatTok;
81   }
82 
83   UnwrappedLine &Line;
84   FormatTokenSource *&TokenSource;
85   FormatToken &ResetToken;
86   unsigned PreviousLineLevel;
87   FormatTokenSource *PreviousTokenSource;
88 
89   FormatToken Token;
90 };
91 
92 class ScopedLineState {
93 public:
94   ScopedLineState(UnwrappedLineParser &Parser,
95                   bool SwitchToPreprocessorLines = false)
96       : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
97     if (SwitchToPreprocessorLines)
98       Parser.CurrentLines = &Parser.PreprocessorDirectives;
99     PreBlockLine = Parser.Line.take();
100     Parser.Line.reset(new UnwrappedLine());
101     Parser.Line->Level = PreBlockLine->Level;
102     Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
103   }
104 
105   ~ScopedLineState() {
106     if (!Parser.Line->Tokens.empty()) {
107       Parser.addUnwrappedLine();
108     }
109     assert(Parser.Line->Tokens.empty());
110     Parser.Line.reset(PreBlockLine);
111     Parser.MustBreakBeforeNextToken = true;
112     if (SwitchToPreprocessorLines)
113       Parser.CurrentLines = &Parser.Lines;
114   }
115 
116 private:
117   UnwrappedLineParser &Parser;
118   const bool SwitchToPreprocessorLines;
119 
120   UnwrappedLine *PreBlockLine;
121 };
122 
123 UnwrappedLineParser::UnwrappedLineParser(
124     clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
125     FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
126     : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
127       CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
128       Callback(Callback) {}
129 
130 bool UnwrappedLineParser::parse() {
131   DEBUG(llvm::dbgs() << "----\n");
132   readToken();
133   bool Error = parseFile();
134   for (std::vector<UnwrappedLine>::iterator I = Lines.begin(), E = Lines.end();
135        I != E; ++I) {
136     Callback.consumeUnwrappedLine(*I);
137   }
138 
139   // Create line with eof token.
140   pushToken(FormatTok);
141   Callback.consumeUnwrappedLine(*Line);
142 
143   return Error;
144 }
145 
146 bool UnwrappedLineParser::parseFile() {
147   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
148                                           /*MustBeDeclaration=*/ true);
149   bool Error = parseLevel(/*HasOpeningBrace=*/ false);
150   // Make sure to format the remaining tokens.
151   flushComments(true);
152   addUnwrappedLine();
153   return Error;
154 }
155 
156 bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
157   bool Error = false;
158   do {
159     switch (FormatTok.Tok.getKind()) {
160     case tok::comment:
161       nextToken();
162       addUnwrappedLine();
163       break;
164     case tok::l_brace:
165       // FIXME: Add parameter whether this can happen - if this happens, we must
166       // be in a non-declaration context.
167       Error |= parseBlock(/*MustBeDeclaration=*/ false);
168       addUnwrappedLine();
169       break;
170     case tok::r_brace:
171       if (HasOpeningBrace) {
172         return false;
173       } else {
174         Diag.Report(FormatTok.Tok.getLocation(),
175                     Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
176                                          "unexpected '}'"));
177         Error = true;
178         nextToken();
179         addUnwrappedLine();
180       }
181       break;
182     default:
183       parseStructuralElement();
184       break;
185     }
186   } while (!eof());
187   return Error;
188 }
189 
190 bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
191                                      unsigned AddLevels) {
192   assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
193   nextToken();
194 
195   addUnwrappedLine();
196 
197   ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
198                                           MustBeDeclaration);
199   Line->Level += AddLevels;
200   parseLevel(/*HasOpeningBrace=*/ true);
201 
202   if (!FormatTok.Tok.is(tok::r_brace)) {
203     Line->Level -= AddLevels;
204     return true;
205   }
206 
207   nextToken(); // Munch the closing brace.
208   Line->Level -= AddLevels;
209   return false;
210 }
211 
212 void UnwrappedLineParser::parsePPDirective() {
213   assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
214   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
215   nextToken();
216 
217   if (FormatTok.Tok.getIdentifierInfo() == NULL) {
218     parsePPUnknown();
219     return;
220   }
221 
222   switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
223   case tok::pp_define:
224     parsePPDefine();
225     break;
226   default:
227     parsePPUnknown();
228     break;
229   }
230 }
231 
232 void UnwrappedLineParser::parsePPDefine() {
233   nextToken();
234 
235   if (FormatTok.Tok.getKind() != tok::identifier) {
236     parsePPUnknown();
237     return;
238   }
239   nextToken();
240   if (FormatTok.Tok.getKind() == tok::l_paren &&
241       FormatTok.WhiteSpaceLength == 0) {
242     parseParens();
243   }
244   addUnwrappedLine();
245   Line->Level = 1;
246 
247   // Errors during a preprocessor directive can only affect the layout of the
248   // preprocessor directive, and thus we ignore them. An alternative approach
249   // would be to use the same approach we use on the file level (no
250   // re-indentation if there was a structural error) within the macro
251   // definition.
252   parseFile();
253 }
254 
255 void UnwrappedLineParser::parsePPUnknown() {
256   do {
257     nextToken();
258   } while (!eof());
259   addUnwrappedLine();
260 }
261 
262 void UnwrappedLineParser::parseStructuralElement() {
263   assert(!FormatTok.Tok.is(tok::l_brace));
264   int TokenNumber = 0;
265   switch (FormatTok.Tok.getKind()) {
266   case tok::at:
267     nextToken();
268     if (FormatTok.Tok.is(tok::l_brace)) {
269       parseBracedList();
270       break;
271     }
272     switch (FormatTok.Tok.getObjCKeywordID()) {
273     case tok::objc_public:
274     case tok::objc_protected:
275     case tok::objc_package:
276     case tok::objc_private:
277       return parseAccessSpecifier();
278     case tok::objc_interface:
279     case tok::objc_implementation:
280       return parseObjCInterfaceOrImplementation();
281     case tok::objc_protocol:
282       return parseObjCProtocol();
283     case tok::objc_end:
284       return; // Handled by the caller.
285     case tok::objc_optional:
286     case tok::objc_required:
287       nextToken();
288       addUnwrappedLine();
289       return;
290     default:
291       break;
292     }
293     break;
294   case tok::kw_namespace:
295     parseNamespace();
296     return;
297   case tok::kw_inline:
298     nextToken();
299     TokenNumber++;
300     if (FormatTok.Tok.is(tok::kw_namespace)) {
301       parseNamespace();
302       return;
303     }
304     break;
305   case tok::kw_public:
306   case tok::kw_protected:
307   case tok::kw_private:
308     parseAccessSpecifier();
309     return;
310   case tok::kw_if:
311     parseIfThenElse();
312     return;
313   case tok::kw_for:
314   case tok::kw_while:
315     parseForOrWhileLoop();
316     return;
317   case tok::kw_do:
318     parseDoWhile();
319     return;
320   case tok::kw_switch:
321     parseSwitch();
322     return;
323   case tok::kw_default:
324     nextToken();
325     parseLabel();
326     return;
327   case tok::kw_case:
328     parseCaseLabel();
329     return;
330   case tok::kw_return:
331     parseReturn();
332     return;
333   case tok::kw_extern:
334     nextToken();
335     if (FormatTok.Tok.is(tok::string_literal)) {
336       nextToken();
337       if (FormatTok.Tok.is(tok::l_brace)) {
338         parseBlock(/*MustBeDeclaration=*/ true, 0);
339         addUnwrappedLine();
340         return;
341       }
342     }
343     // In all other cases, parse the declaration.
344     break;
345   default:
346     break;
347   }
348   do {
349     ++TokenNumber;
350     switch (FormatTok.Tok.getKind()) {
351     case tok::at:
352       nextToken();
353       if (FormatTok.Tok.is(tok::l_brace))
354         parseBracedList();
355       break;
356     case tok::kw_enum:
357       parseEnum();
358       break;
359     case tok::kw_struct:
360     case tok::kw_union:
361     case tok::kw_class:
362       parseRecord();
363       // A record declaration or definition is always the start of a structural
364       // element.
365       break;
366     case tok::semi:
367       nextToken();
368       addUnwrappedLine();
369       return;
370     case tok::r_brace:
371       addUnwrappedLine();
372       return;
373     case tok::l_paren:
374       parseParens();
375       break;
376     case tok::l_brace:
377       // A block outside of parentheses must be the last part of a
378       // structural element.
379       // FIXME: Figure out cases where this is not true, and add projections for
380       // them (the one we know is missing are lambdas).
381       parseBlock(/*MustBeDeclaration=*/ false);
382       addUnwrappedLine();
383       return;
384     case tok::identifier:
385       nextToken();
386       if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
387         parseLabel();
388         return;
389       }
390       break;
391     case tok::equal:
392       nextToken();
393       if (FormatTok.Tok.is(tok::l_brace)) {
394         parseBracedList();
395       }
396       break;
397     default:
398       nextToken();
399       break;
400     }
401   } while (!eof());
402 }
403 
404 void UnwrappedLineParser::parseBracedList() {
405   nextToken();
406 
407   do {
408     switch (FormatTok.Tok.getKind()) {
409     case tok::l_brace:
410       parseBracedList();
411       break;
412     case tok::r_brace:
413       nextToken();
414       return;
415     default:
416       nextToken();
417       break;
418     }
419   } while (!eof());
420 }
421 
422 void UnwrappedLineParser::parseReturn() {
423   nextToken();
424 
425   do {
426     switch (FormatTok.Tok.getKind()) {
427     case tok::l_brace:
428       parseBracedList();
429       break;
430     case tok::l_paren:
431       parseParens();
432       break;
433     case tok::r_brace:
434       // Assume missing ';'.
435       addUnwrappedLine();
436       return;
437     case tok::semi:
438       nextToken();
439       addUnwrappedLine();
440       return;
441     default:
442       nextToken();
443       break;
444     }
445   } while (!eof());
446 }
447 
448 void UnwrappedLineParser::parseParens() {
449   assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
450   nextToken();
451   do {
452     switch (FormatTok.Tok.getKind()) {
453     case tok::l_paren:
454       parseParens();
455       break;
456     case tok::r_paren:
457       nextToken();
458       return;
459     case tok::l_brace: {
460       nextToken();
461       ScopedLineState LineState(*this);
462       ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
463                                               /*MustBeDeclaration=*/ false);
464       Line->Level += 1;
465       parseLevel(/*HasOpeningBrace=*/ true);
466       Line->Level -= 1;
467       break;
468     }
469     case tok::at:
470       nextToken();
471       if (FormatTok.Tok.is(tok::l_brace))
472         parseBracedList();
473       break;
474     default:
475       nextToken();
476       break;
477     }
478   } while (!eof());
479 }
480 
481 void UnwrappedLineParser::parseIfThenElse() {
482   assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
483   nextToken();
484   if (FormatTok.Tok.is(tok::l_paren))
485     parseParens();
486   bool NeedsUnwrappedLine = false;
487   if (FormatTok.Tok.is(tok::l_brace)) {
488     parseBlock(/*MustBeDeclaration=*/ false);
489     NeedsUnwrappedLine = true;
490   } else {
491     addUnwrappedLine();
492     ++Line->Level;
493     parseStructuralElement();
494     --Line->Level;
495   }
496   if (FormatTok.Tok.is(tok::kw_else)) {
497     nextToken();
498     if (FormatTok.Tok.is(tok::l_brace)) {
499       parseBlock(/*MustBeDeclaration=*/ false);
500       addUnwrappedLine();
501     } else if (FormatTok.Tok.is(tok::kw_if)) {
502       parseIfThenElse();
503     } else {
504       addUnwrappedLine();
505       ++Line->Level;
506       parseStructuralElement();
507       --Line->Level;
508     }
509   } else if (NeedsUnwrappedLine) {
510     addUnwrappedLine();
511   }
512 }
513 
514 void UnwrappedLineParser::parseNamespace() {
515   assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
516   nextToken();
517   if (FormatTok.Tok.is(tok::identifier))
518     nextToken();
519   if (FormatTok.Tok.is(tok::l_brace)) {
520     parseBlock(/*MustBeDeclaration=*/ true, 0);
521     // Munch the semicolon after a namespace. This is more common than one would
522     // think. Puttin the semicolon into its own line is very ugly.
523     if (FormatTok.Tok.is(tok::semi))
524       nextToken();
525     addUnwrappedLine();
526   }
527   // FIXME: Add error handling.
528 }
529 
530 void UnwrappedLineParser::parseForOrWhileLoop() {
531   assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
532          "'for' or 'while' expected");
533   nextToken();
534   if (FormatTok.Tok.is(tok::l_paren))
535     parseParens();
536   if (FormatTok.Tok.is(tok::l_brace)) {
537     parseBlock(/*MustBeDeclaration=*/ false);
538     addUnwrappedLine();
539   } else {
540     addUnwrappedLine();
541     ++Line->Level;
542     parseStructuralElement();
543     --Line->Level;
544   }
545 }
546 
547 void UnwrappedLineParser::parseDoWhile() {
548   assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
549   nextToken();
550   if (FormatTok.Tok.is(tok::l_brace)) {
551     parseBlock(/*MustBeDeclaration=*/ false);
552   } else {
553     addUnwrappedLine();
554     ++Line->Level;
555     parseStructuralElement();
556     --Line->Level;
557   }
558 
559   // FIXME: Add error handling.
560   if (!FormatTok.Tok.is(tok::kw_while)) {
561     addUnwrappedLine();
562     return;
563   }
564 
565   nextToken();
566   parseStructuralElement();
567 }
568 
569 void UnwrappedLineParser::parseLabel() {
570   if (FormatTok.Tok.isNot(tok::colon))
571     return;
572   nextToken();
573   unsigned OldLineLevel = Line->Level;
574   if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
575     --Line->Level;
576   if (CommentsBeforeNextToken.empty() && FormatTok.Tok.is(tok::l_brace)) {
577     parseBlock(/*MustBeDeclaration=*/ false);
578     if (FormatTok.Tok.is(tok::kw_break))
579       parseStructuralElement(); // "break;" after "}" goes on the same line.
580   }
581   addUnwrappedLine();
582   Line->Level = OldLineLevel;
583 }
584 
585 void UnwrappedLineParser::parseCaseLabel() {
586   assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
587   // FIXME: fix handling of complex expressions here.
588   do {
589     nextToken();
590   } while (!eof() && !FormatTok.Tok.is(tok::colon));
591   parseLabel();
592 }
593 
594 void UnwrappedLineParser::parseSwitch() {
595   assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
596   nextToken();
597   if (FormatTok.Tok.is(tok::l_paren))
598     parseParens();
599   if (FormatTok.Tok.is(tok::l_brace)) {
600     parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
601     addUnwrappedLine();
602   } else {
603     addUnwrappedLine();
604     Line->Level += (Style.IndentCaseLabels ? 2 : 1);
605     parseStructuralElement();
606     Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
607   }
608 }
609 
610 void UnwrappedLineParser::parseAccessSpecifier() {
611   nextToken();
612   // Otherwise, we don't know what it is, and we'd better keep the next token.
613   if (FormatTok.Tok.is(tok::colon))
614     nextToken();
615   addUnwrappedLine();
616 }
617 
618 void UnwrappedLineParser::parseEnum() {
619   nextToken();
620   if (FormatTok.Tok.is(tok::identifier) ||
621       FormatTok.Tok.is(tok::kw___attribute) ||
622       FormatTok.Tok.is(tok::kw___declspec)) {
623     nextToken();
624     // We can have macros or attributes in between 'enum' and the enum name.
625     if (FormatTok.Tok.is(tok::l_paren)) {
626       parseParens();
627     }
628     if (FormatTok.Tok.is(tok::identifier))
629       nextToken();
630   }
631   if (FormatTok.Tok.is(tok::l_brace)) {
632     nextToken();
633     addUnwrappedLine();
634     ++Line->Level;
635     do {
636       switch (FormatTok.Tok.getKind()) {
637       case tok::l_paren:
638         parseParens();
639         break;
640       case tok::r_brace:
641         addUnwrappedLine();
642         nextToken();
643         --Line->Level;
644         return;
645       case tok::comma:
646         nextToken();
647         addUnwrappedLine();
648         break;
649       default:
650         nextToken();
651         break;
652       }
653     } while (!eof());
654   }
655   // We fall through to parsing a structural element afterwards, so that in
656   // enum A {} n, m;
657   // "} n, m;" will end up in one unwrapped line.
658 }
659 
660 void UnwrappedLineParser::parseRecord() {
661   nextToken();
662   if (FormatTok.Tok.is(tok::identifier) ||
663       FormatTok.Tok.is(tok::kw___attribute) ||
664       FormatTok.Tok.is(tok::kw___declspec)) {
665     nextToken();
666     // We can have macros or attributes in between 'class' and the class name.
667     if (FormatTok.Tok.is(tok::l_paren)) {
668       parseParens();
669     }
670     // The actual identifier can be a nested name specifier, and in macros
671     // it is often token-pasted.
672     while (FormatTok.Tok.is(tok::identifier) ||
673            FormatTok.Tok.is(tok::coloncolon) || FormatTok.Tok.is(tok::hashhash))
674       nextToken();
675 
676     // Note that parsing away template declarations here leads to incorrectly
677     // accepting function declarations as record declarations.
678     // In general, we cannot solve this problem. Consider:
679     // class A<int> B() {}
680     // which can be a function definition or a class definition when B() is a
681     // macro. If we find enough real-world cases where this is a problem, we
682     // can parse for the 'template' keyword in the beginning of the statement,
683     // and thus rule out the record production in case there is no template
684     // (this would still leave us with an ambiguity between template function
685     // and class declarations).
686     if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
687       while (!eof() && FormatTok.Tok.isNot(tok::l_brace)) {
688         if (FormatTok.Tok.is(tok::semi))
689           return;
690         nextToken();
691       }
692     }
693   }
694   if (FormatTok.Tok.is(tok::l_brace))
695     parseBlock(/*MustBeDeclaration=*/ true);
696   // We fall through to parsing a structural element afterwards, so
697   // class A {} n, m;
698   // will end up in one unwrapped line.
699 }
700 
701 void UnwrappedLineParser::parseObjCProtocolList() {
702   assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
703   do
704     nextToken();
705   while (!eof() && FormatTok.Tok.isNot(tok::greater));
706   nextToken(); // Skip '>'.
707 }
708 
709 void UnwrappedLineParser::parseObjCUntilAtEnd() {
710   do {
711     if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
712       nextToken();
713       addUnwrappedLine();
714       break;
715     }
716     parseStructuralElement();
717   } while (!eof());
718 }
719 
720 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
721   nextToken();
722   nextToken(); // interface name
723 
724   // @interface can be followed by either a base class, or a category.
725   if (FormatTok.Tok.is(tok::colon)) {
726     nextToken();
727     nextToken(); // base class name
728   } else if (FormatTok.Tok.is(tok::l_paren))
729     // Skip category, if present.
730     parseParens();
731 
732   if (FormatTok.Tok.is(tok::less))
733     parseObjCProtocolList();
734 
735   // If instance variables are present, keep the '{' on the first line too.
736   if (FormatTok.Tok.is(tok::l_brace))
737     parseBlock(/*MustBeDeclaration=*/ true);
738 
739   // With instance variables, this puts '}' on its own line.  Without instance
740   // variables, this ends the @interface line.
741   addUnwrappedLine();
742 
743   parseObjCUntilAtEnd();
744 }
745 
746 void UnwrappedLineParser::parseObjCProtocol() {
747   nextToken();
748   nextToken(); // protocol name
749 
750   if (FormatTok.Tok.is(tok::less))
751     parseObjCProtocolList();
752 
753   // Check for protocol declaration.
754   if (FormatTok.Tok.is(tok::semi)) {
755     nextToken();
756     return addUnwrappedLine();
757   }
758 
759   addUnwrappedLine();
760   parseObjCUntilAtEnd();
761 }
762 
763 void UnwrappedLineParser::addUnwrappedLine() {
764   if (Line->Tokens.empty())
765     return;
766   DEBUG({
767     llvm::dbgs() << "Line(" << Line->Level << ")"
768                  << (Line->InPPDirective ? " MACRO" : "") << ": ";
769     for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
770                                           E = Line->Tokens.end();
771          I != E; ++I) {
772       llvm::dbgs() << I->Tok.getName() << " ";
773 
774     }
775     llvm::dbgs() << "\n";
776   });
777   CurrentLines->push_back(*Line);
778   Line->Tokens.clear();
779   if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
780     for (std::vector<UnwrappedLine>::iterator
781              I = PreprocessorDirectives.begin(),
782              E = PreprocessorDirectives.end();
783          I != E; ++I) {
784       CurrentLines->push_back(*I);
785     }
786     PreprocessorDirectives.clear();
787   }
788 }
789 
790 bool UnwrappedLineParser::eof() const { return FormatTok.Tok.is(tok::eof); }
791 
792 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
793   bool JustComments = Line->Tokens.empty();
794   for (SmallVectorImpl<FormatToken>::const_iterator
795            I = CommentsBeforeNextToken.begin(),
796            E = CommentsBeforeNextToken.end();
797        I != E; ++I) {
798     if (I->NewlinesBefore && JustComments) {
799       addUnwrappedLine();
800     }
801     pushToken(*I);
802   }
803   if (NewlineBeforeNext && JustComments) {
804     addUnwrappedLine();
805   }
806   CommentsBeforeNextToken.clear();
807 }
808 
809 void UnwrappedLineParser::nextToken() {
810   if (eof())
811     return;
812   flushComments(FormatTok.NewlinesBefore > 0);
813   pushToken(FormatTok);
814   readToken();
815 }
816 
817 void UnwrappedLineParser::readToken() {
818   bool CommentsInCurrentLine = true;
819   do {
820     FormatTok = Tokens->getNextToken();
821     while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
822            ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
823             FormatTok.IsFirst)) {
824       // If there is an unfinished unwrapped line, we flush the preprocessor
825       // directives only after that unwrapped line was finished later.
826       bool SwitchToPreprocessorLines =
827           !Line->Tokens.empty() && CurrentLines == &Lines;
828       ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
829       parsePPDirective();
830     }
831     if (!FormatTok.Tok.is(tok::comment))
832       return;
833     if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
834       CommentsInCurrentLine = false;
835     }
836     if (CommentsInCurrentLine) {
837       pushToken(FormatTok);
838     } else {
839       CommentsBeforeNextToken.push_back(FormatTok);
840     }
841   } while (!eof());
842 }
843 
844 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
845   Line->Tokens.push_back(Tok);
846   if (MustBreakBeforeNextToken) {
847     Line->Tokens.back().MustBreakBefore = true;
848     MustBreakBeforeNextToken = false;
849   }
850 }
851 
852 } // end namespace format
853 } // end namespace clang
854