1 //===--- BreakableToken.cpp - Format C++ code -----------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Contains implementation of BreakableToken class and classes derived
12 /// from it.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #define DEBUG_TYPE "format-token-breaker"
17 
18 #include "BreakableToken.h"
19 #include "clang/Basic/CharInfo.h"
20 #include "clang/Format/Format.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/Support/Debug.h"
23 #include <algorithm>
24 
25 namespace clang {
26 namespace format {
27 namespace {
28 
29 BreakableToken::Split getCommentSplit(StringRef Text,
30                                       unsigned ContentStartColumn,
31                                       unsigned ColumnLimit,
32                                       encoding::Encoding Encoding) {
33   if (ColumnLimit <= ContentStartColumn + 1)
34     return BreakableToken::Split(StringRef::npos, 0);
35 
36   unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
37   unsigned MaxSplitBytes = 0;
38 
39   for (unsigned NumChars = 0;
40        NumChars < MaxSplit && MaxSplitBytes < Text.size(); ++NumChars)
41     MaxSplitBytes +=
42         encoding::getCodePointNumBytes(Text[MaxSplitBytes], Encoding);
43 
44   StringRef::size_type SpaceOffset = Text.rfind(' ', MaxSplitBytes);
45   if (SpaceOffset == StringRef::npos ||
46       // Don't break at leading whitespace.
47       Text.find_last_not_of(' ', SpaceOffset) == StringRef::npos) {
48     // Make sure that we don't break at leading whitespace that
49     // reaches past MaxSplit.
50     StringRef::size_type FirstNonWhitespace = Text.find_first_not_of(" ");
51     if (FirstNonWhitespace == StringRef::npos)
52       // If the comment is only whitespace, we cannot split.
53       return BreakableToken::Split(StringRef::npos, 0);
54     SpaceOffset =
55         Text.find(' ', std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
56   }
57   if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
58     StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim();
59     StringRef AfterCut = Text.substr(SpaceOffset).ltrim();
60     return BreakableToken::Split(BeforeCut.size(),
61                                  AfterCut.begin() - BeforeCut.end());
62   }
63   return BreakableToken::Split(StringRef::npos, 0);
64 }
65 
66 BreakableToken::Split getStringSplit(StringRef Text,
67                                      unsigned ContentStartColumn,
68                                      unsigned ColumnLimit,
69                                      encoding::Encoding Encoding) {
70   // FIXME: Reduce unit test case.
71   if (Text.empty())
72     return BreakableToken::Split(StringRef::npos, 0);
73   if (ColumnLimit <= ContentStartColumn)
74     return BreakableToken::Split(StringRef::npos, 0);
75   unsigned MaxSplit =
76       std::min<unsigned>(ColumnLimit - ContentStartColumn,
77                          encoding::getCodePointCount(Text, Encoding) - 1);
78   StringRef::size_type SpaceOffset = 0;
79   StringRef::size_type SlashOffset = 0;
80   StringRef::size_type SplitPoint = 0;
81   for (unsigned Chars = 0;;) {
82     unsigned Advance;
83     if (Text[0] == '\\') {
84       Advance = encoding::getEscapeSequenceLength(Text);
85       Chars += Advance;
86     } else {
87       Advance = encoding::getCodePointNumBytes(Text[0], Encoding);
88       Chars += 1;
89     }
90 
91     if (Chars > MaxSplit)
92       break;
93 
94     if (Text[0] == ' ')
95       SpaceOffset = SplitPoint;
96     if (Text[0] == '/')
97       SlashOffset = SplitPoint;
98 
99     SplitPoint += Advance;
100     Text = Text.substr(Advance);
101   }
102 
103   if (SpaceOffset != 0)
104     return BreakableToken::Split(SpaceOffset + 1, 0);
105   if (SlashOffset != 0)
106     return BreakableToken::Split(SlashOffset + 1, 0);
107   if (SplitPoint != 0)
108     return BreakableToken::Split(SplitPoint, 0);
109   return BreakableToken::Split(StringRef::npos, 0);
110 }
111 
112 } // namespace
113 
114 unsigned BreakableSingleLineToken::getLineCount() const { return 1; }
115 
116 unsigned BreakableSingleLineToken::getLineLengthAfterSplit(
117     unsigned LineIndex, unsigned Offset, StringRef::size_type Length) const {
118   return StartColumn + Prefix.size() + Postfix.size() +
119          encoding::getCodePointCount(Line.substr(Offset, Length), Encoding);
120 }
121 
122 BreakableSingleLineToken::BreakableSingleLineToken(
123     const FormatToken &Tok, unsigned StartColumn, StringRef Prefix,
124     StringRef Postfix, bool InPPDirective, encoding::Encoding Encoding)
125     : BreakableToken(Tok, InPPDirective, Encoding), StartColumn(StartColumn),
126       Prefix(Prefix), Postfix(Postfix) {
127   assert(Tok.TokenText.startswith(Prefix) && Tok.TokenText.endswith(Postfix));
128   Line = Tok.TokenText.substr(
129       Prefix.size(), Tok.TokenText.size() - Prefix.size() - Postfix.size());
130 }
131 
132 BreakableStringLiteral::BreakableStringLiteral(const FormatToken &Tok,
133                                                unsigned StartColumn,
134                                                bool InPPDirective,
135                                                encoding::Encoding Encoding)
136     : BreakableSingleLineToken(Tok, StartColumn, "\"", "\"", InPPDirective,
137                                Encoding) {}
138 
139 BreakableToken::Split
140 BreakableStringLiteral::getSplit(unsigned LineIndex, unsigned TailOffset,
141                                  unsigned ColumnLimit) const {
142   return getStringSplit(Line.substr(TailOffset), StartColumn + 2, ColumnLimit,
143                         Encoding);
144 }
145 
146 void BreakableStringLiteral::insertBreak(unsigned LineIndex,
147                                          unsigned TailOffset, Split Split,
148                                          WhitespaceManager &Whitespaces) {
149   Whitespaces.replaceWhitespaceInToken(
150       Tok, Prefix.size() + TailOffset + Split.first, Split.second, Postfix,
151       Prefix, InPPDirective, 1, StartColumn);
152 }
153 
154 static StringRef getLineCommentPrefix(StringRef Comment) {
155   const char *KnownPrefixes[] = { "/// ", "///", "// ", "//" };
156   for (size_t i = 0, e = llvm::array_lengthof(KnownPrefixes); i != e; ++i)
157     if (Comment.startswith(KnownPrefixes[i]))
158       return KnownPrefixes[i];
159   return "";
160 }
161 
162 BreakableLineComment::BreakableLineComment(const FormatToken &Token,
163                                            unsigned StartColumn,
164                                            bool InPPDirective,
165                                            encoding::Encoding Encoding)
166     : BreakableSingleLineToken(Token, StartColumn,
167                                getLineCommentPrefix(Token.TokenText), "",
168                                InPPDirective, Encoding) {
169   OriginalPrefix = Prefix;
170   if (Token.TokenText.size() > Prefix.size() &&
171       isAlphanumeric(Token.TokenText[Prefix.size()])) {
172     if (Prefix == "//")
173       Prefix = "// ";
174     else if (Prefix == "///")
175       Prefix = "/// ";
176   }
177 }
178 
179 BreakableToken::Split
180 BreakableLineComment::getSplit(unsigned LineIndex, unsigned TailOffset,
181                                unsigned ColumnLimit) const {
182   return getCommentSplit(Line.substr(TailOffset), StartColumn + Prefix.size(),
183                          ColumnLimit, Encoding);
184 }
185 
186 void BreakableLineComment::insertBreak(unsigned LineIndex, unsigned TailOffset,
187                                        Split Split,
188                                        WhitespaceManager &Whitespaces) {
189   Whitespaces.replaceWhitespaceInToken(
190       Tok, OriginalPrefix.size() + TailOffset + Split.first, Split.second,
191       Postfix, Prefix, InPPDirective, 1, StartColumn);
192 }
193 
194 void
195 BreakableLineComment::replaceWhitespaceBefore(unsigned LineIndex,
196                                               WhitespaceManager &Whitespaces) {
197   if (OriginalPrefix != Prefix) {
198     Whitespaces.replaceWhitespaceInToken(Tok, OriginalPrefix.size(), 0, "", "",
199                                          false, 0, 1);
200   }
201 }
202 
203 BreakableBlockComment::BreakableBlockComment(
204     const FormatStyle &Style, const FormatToken &Token, unsigned StartColumn,
205     unsigned OriginalStartColumn, bool FirstInLine, bool InPPDirective,
206     encoding::Encoding Encoding)
207     : BreakableToken(Token, InPPDirective, Encoding) {
208   StringRef TokenText(Token.TokenText);
209   assert(TokenText.startswith("/*") && TokenText.endswith("*/"));
210   TokenText.substr(2, TokenText.size() - 4).split(Lines, "\n");
211 
212   int IndentDelta = StartColumn - OriginalStartColumn;
213   bool NeedsStar = true;
214   LeadingWhitespace.resize(Lines.size());
215   StartOfLineColumn.resize(Lines.size());
216   if (Lines.size() == 1 && !FirstInLine) {
217     // Comments for which FirstInLine is false can start on arbitrary column,
218     // and available horizontal space can be too small to align consecutive
219     // lines with the first one.
220     // FIXME: We could, probably, align them to current indentation level, but
221     // now we just wrap them without stars.
222     NeedsStar = false;
223   }
224   StartOfLineColumn[0] = StartColumn + 2;
225   for (size_t i = 1; i < Lines.size(); ++i) {
226     adjustWhitespace(Style, i, IndentDelta);
227     if (Lines[i].empty())
228       // If the last line is empty, the closing "*/" will have a star.
229       NeedsStar = NeedsStar && i + 1 == Lines.size();
230     else
231       NeedsStar = NeedsStar && Lines[i][0] == '*';
232   }
233   Decoration = NeedsStar ? "* " : "";
234   IndentAtLineBreak = StartOfLineColumn[0] + 1;
235   for (size_t i = 1; i < Lines.size(); ++i) {
236     if (Lines[i].empty()) {
237       if (!NeedsStar && i + 1 != Lines.size())
238         // For all but the last line (which always ends in */), set the
239         // start column to 0 if they're empty, so we do not insert
240         // trailing whitespace anywhere.
241         StartOfLineColumn[i] = 0;
242       continue;
243     }
244     if (NeedsStar) {
245       // The first line already excludes the star.
246       // For all other lines, adjust the line to exclude the star and
247       // (optionally) the first whitespace.
248       int Offset = Lines[i].startswith("* ") ? 2 : 1;
249       StartOfLineColumn[i] += Offset;
250       Lines[i] = Lines[i].substr(Offset);
251       LeadingWhitespace[i] += Offset;
252     }
253     IndentAtLineBreak = std::min<int>(IndentAtLineBreak, StartOfLineColumn[i]);
254   }
255   IndentAtLineBreak = std::max<unsigned>(IndentAtLineBreak, Decoration.size());
256   DEBUG({
257     for (size_t i = 0; i < Lines.size(); ++i) {
258       llvm::dbgs() << i << " |" << Lines[i] << "| " << LeadingWhitespace[i]
259                    << "\n";
260     }
261   });
262 }
263 
264 void BreakableBlockComment::adjustWhitespace(const FormatStyle &Style,
265                                              unsigned LineIndex,
266                                              int IndentDelta) {
267   // When in a preprocessor directive, the trailing backslash in a block comment
268   // is not needed, but can serve a purpose of uniformity with necessary escaped
269   // newlines outside the comment. In this case we remove it here before
270   // trimming the trailing whitespace. The backslash will be re-added later when
271   // inserting a line break.
272   size_t EndOfPreviousLine = Lines[LineIndex - 1].size();
273   if (InPPDirective && Lines[LineIndex - 1].endswith("\\"))
274     --EndOfPreviousLine;
275 
276   // Calculate the end of the non-whitespace text in the previous line.
277   EndOfPreviousLine =
278       Lines[LineIndex - 1].find_last_not_of(" \t", EndOfPreviousLine);
279   if (EndOfPreviousLine == StringRef::npos)
280     EndOfPreviousLine = 0;
281   else
282     ++EndOfPreviousLine;
283   // Calculate the start of the non-whitespace text in the current line.
284   size_t StartOfLine = Lines[LineIndex].find_first_not_of(" \t");
285   if (StartOfLine == StringRef::npos)
286     StartOfLine = Lines[LineIndex].size();
287 
288   // Adjust Lines to only contain relevant text.
289   Lines[LineIndex - 1] = Lines[LineIndex - 1].substr(0, EndOfPreviousLine);
290   Lines[LineIndex] = Lines[LineIndex].substr(StartOfLine);
291   // Adjust LeadingWhitespace to account all whitespace between the lines
292   // to the current line.
293   LeadingWhitespace[LineIndex] =
294       Lines[LineIndex].begin() - Lines[LineIndex - 1].end();
295 
296   // FIXME: We currently count tabs as 1 character. To solve this, we need to
297   // get the correct indentation width of the start of the comment, which
298   // requires correct counting of the tab expansions before the comment, and
299   // a configurable tab width. Since the current implementation only breaks
300   // if leading tabs are intermixed with spaces, that is not a high priority.
301 
302   // Adjust the start column uniformly accross all lines.
303   StartOfLineColumn[LineIndex] = std::max<int>(0, StartOfLine + IndentDelta);
304 }
305 
306 unsigned BreakableBlockComment::getLineCount() const { return Lines.size(); }
307 
308 unsigned BreakableBlockComment::getLineLengthAfterSplit(
309     unsigned LineIndex, unsigned Offset, StringRef::size_type Length) const {
310   return getContentStartColumn(LineIndex, Offset) +
311          encoding::getCodePointCount(Lines[LineIndex].substr(Offset, Length),
312                                      Encoding) +
313          // The last line gets a "*/" postfix.
314          (LineIndex + 1 == Lines.size() ? 2 : 0);
315 }
316 
317 BreakableToken::Split
318 BreakableBlockComment::getSplit(unsigned LineIndex, unsigned TailOffset,
319                                 unsigned ColumnLimit) const {
320   return getCommentSplit(Lines[LineIndex].substr(TailOffset),
321                          getContentStartColumn(LineIndex, TailOffset),
322                          ColumnLimit, Encoding);
323 }
324 
325 void BreakableBlockComment::insertBreak(unsigned LineIndex, unsigned TailOffset,
326                                         Split Split,
327                                         WhitespaceManager &Whitespaces) {
328   StringRef Text = Lines[LineIndex].substr(TailOffset);
329   StringRef Prefix = Decoration;
330   if (LineIndex + 1 == Lines.size() &&
331       Text.size() == Split.first + Split.second) {
332     // For the last line we need to break before "*/", but not to add "* ".
333     Prefix = "";
334   }
335 
336   unsigned BreakOffsetInToken =
337       Text.data() - Tok.TokenText.data() + Split.first;
338   unsigned CharsToRemove = Split.second;
339   assert(IndentAtLineBreak >= Decoration.size());
340   Whitespaces.replaceWhitespaceInToken(Tok, BreakOffsetInToken, CharsToRemove,
341                                        "", Prefix, InPPDirective, 1,
342                                        IndentAtLineBreak - Decoration.size());
343 }
344 
345 void
346 BreakableBlockComment::replaceWhitespaceBefore(unsigned LineIndex,
347                                                WhitespaceManager &Whitespaces) {
348   if (LineIndex == 0)
349     return;
350   StringRef Prefix = Decoration;
351   if (Lines[LineIndex].empty()) {
352     if (LineIndex + 1 == Lines.size()) {
353       // If the last line is empty, we don't need a prefix, as the */ will line
354       // up with the decoration (if it exists).
355       Prefix = "";
356     } else if (!Decoration.empty()) {
357       // For other empty lines, if we do have a decoration, adapt it to not
358       // contain a trailing whitespace.
359       Prefix = Prefix.substr(0, 1);
360     }
361   } else {
362     if (StartOfLineColumn[LineIndex] == 1) {
363       // This lines starts immediately after the decorating *.
364       Prefix = Prefix.substr(0, 1);
365     }
366   }
367 
368   unsigned WhitespaceOffsetInToken =
369       Lines[LineIndex].data() - Tok.TokenText.data() -
370       LeadingWhitespace[LineIndex];
371   assert(StartOfLineColumn[LineIndex] >= Prefix.size());
372   Whitespaces.replaceWhitespaceInToken(
373       Tok, WhitespaceOffsetInToken, LeadingWhitespace[LineIndex], "", Prefix,
374       InPPDirective, 1, StartOfLineColumn[LineIndex] - Prefix.size());
375 }
376 
377 unsigned
378 BreakableBlockComment::getContentStartColumn(unsigned LineIndex,
379                                              unsigned TailOffset) const {
380   // If we break, we always break at the predefined indent.
381   if (TailOffset != 0)
382     return IndentAtLineBreak;
383   return StartOfLineColumn[LineIndex];
384 }
385 
386 } // namespace format
387 } // namespace clang
388