1 //===--- FormatToken.h - Format C++ code ------------------------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 /// 10 /// \file 11 /// \brief This file contains the declaration of the FormatToken, a wrapper 12 /// around Token with additional information related to formatting. 13 /// 14 //===----------------------------------------------------------------------===// 15 16 #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H 17 #define LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H 18 19 #include "clang/Basic/IdentifierTable.h" 20 #include "clang/Basic/OperatorPrecedence.h" 21 #include "clang/Format/Format.h" 22 #include "clang/Lex/Lexer.h" 23 #include <memory> 24 25 namespace clang { 26 namespace format { 27 28 enum TokenType { 29 TT_ArrayInitializerLSquare, 30 TT_ArraySubscriptLSquare, 31 TT_AttributeParen, 32 TT_BinaryOperator, 33 TT_BitFieldColon, 34 TT_BlockComment, 35 TT_CastRParen, 36 TT_ConditionalExpr, 37 TT_ConflictAlternative, 38 TT_ConflictEnd, 39 TT_ConflictStart, 40 TT_CtorInitializerColon, 41 TT_CtorInitializerComma, 42 TT_DesignatedInitializerPeriod, 43 TT_DictLiteral, 44 TT_FunctionDeclarationName, 45 TT_FunctionLBrace, 46 TT_FunctionTypeLParen, 47 TT_ImplicitStringLiteral, 48 TT_InheritanceColon, 49 TT_InlineASMColon, 50 TT_JavaAnnotation, 51 TT_LambdaLSquare, 52 TT_LeadingJavaAnnotation, 53 TT_LineComment, 54 TT_ObjCBlockLBrace, 55 TT_ObjCBlockLParen, 56 TT_ObjCDecl, 57 TT_ObjCForIn, 58 TT_ObjCMethodExpr, 59 TT_ObjCMethodSpecifier, 60 TT_ObjCProperty, 61 TT_OverloadedOperator, 62 TT_OverloadedOperatorLParen, 63 TT_PointerOrReference, 64 TT_PureVirtualSpecifier, 65 TT_RangeBasedForLoopColon, 66 TT_RegexLiteral, 67 TT_SelectorName, 68 TT_StartOfName, 69 TT_TemplateCloser, 70 TT_TemplateOpener, 71 TT_TrailingAnnotation, 72 TT_TrailingReturnArrow, 73 TT_TrailingUnaryOperator, 74 TT_UnaryOperator, 75 TT_Unknown 76 }; 77 78 // Represents what type of block a set of braces open. 79 enum BraceBlockKind { 80 BK_Unknown, 81 BK_Block, 82 BK_BracedInit 83 }; 84 85 // The packing kind of a function's parameters. 86 enum ParameterPackingKind { 87 PPK_BinPacked, 88 PPK_OnePerLine, 89 PPK_Inconclusive 90 }; 91 92 enum FormatDecision { 93 FD_Unformatted, 94 FD_Continue, 95 FD_Break 96 }; 97 98 class TokenRole; 99 class AnnotatedLine; 100 101 /// \brief A wrapper around a \c Token storing information about the 102 /// whitespace characters preceding it. 103 struct FormatToken { 104 FormatToken() 105 : NewlinesBefore(0), HasUnescapedNewline(false), LastNewlineOffset(0), 106 ColumnWidth(0), LastLineColumnWidth(0), IsMultiline(false), 107 IsFirst(false), MustBreakBefore(false), IsUnterminatedLiteral(false), 108 BlockKind(BK_Unknown), Type(TT_Unknown), SpacesRequiredBefore(0), 109 CanBreakBefore(false), ClosesTemplateDeclaration(false), 110 ParameterCount(0), BlockParameterCount(0), 111 PackingKind(PPK_Inconclusive), TotalLength(0), UnbreakableTailLength(0), 112 BindingStrength(0), NestingLevel(0), SplitPenalty(0), 113 LongestObjCSelectorName(0), FakeRParens(0), 114 StartsBinaryExpression(false), EndsBinaryExpression(false), 115 OperatorIndex(0), LastOperator(false), 116 PartOfMultiVariableDeclStmt(false), IsForEachMacro(false), 117 MatchingParen(nullptr), Previous(nullptr), Next(nullptr), 118 Decision(FD_Unformatted), Finalized(false) {} 119 120 /// \brief The \c Token. 121 Token Tok; 122 123 /// \brief The number of newlines immediately before the \c Token. 124 /// 125 /// This can be used to determine what the user wrote in the original code 126 /// and thereby e.g. leave an empty line between two function definitions. 127 unsigned NewlinesBefore; 128 129 /// \brief Whether there is at least one unescaped newline before the \c 130 /// Token. 131 bool HasUnescapedNewline; 132 133 /// \brief The range of the whitespace immediately preceding the \c Token. 134 SourceRange WhitespaceRange; 135 136 /// \brief The offset just past the last '\n' in this token's leading 137 /// whitespace (relative to \c WhiteSpaceStart). 0 if there is no '\n'. 138 unsigned LastNewlineOffset; 139 140 /// \brief The width of the non-whitespace parts of the token (or its first 141 /// line for multi-line tokens) in columns. 142 /// We need this to correctly measure number of columns a token spans. 143 unsigned ColumnWidth; 144 145 /// \brief Contains the width in columns of the last line of a multi-line 146 /// token. 147 unsigned LastLineColumnWidth; 148 149 /// \brief Whether the token text contains newlines (escaped or not). 150 bool IsMultiline; 151 152 /// \brief Indicates that this is the first token. 153 bool IsFirst; 154 155 /// \brief Whether there must be a line break before this token. 156 /// 157 /// This happens for example when a preprocessor directive ended directly 158 /// before the token. 159 bool MustBreakBefore; 160 161 /// \brief Returns actual token start location without leading escaped 162 /// newlines and whitespace. 163 /// 164 /// This can be different to Tok.getLocation(), which includes leading escaped 165 /// newlines. 166 SourceLocation getStartOfNonWhitespace() const { 167 return WhitespaceRange.getEnd(); 168 } 169 170 /// \brief The raw text of the token. 171 /// 172 /// Contains the raw token text without leading whitespace and without leading 173 /// escaped newlines. 174 StringRef TokenText; 175 176 /// \brief Set to \c true if this token is an unterminated literal. 177 bool IsUnterminatedLiteral; 178 179 /// \brief Contains the kind of block if this token is a brace. 180 BraceBlockKind BlockKind; 181 182 TokenType Type; 183 184 /// \brief The number of spaces that should be inserted before this token. 185 unsigned SpacesRequiredBefore; 186 187 /// \brief \c true if it is allowed to break before this token. 188 bool CanBreakBefore; 189 190 bool ClosesTemplateDeclaration; 191 192 /// \brief Number of parameters, if this is "(", "[" or "<". 193 /// 194 /// This is initialized to 1 as we don't need to distinguish functions with 195 /// 0 parameters from functions with 1 parameter. Thus, we can simply count 196 /// the number of commas. 197 unsigned ParameterCount; 198 199 /// \brief Number of parameters that are nested blocks, 200 /// if this is "(", "[" or "<". 201 unsigned BlockParameterCount; 202 203 /// \brief A token can have a special role that can carry extra information 204 /// about the token's formatting. 205 std::unique_ptr<TokenRole> Role; 206 207 /// \brief If this is an opening parenthesis, how are the parameters packed? 208 ParameterPackingKind PackingKind; 209 210 /// \brief The total length of the unwrapped line up to and including this 211 /// token. 212 unsigned TotalLength; 213 214 /// \brief The original 0-based column of this token, including expanded tabs. 215 /// The configured TabWidth is used as tab width. 216 unsigned OriginalColumn; 217 218 /// \brief The length of following tokens until the next natural split point, 219 /// or the next token that can be broken. 220 unsigned UnbreakableTailLength; 221 222 // FIXME: Come up with a 'cleaner' concept. 223 /// \brief The binding strength of a token. This is a combined value of 224 /// operator precedence, parenthesis nesting, etc. 225 unsigned BindingStrength; 226 227 /// \brief The nesting level of this token, i.e. the number of surrounding (), 228 /// [], {} or <>. 229 unsigned NestingLevel; 230 231 /// \brief Penalty for inserting a line break before this token. 232 unsigned SplitPenalty; 233 234 /// \brief If this is the first ObjC selector name in an ObjC method 235 /// definition or call, this contains the length of the longest name. 236 /// 237 /// This being set to 0 means that the selectors should not be colon-aligned, 238 /// e.g. because several of them are block-type. 239 unsigned LongestObjCSelectorName; 240 241 /// \brief Stores the number of required fake parentheses and the 242 /// corresponding operator precedence. 243 /// 244 /// If multiple fake parentheses start at a token, this vector stores them in 245 /// reverse order, i.e. inner fake parenthesis first. 246 SmallVector<prec::Level, 4> FakeLParens; 247 /// \brief Insert this many fake ) after this token for correct indentation. 248 unsigned FakeRParens; 249 250 /// \brief \c true if this token starts a binary expression, i.e. has at least 251 /// one fake l_paren with a precedence greater than prec::Unknown. 252 bool StartsBinaryExpression; 253 /// \brief \c true if this token ends a binary expression. 254 bool EndsBinaryExpression; 255 256 /// \brief Is this is an operator (or "."/"->") in a sequence of operators 257 /// with the same precedence, contains the 0-based operator index. 258 unsigned OperatorIndex; 259 260 /// \brief Is this the last operator (or "."/"->") in a sequence of operators 261 /// with the same precedence? 262 bool LastOperator; 263 264 /// \brief Is this token part of a \c DeclStmt defining multiple variables? 265 /// 266 /// Only set if \c Type == \c TT_StartOfName. 267 bool PartOfMultiVariableDeclStmt; 268 269 /// \brief Is this a foreach macro? 270 bool IsForEachMacro; 271 272 bool is(tok::TokenKind Kind) const { return Tok.is(Kind); } 273 274 bool is(const IdentifierInfo *II) const { 275 return II && II == Tok.getIdentifierInfo(); 276 } 277 278 template <typename T> 279 bool isOneOf(T K1, T K2) const { 280 return is(K1) || is(K2); 281 } 282 283 template <typename T> 284 bool isOneOf(T K1, T K2, T K3) const { 285 return is(K1) || is(K2) || is(K3); 286 } 287 288 template <typename T> 289 bool isOneOf(T K1, T K2, T K3, T K4, T K5 = tok::NUM_TOKENS, 290 T K6 = tok::NUM_TOKENS, T K7 = tok::NUM_TOKENS, 291 T K8 = tok::NUM_TOKENS, T K9 = tok::NUM_TOKENS, 292 T K10 = tok::NUM_TOKENS, T K11 = tok::NUM_TOKENS, 293 T K12 = tok::NUM_TOKENS) const { 294 return is(K1) || is(K2) || is(K3) || is(K4) || is(K5) || is(K6) || is(K7) || 295 is(K8) || is(K9) || is(K10) || is(K11) || is(K12); 296 } 297 298 template <typename T> 299 bool isNot(T Kind) const { 300 return Tok.isNot(Kind); 301 } 302 bool isNot(IdentifierInfo *II) const { return II != Tok.getIdentifierInfo(); } 303 304 bool isStringLiteral() const { return tok::isStringLiteral(Tok.getKind()); } 305 306 bool isObjCAtKeyword(tok::ObjCKeywordKind Kind) const { 307 return Tok.isObjCAtKeyword(Kind); 308 } 309 310 bool isAccessSpecifier(bool ColonRequired = true) const { 311 return isOneOf(tok::kw_public, tok::kw_protected, tok::kw_private) && 312 (!ColonRequired || (Next && Next->is(tok::colon))); 313 } 314 315 /// \brief Determine whether the token is a simple-type-specifier. 316 bool isSimpleTypeSpecifier() const; 317 318 bool isObjCAccessSpecifier() const { 319 return is(tok::at) && Next && (Next->isObjCAtKeyword(tok::objc_public) || 320 Next->isObjCAtKeyword(tok::objc_protected) || 321 Next->isObjCAtKeyword(tok::objc_package) || 322 Next->isObjCAtKeyword(tok::objc_private)); 323 } 324 325 /// \brief Returns whether \p Tok is ([{ or a template opening <. 326 bool opensScope() const { 327 return isOneOf(tok::l_paren, tok::l_brace, tok::l_square) || 328 Type == TT_TemplateOpener; 329 } 330 /// \brief Returns whether \p Tok is )]} or a template closing >. 331 bool closesScope() const { 332 return isOneOf(tok::r_paren, tok::r_brace, tok::r_square) || 333 Type == TT_TemplateCloser; 334 } 335 336 /// \brief Returns \c true if this is a "." or "->" accessing a member. 337 bool isMemberAccess() const { 338 return isOneOf(tok::arrow, tok::period, tok::arrowstar) && 339 Type != TT_DesignatedInitializerPeriod && 340 Type != TT_TrailingReturnArrow; 341 } 342 343 bool isUnaryOperator() const { 344 switch (Tok.getKind()) { 345 case tok::plus: 346 case tok::plusplus: 347 case tok::minus: 348 case tok::minusminus: 349 case tok::exclaim: 350 case tok::tilde: 351 case tok::kw_sizeof: 352 case tok::kw_alignof: 353 return true; 354 default: 355 return false; 356 } 357 } 358 359 bool isBinaryOperator() const { 360 // Comma is a binary operator, but does not behave as such wrt. formatting. 361 return getPrecedence() > prec::Comma; 362 } 363 364 bool isTrailingComment() const { 365 return is(tok::comment) && 366 (Type == TT_LineComment || !Next || Next->NewlinesBefore > 0); 367 } 368 369 /// \brief Returns \c true if this is a keyword that can be used 370 /// like a function call (e.g. sizeof, typeid, ...). 371 bool isFunctionLikeKeyword() const { 372 switch (Tok.getKind()) { 373 case tok::kw_throw: 374 case tok::kw_typeid: 375 case tok::kw_return: 376 case tok::kw_sizeof: 377 case tok::kw_alignof: 378 case tok::kw_alignas: 379 case tok::kw_decltype: 380 case tok::kw_noexcept: 381 case tok::kw_static_assert: 382 case tok::kw___attribute: 383 return true; 384 default: 385 return false; 386 } 387 } 388 389 prec::Level getPrecedence() const { 390 return getBinOpPrecedence(Tok.getKind(), true, true); 391 } 392 393 /// \brief Returns the previous token ignoring comments. 394 FormatToken *getPreviousNonComment() const { 395 FormatToken *Tok = Previous; 396 while (Tok && Tok->is(tok::comment)) 397 Tok = Tok->Previous; 398 return Tok; 399 } 400 401 /// \brief Returns the next token ignoring comments. 402 const FormatToken *getNextNonComment() const { 403 const FormatToken *Tok = Next; 404 while (Tok && Tok->is(tok::comment)) 405 Tok = Tok->Next; 406 return Tok; 407 } 408 409 /// \brief Returns \c true if this tokens starts a block-type list, i.e. a 410 /// list that should be indented with a block indent. 411 bool opensBlockTypeList(const FormatStyle &Style) const { 412 return Type == TT_ArrayInitializerLSquare || 413 (is(tok::l_brace) && 414 (BlockKind == BK_Block || Type == TT_DictLiteral || 415 !Style.Cpp11BracedListStyle)); 416 } 417 418 /// \brief Same as opensBlockTypeList, but for the closing token. 419 bool closesBlockTypeList(const FormatStyle &Style) const { 420 return MatchingParen && MatchingParen->opensBlockTypeList(Style); 421 } 422 423 FormatToken *MatchingParen; 424 425 FormatToken *Previous; 426 FormatToken *Next; 427 428 SmallVector<AnnotatedLine *, 1> Children; 429 430 /// \brief Stores the formatting decision for the token once it was made. 431 FormatDecision Decision; 432 433 /// \brief If \c true, this token has been fully formatted (indented and 434 /// potentially re-formatted inside), and we do not allow further formatting 435 /// changes. 436 bool Finalized; 437 438 private: 439 // Disallow copying. 440 FormatToken(const FormatToken &) LLVM_DELETED_FUNCTION; 441 void operator=(const FormatToken &) LLVM_DELETED_FUNCTION; 442 }; 443 444 class ContinuationIndenter; 445 struct LineState; 446 447 class TokenRole { 448 public: 449 TokenRole(const FormatStyle &Style) : Style(Style) {} 450 virtual ~TokenRole(); 451 452 /// \brief After the \c TokenAnnotator has finished annotating all the tokens, 453 /// this function precomputes required information for formatting. 454 virtual void precomputeFormattingInfos(const FormatToken *Token); 455 456 /// \brief Apply the special formatting that the given role demands. 457 /// 458 /// Assumes that the token having this role is already formatted. 459 /// 460 /// Continues formatting from \p State leaving indentation to \p Indenter and 461 /// returns the total penalty that this formatting incurs. 462 virtual unsigned formatFromToken(LineState &State, 463 ContinuationIndenter *Indenter, 464 bool DryRun) { 465 return 0; 466 } 467 468 /// \brief Same as \c formatFromToken, but assumes that the first token has 469 /// already been set thereby deciding on the first line break. 470 virtual unsigned formatAfterToken(LineState &State, 471 ContinuationIndenter *Indenter, 472 bool DryRun) { 473 return 0; 474 } 475 476 /// \brief Notifies the \c Role that a comma was found. 477 virtual void CommaFound(const FormatToken *Token) {} 478 479 protected: 480 const FormatStyle &Style; 481 }; 482 483 class CommaSeparatedList : public TokenRole { 484 public: 485 CommaSeparatedList(const FormatStyle &Style) 486 : TokenRole(Style), HasNestedBracedList(false) {} 487 488 void precomputeFormattingInfos(const FormatToken *Token) override; 489 490 unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, 491 bool DryRun) override; 492 493 unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, 494 bool DryRun) override; 495 496 /// \brief Adds \p Token as the next comma to the \c CommaSeparated list. 497 void CommaFound(const FormatToken *Token) override { 498 Commas.push_back(Token); 499 } 500 501 private: 502 /// \brief A struct that holds information on how to format a given list with 503 /// a specific number of columns. 504 struct ColumnFormat { 505 /// \brief The number of columns to use. 506 unsigned Columns; 507 508 /// \brief The total width in characters. 509 unsigned TotalWidth; 510 511 /// \brief The number of lines required for this format. 512 unsigned LineCount; 513 514 /// \brief The size of each column in characters. 515 SmallVector<unsigned, 8> ColumnSizes; 516 }; 517 518 /// \brief Calculate which \c ColumnFormat fits best into 519 /// \p RemainingCharacters. 520 const ColumnFormat *getColumnFormat(unsigned RemainingCharacters) const; 521 522 /// \brief The ordered \c FormatTokens making up the commas of this list. 523 SmallVector<const FormatToken *, 8> Commas; 524 525 /// \brief The length of each of the list's items in characters including the 526 /// trailing comma. 527 SmallVector<unsigned, 8> ItemLengths; 528 529 /// \brief Precomputed formats that can be used for this list. 530 SmallVector<ColumnFormat, 4> Formats; 531 532 bool HasNestedBracedList; 533 }; 534 535 /// \brief Encapsulates keywords that are context sensitive or for languages not 536 /// properly supported by Clang's lexer. 537 struct AdditionalKeywords { 538 AdditionalKeywords(IdentifierTable &IdentTable) { 539 kw_in = &IdentTable.get("in"); 540 kw_NS_ENUM = &IdentTable.get("NS_ENUM"); 541 542 kw_finally = &IdentTable.get("finally"); 543 kw_function = &IdentTable.get("function"); 544 kw_var = &IdentTable.get("var"); 545 546 kw_extends = &IdentTable.get("extends"); 547 kw_implements = &IdentTable.get("implements"); 548 kw_interface = &IdentTable.get("interface"); 549 kw_synchronized = &IdentTable.get("synchronized"); 550 kw_throws = &IdentTable.get("throws"); 551 552 kw_option = &IdentTable.get("option"); 553 kw_optional = &IdentTable.get("optional"); 554 kw_repeated = &IdentTable.get("repeated"); 555 kw_required = &IdentTable.get("required"); 556 kw_returns = &IdentTable.get("returns"); 557 } 558 559 // ObjC context sensitive keywords. 560 IdentifierInfo *kw_in; 561 IdentifierInfo *kw_NS_ENUM; 562 563 // JavaScript keywords. 564 IdentifierInfo *kw_finally; 565 IdentifierInfo *kw_function; 566 IdentifierInfo *kw_var; 567 568 // Java keywords. 569 IdentifierInfo *kw_extends; 570 IdentifierInfo *kw_implements; 571 IdentifierInfo *kw_interface; 572 IdentifierInfo *kw_synchronized; 573 IdentifierInfo *kw_throws; 574 575 // Proto keywords. 576 IdentifierInfo *kw_option; 577 IdentifierInfo *kw_optional; 578 IdentifierInfo *kw_repeated; 579 IdentifierInfo *kw_required; 580 IdentifierInfo *kw_returns; 581 }; 582 583 } // namespace format 584 } // namespace clang 585 586 #endif 587