1 //===--- FormatToken.h - Format C++ code ------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// 9 /// \file 10 /// This file contains the declaration of the FormatToken, a wrapper 11 /// around Token with additional information related to formatting. 12 /// 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H 16 #define LLVM_CLANG_LIB_FORMAT_FORMATTOKEN_H 17 18 #include "clang/Basic/IdentifierTable.h" 19 #include "clang/Basic/OperatorPrecedence.h" 20 #include "clang/Format/Format.h" 21 #include "clang/Lex/Lexer.h" 22 #include <memory> 23 #include <unordered_set> 24 25 namespace clang { 26 namespace format { 27 28 #define LIST_TOKEN_TYPES \ 29 TYPE(ArrayInitializerLSquare) \ 30 TYPE(ArraySubscriptLSquare) \ 31 TYPE(AttributeColon) \ 32 TYPE(AttributeMacro) \ 33 TYPE(AttributeParen) \ 34 TYPE(AttributeSquare) \ 35 TYPE(BinaryOperator) \ 36 TYPE(BitFieldColon) \ 37 TYPE(BlockComment) \ 38 TYPE(CastRParen) \ 39 TYPE(ConditionalExpr) \ 40 TYPE(ConflictAlternative) \ 41 TYPE(ConflictEnd) \ 42 TYPE(ConflictStart) \ 43 TYPE(ConstraintJunctions) \ 44 TYPE(CtorInitializerColon) \ 45 TYPE(CtorInitializerComma) \ 46 TYPE(DesignatedInitializerLSquare) \ 47 TYPE(DesignatedInitializerPeriod) \ 48 TYPE(DictLiteral) \ 49 TYPE(FatArrow) \ 50 TYPE(ForEachMacro) \ 51 TYPE(FunctionAnnotationRParen) \ 52 TYPE(FunctionDeclarationName) \ 53 TYPE(FunctionLBrace) \ 54 TYPE(FunctionTypeLParen) \ 55 TYPE(IfMacro) \ 56 TYPE(ImplicitStringLiteral) \ 57 TYPE(InheritanceColon) \ 58 TYPE(InheritanceComma) \ 59 TYPE(InlineASMBrace) \ 60 TYPE(InlineASMColon) \ 61 TYPE(InlineASMSymbolicNameLSquare) \ 62 TYPE(JavaAnnotation) \ 63 TYPE(JsComputedPropertyName) \ 64 TYPE(JsExponentiation) \ 65 TYPE(JsExponentiationEqual) \ 66 TYPE(JsPipePipeEqual) \ 67 TYPE(JsPrivateIdentifier) \ 68 TYPE(JsTypeColon) \ 69 TYPE(JsTypeOperator) \ 70 TYPE(JsTypeOptionalQuestion) \ 71 TYPE(JsAndAndEqual) \ 72 TYPE(LambdaArrow) \ 73 TYPE(LambdaLBrace) \ 74 TYPE(LambdaLSquare) \ 75 TYPE(LeadingJavaAnnotation) \ 76 TYPE(LineComment) \ 77 TYPE(MacroBlockBegin) \ 78 TYPE(MacroBlockEnd) \ 79 TYPE(ModulePartitionColon) \ 80 TYPE(NamespaceMacro) \ 81 TYPE(NonNullAssertion) \ 82 TYPE(NullCoalescingEqual) \ 83 TYPE(NullCoalescingOperator) \ 84 TYPE(NullPropagatingOperator) \ 85 TYPE(ObjCBlockLBrace) \ 86 TYPE(ObjCBlockLParen) \ 87 TYPE(ObjCDecl) \ 88 TYPE(ObjCForIn) \ 89 TYPE(ObjCMethodExpr) \ 90 TYPE(ObjCMethodSpecifier) \ 91 TYPE(ObjCProperty) \ 92 TYPE(ObjCStringLiteral) \ 93 TYPE(OverloadedOperator) \ 94 TYPE(OverloadedOperatorLParen) \ 95 TYPE(PointerOrReference) \ 96 TYPE(PureVirtualSpecifier) \ 97 TYPE(RangeBasedForLoopColon) \ 98 TYPE(RegexLiteral) \ 99 TYPE(SelectorName) \ 100 TYPE(StartOfName) \ 101 TYPE(StatementAttributeLikeMacro) \ 102 TYPE(StatementMacro) \ 103 TYPE(StructuredBindingLSquare) \ 104 TYPE(TemplateCloser) \ 105 TYPE(TemplateOpener) \ 106 TYPE(TemplateString) \ 107 TYPE(ProtoExtensionLSquare) \ 108 TYPE(TrailingAnnotation) \ 109 TYPE(TrailingReturnArrow) \ 110 TYPE(TrailingUnaryOperator) \ 111 TYPE(TypeDeclarationParen) \ 112 TYPE(TypenameMacro) \ 113 TYPE(UnaryOperator) \ 114 TYPE(UntouchableMacroFunc) \ 115 TYPE(CSharpStringLiteral) \ 116 TYPE(CSharpNamedArgumentColon) \ 117 TYPE(CSharpNullable) \ 118 TYPE(CSharpNullConditionalLSquare) \ 119 TYPE(CSharpGenericTypeConstraint) \ 120 TYPE(CSharpGenericTypeConstraintColon) \ 121 TYPE(CSharpGenericTypeConstraintComma) \ 122 TYPE(Unknown) 123 124 /// Determines the semantic type of a syntactic token, e.g. whether "<" is a 125 /// template opener or binary operator. 126 enum TokenType : uint8_t { 127 #define TYPE(X) TT_##X, 128 LIST_TOKEN_TYPES 129 #undef TYPE 130 NUM_TOKEN_TYPES 131 }; 132 133 /// Determines the name of a token type. 134 const char *getTokenTypeName(TokenType Type); 135 136 // Represents what type of block a set of braces open. 137 enum BraceBlockKind { BK_Unknown, BK_Block, BK_BracedInit }; 138 139 // The packing kind of a function's parameters. 140 enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive }; 141 142 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break }; 143 144 /// Roles a token can take in a configured macro expansion. 145 enum MacroRole { 146 /// The token was expanded from a macro argument when formatting the expanded 147 /// token sequence. 148 MR_ExpandedArg, 149 /// The token is part of a macro argument that was previously formatted as 150 /// expansion when formatting the unexpanded macro call. 151 MR_UnexpandedArg, 152 /// The token was expanded from a macro definition, and is not visible as part 153 /// of the macro call. 154 MR_Hidden, 155 }; 156 157 struct FormatToken; 158 159 /// Contains information on the token's role in a macro expansion. 160 /// 161 /// Given the following definitions: 162 /// A(X) = [ X ] 163 /// B(X) = < X > 164 /// C(X) = X 165 /// 166 /// Consider the macro call: 167 /// A({B(C(C(x)))}) -> [{<x>}] 168 /// 169 /// In this case, the tokens of the unexpanded macro call will have the 170 /// following relevant entries in their macro context (note that formatting 171 /// the unexpanded macro call happens *after* formatting the expanded macro 172 /// call): 173 /// A( { B( C( C(x) ) ) } ) 174 /// Role: NN U NN NN NNUN N N U N (N=None, U=UnexpandedArg) 175 /// 176 /// [ { < x > } ] 177 /// Role: H E H E H E H (H=Hidden, E=ExpandedArg) 178 /// ExpandedFrom[0]: A A A A A A A 179 /// ExpandedFrom[1]: B B B 180 /// ExpandedFrom[2]: C 181 /// ExpandedFrom[3]: C 182 /// StartOfExpansion: 1 0 1 2 0 0 0 183 /// EndOfExpansion: 0 0 0 2 1 0 1 184 struct MacroExpansion { 185 MacroExpansion(MacroRole Role) : Role(Role) {} 186 187 /// The token's role in the macro expansion. 188 /// When formatting an expanded macro, all tokens that are part of macro 189 /// arguments will be MR_ExpandedArg, while all tokens that are not visible in 190 /// the macro call will be MR_Hidden. 191 /// When formatting an unexpanded macro call, all tokens that are part of 192 /// macro arguments will be MR_UnexpandedArg. 193 MacroRole Role; 194 195 /// The stack of macro call identifier tokens this token was expanded from. 196 llvm::SmallVector<FormatToken *, 1> ExpandedFrom; 197 198 /// The number of expansions of which this macro is the first entry. 199 unsigned StartOfExpansion = 0; 200 201 /// The number of currently open expansions in \c ExpandedFrom this macro is 202 /// the last token in. 203 unsigned EndOfExpansion = 0; 204 }; 205 206 class TokenRole; 207 class AnnotatedLine; 208 209 /// A wrapper around a \c Token storing information about the 210 /// whitespace characters preceding it. 211 struct FormatToken { 212 FormatToken() 213 : HasUnescapedNewline(false), IsMultiline(false), IsFirst(false), 214 MustBreakBefore(false), IsUnterminatedLiteral(false), 215 CanBreakBefore(false), ClosesTemplateDeclaration(false), 216 StartsBinaryExpression(false), EndsBinaryExpression(false), 217 PartOfMultiVariableDeclStmt(false), ContinuesLineCommentSection(false), 218 Finalized(false), BlockKind(BK_Unknown), Decision(FD_Unformatted), 219 PackingKind(PPK_Inconclusive), Type(TT_Unknown) {} 220 221 /// The \c Token. 222 Token Tok; 223 224 /// The raw text of the token. 225 /// 226 /// Contains the raw token text without leading whitespace and without leading 227 /// escaped newlines. 228 StringRef TokenText; 229 230 /// A token can have a special role that can carry extra information 231 /// about the token's formatting. 232 /// FIXME: Make FormatToken for parsing and AnnotatedToken two different 233 /// classes and make this a unique_ptr in the AnnotatedToken class. 234 std::shared_ptr<TokenRole> Role; 235 236 /// The range of the whitespace immediately preceding the \c Token. 237 SourceRange WhitespaceRange; 238 239 /// Whether there is at least one unescaped newline before the \c 240 /// Token. 241 unsigned HasUnescapedNewline : 1; 242 243 /// Whether the token text contains newlines (escaped or not). 244 unsigned IsMultiline : 1; 245 246 /// Indicates that this is the first token of the file. 247 unsigned IsFirst : 1; 248 249 /// Whether there must be a line break before this token. 250 /// 251 /// This happens for example when a preprocessor directive ended directly 252 /// before the token. 253 unsigned MustBreakBefore : 1; 254 255 /// Set to \c true if this token is an unterminated literal. 256 unsigned IsUnterminatedLiteral : 1; 257 258 /// \c true if it is allowed to break before this token. 259 unsigned CanBreakBefore : 1; 260 261 /// \c true if this is the ">" of "template<..>". 262 unsigned ClosesTemplateDeclaration : 1; 263 264 /// \c true if this token starts a binary expression, i.e. has at least 265 /// one fake l_paren with a precedence greater than prec::Unknown. 266 unsigned StartsBinaryExpression : 1; 267 /// \c true if this token ends a binary expression. 268 unsigned EndsBinaryExpression : 1; 269 270 /// Is this token part of a \c DeclStmt defining multiple variables? 271 /// 272 /// Only set if \c Type == \c TT_StartOfName. 273 unsigned PartOfMultiVariableDeclStmt : 1; 274 275 /// Does this line comment continue a line comment section? 276 /// 277 /// Only set to true if \c Type == \c TT_LineComment. 278 unsigned ContinuesLineCommentSection : 1; 279 280 /// If \c true, this token has been fully formatted (indented and 281 /// potentially re-formatted inside), and we do not allow further formatting 282 /// changes. 283 unsigned Finalized : 1; 284 285 private: 286 /// Contains the kind of block if this token is a brace. 287 unsigned BlockKind : 2; 288 289 public: 290 BraceBlockKind getBlockKind() const { 291 return static_cast<BraceBlockKind>(BlockKind); 292 } 293 void setBlockKind(BraceBlockKind BBK) { 294 BlockKind = BBK; 295 assert(getBlockKind() == BBK && "BraceBlockKind overflow!"); 296 } 297 298 private: 299 /// Stores the formatting decision for the token once it was made. 300 unsigned Decision : 2; 301 302 public: 303 FormatDecision getDecision() const { 304 return static_cast<FormatDecision>(Decision); 305 } 306 void setDecision(FormatDecision D) { 307 Decision = D; 308 assert(getDecision() == D && "FormatDecision overflow!"); 309 } 310 311 private: 312 /// If this is an opening parenthesis, how are the parameters packed? 313 unsigned PackingKind : 2; 314 315 public: 316 ParameterPackingKind getPackingKind() const { 317 return static_cast<ParameterPackingKind>(PackingKind); 318 } 319 void setPackingKind(ParameterPackingKind K) { 320 PackingKind = K; 321 assert(getPackingKind() == K && "ParameterPackingKind overflow!"); 322 } 323 324 private: 325 TokenType Type; 326 327 public: 328 /// Returns the token's type, e.g. whether "<" is a template opener or 329 /// binary operator. 330 TokenType getType() const { return Type; } 331 void setType(TokenType T) { Type = T; } 332 333 /// The number of newlines immediately before the \c Token. 334 /// 335 /// This can be used to determine what the user wrote in the original code 336 /// and thereby e.g. leave an empty line between two function definitions. 337 unsigned NewlinesBefore = 0; 338 339 /// The offset just past the last '\n' in this token's leading 340 /// whitespace (relative to \c WhiteSpaceStart). 0 if there is no '\n'. 341 unsigned LastNewlineOffset = 0; 342 343 /// The width of the non-whitespace parts of the token (or its first 344 /// line for multi-line tokens) in columns. 345 /// We need this to correctly measure number of columns a token spans. 346 unsigned ColumnWidth = 0; 347 348 /// Contains the width in columns of the last line of a multi-line 349 /// token. 350 unsigned LastLineColumnWidth = 0; 351 352 /// The number of spaces that should be inserted before this token. 353 unsigned SpacesRequiredBefore = 0; 354 355 /// Number of parameters, if this is "(", "[" or "<". 356 unsigned ParameterCount = 0; 357 358 /// Number of parameters that are nested blocks, 359 /// if this is "(", "[" or "<". 360 unsigned BlockParameterCount = 0; 361 362 /// If this is a bracket ("<", "(", "[" or "{"), contains the kind of 363 /// the surrounding bracket. 364 tok::TokenKind ParentBracket = tok::unknown; 365 366 /// The total length of the unwrapped line up to and including this 367 /// token. 368 unsigned TotalLength = 0; 369 370 /// The original 0-based column of this token, including expanded tabs. 371 /// The configured TabWidth is used as tab width. 372 unsigned OriginalColumn = 0; 373 374 /// The length of following tokens until the next natural split point, 375 /// or the next token that can be broken. 376 unsigned UnbreakableTailLength = 0; 377 378 // FIXME: Come up with a 'cleaner' concept. 379 /// The binding strength of a token. This is a combined value of 380 /// operator precedence, parenthesis nesting, etc. 381 unsigned BindingStrength = 0; 382 383 /// The nesting level of this token, i.e. the number of surrounding (), 384 /// [], {} or <>. 385 unsigned NestingLevel = 0; 386 387 /// The indent level of this token. Copied from the surrounding line. 388 unsigned IndentLevel = 0; 389 390 /// Penalty for inserting a line break before this token. 391 unsigned SplitPenalty = 0; 392 393 /// If this is the first ObjC selector name in an ObjC method 394 /// definition or call, this contains the length of the longest name. 395 /// 396 /// This being set to 0 means that the selectors should not be colon-aligned, 397 /// e.g. because several of them are block-type. 398 unsigned LongestObjCSelectorName = 0; 399 400 /// If this is the first ObjC selector name in an ObjC method 401 /// definition or call, this contains the number of parts that the whole 402 /// selector consist of. 403 unsigned ObjCSelectorNameParts = 0; 404 405 /// The 0-based index of the parameter/argument. For ObjC it is set 406 /// for the selector name token. 407 /// For now calculated only for ObjC. 408 unsigned ParameterIndex = 0; 409 410 /// Stores the number of required fake parentheses and the 411 /// corresponding operator precedence. 412 /// 413 /// If multiple fake parentheses start at a token, this vector stores them in 414 /// reverse order, i.e. inner fake parenthesis first. 415 SmallVector<prec::Level, 4> FakeLParens; 416 /// Insert this many fake ) after this token for correct indentation. 417 unsigned FakeRParens = 0; 418 419 /// If this is an operator (or "."/"->") in a sequence of operators 420 /// with the same precedence, contains the 0-based operator index. 421 unsigned OperatorIndex = 0; 422 423 /// If this is an operator (or "."/"->") in a sequence of operators 424 /// with the same precedence, points to the next operator. 425 FormatToken *NextOperator = nullptr; 426 427 /// If this is a bracket, this points to the matching one. 428 FormatToken *MatchingParen = nullptr; 429 430 /// The previous token in the unwrapped line. 431 FormatToken *Previous = nullptr; 432 433 /// The next token in the unwrapped line. 434 FormatToken *Next = nullptr; 435 436 /// The first token in set of column elements. 437 bool StartsColumn = false; 438 439 /// This notes the start of the line of an array initializer. 440 bool ArrayInitializerLineStart = false; 441 442 /// This starts an array initializer. 443 bool IsArrayInitializer = false; 444 445 /// Is optional and can be removed. 446 bool Optional = false; 447 448 /// If this token starts a block, this contains all the unwrapped lines 449 /// in it. 450 SmallVector<AnnotatedLine *, 1> Children; 451 452 // Contains all attributes related to how this token takes part 453 // in a configured macro expansion. 454 llvm::Optional<MacroExpansion> MacroCtx; 455 456 bool is(tok::TokenKind Kind) const { return Tok.is(Kind); } 457 bool is(TokenType TT) const { return getType() == TT; } 458 bool is(const IdentifierInfo *II) const { 459 return II && II == Tok.getIdentifierInfo(); 460 } 461 bool is(tok::PPKeywordKind Kind) const { 462 return Tok.getIdentifierInfo() && 463 Tok.getIdentifierInfo()->getPPKeywordID() == Kind; 464 } 465 bool is(BraceBlockKind BBK) const { return getBlockKind() == BBK; } 466 bool is(ParameterPackingKind PPK) const { return getPackingKind() == PPK; } 467 468 template <typename A, typename B> bool isOneOf(A K1, B K2) const { 469 return is(K1) || is(K2); 470 } 471 template <typename A, typename B, typename... Ts> 472 bool isOneOf(A K1, B K2, Ts... Ks) const { 473 return is(K1) || isOneOf(K2, Ks...); 474 } 475 template <typename T> bool isNot(T Kind) const { return !is(Kind); } 476 477 bool isIf(bool AllowConstexprMacro = true) const { 478 return is(tok::kw_if) || endsSequence(tok::kw_constexpr, tok::kw_if) || 479 (endsSequence(tok::identifier, tok::kw_if) && AllowConstexprMacro); 480 } 481 482 bool closesScopeAfterBlock() const { 483 if (getBlockKind() == BK_Block) 484 return true; 485 if (closesScope()) 486 return Previous->closesScopeAfterBlock(); 487 return false; 488 } 489 490 /// \c true if this token starts a sequence with the given tokens in order, 491 /// following the ``Next`` pointers, ignoring comments. 492 template <typename A, typename... Ts> 493 bool startsSequence(A K1, Ts... Tokens) const { 494 return startsSequenceInternal(K1, Tokens...); 495 } 496 497 /// \c true if this token ends a sequence with the given tokens in order, 498 /// following the ``Previous`` pointers, ignoring comments. 499 /// For example, given tokens [T1, T2, T3], the function returns true if 500 /// 3 tokens ending at this (ignoring comments) are [T3, T2, T1]. In other 501 /// words, the tokens passed to this function need to the reverse of the 502 /// order the tokens appear in code. 503 template <typename A, typename... Ts> 504 bool endsSequence(A K1, Ts... Tokens) const { 505 return endsSequenceInternal(K1, Tokens...); 506 } 507 508 bool isStringLiteral() const { return tok::isStringLiteral(Tok.getKind()); } 509 510 bool isObjCAtKeyword(tok::ObjCKeywordKind Kind) const { 511 return Tok.isObjCAtKeyword(Kind); 512 } 513 514 bool isAccessSpecifier(bool ColonRequired = true) const { 515 return isOneOf(tok::kw_public, tok::kw_protected, tok::kw_private) && 516 (!ColonRequired || (Next && Next->is(tok::colon))); 517 } 518 519 bool canBePointerOrReferenceQualifier() const { 520 return isOneOf(tok::kw_const, tok::kw_restrict, tok::kw_volatile, 521 tok::kw___attribute, tok::kw__Nonnull, tok::kw__Nullable, 522 tok::kw__Null_unspecified, tok::kw___ptr32, tok::kw___ptr64, 523 TT_AttributeMacro); 524 } 525 526 /// Determine whether the token is a simple-type-specifier. 527 LLVM_NODISCARD bool isSimpleTypeSpecifier() const; 528 529 LLVM_NODISCARD bool isTypeOrIdentifier() const; 530 531 bool isObjCAccessSpecifier() const { 532 return is(tok::at) && Next && 533 (Next->isObjCAtKeyword(tok::objc_public) || 534 Next->isObjCAtKeyword(tok::objc_protected) || 535 Next->isObjCAtKeyword(tok::objc_package) || 536 Next->isObjCAtKeyword(tok::objc_private)); 537 } 538 539 /// Returns whether \p Tok is ([{ or an opening < of a template or in 540 /// protos. 541 bool opensScope() const { 542 if (is(TT_TemplateString) && TokenText.endswith("${")) 543 return true; 544 if (is(TT_DictLiteral) && is(tok::less)) 545 return true; 546 return isOneOf(tok::l_paren, tok::l_brace, tok::l_square, 547 TT_TemplateOpener); 548 } 549 /// Returns whether \p Tok is )]} or a closing > of a template or in 550 /// protos. 551 bool closesScope() const { 552 if (is(TT_TemplateString) && TokenText.startswith("}")) 553 return true; 554 if (is(TT_DictLiteral) && is(tok::greater)) 555 return true; 556 return isOneOf(tok::r_paren, tok::r_brace, tok::r_square, 557 TT_TemplateCloser); 558 } 559 560 /// Returns \c true if this is a "." or "->" accessing a member. 561 bool isMemberAccess() const { 562 return isOneOf(tok::arrow, tok::period, tok::arrowstar) && 563 !isOneOf(TT_DesignatedInitializerPeriod, TT_TrailingReturnArrow, 564 TT_LambdaArrow, TT_LeadingJavaAnnotation); 565 } 566 567 bool isUnaryOperator() const { 568 switch (Tok.getKind()) { 569 case tok::plus: 570 case tok::plusplus: 571 case tok::minus: 572 case tok::minusminus: 573 case tok::exclaim: 574 case tok::tilde: 575 case tok::kw_sizeof: 576 case tok::kw_alignof: 577 return true; 578 default: 579 return false; 580 } 581 } 582 583 bool isBinaryOperator() const { 584 // Comma is a binary operator, but does not behave as such wrt. formatting. 585 return getPrecedence() > prec::Comma; 586 } 587 588 bool isTrailingComment() const { 589 return is(tok::comment) && 590 (is(TT_LineComment) || !Next || Next->NewlinesBefore > 0); 591 } 592 593 /// Returns \c true if this is a keyword that can be used 594 /// like a function call (e.g. sizeof, typeid, ...). 595 bool isFunctionLikeKeyword() const { 596 switch (Tok.getKind()) { 597 case tok::kw_throw: 598 case tok::kw_typeid: 599 case tok::kw_return: 600 case tok::kw_sizeof: 601 case tok::kw_alignof: 602 case tok::kw_alignas: 603 case tok::kw_decltype: 604 case tok::kw_noexcept: 605 case tok::kw_static_assert: 606 case tok::kw__Atomic: 607 case tok::kw___attribute: 608 case tok::kw___underlying_type: 609 case tok::kw_requires: 610 return true; 611 default: 612 return false; 613 } 614 } 615 616 /// Returns \c true if this is a string literal that's like a label, 617 /// e.g. ends with "=" or ":". 618 bool isLabelString() const { 619 if (!is(tok::string_literal)) 620 return false; 621 StringRef Content = TokenText; 622 if (Content.startswith("\"") || Content.startswith("'")) 623 Content = Content.drop_front(1); 624 if (Content.endswith("\"") || Content.endswith("'")) 625 Content = Content.drop_back(1); 626 Content = Content.trim(); 627 return Content.size() > 1 && 628 (Content.back() == ':' || Content.back() == '='); 629 } 630 631 /// Returns actual token start location without leading escaped 632 /// newlines and whitespace. 633 /// 634 /// This can be different to Tok.getLocation(), which includes leading escaped 635 /// newlines. 636 SourceLocation getStartOfNonWhitespace() const { 637 return WhitespaceRange.getEnd(); 638 } 639 640 /// Returns \c true if the range of whitespace immediately preceding the \c 641 /// Token is not empty. 642 bool hasWhitespaceBefore() const { 643 return WhitespaceRange.getBegin() != WhitespaceRange.getEnd(); 644 } 645 646 prec::Level getPrecedence() const { 647 return getBinOpPrecedence(Tok.getKind(), /*GreaterThanIsOperator=*/true, 648 /*CPlusPlus11=*/true); 649 } 650 651 /// Returns the previous token ignoring comments. 652 FormatToken *getPreviousNonComment() const { 653 FormatToken *Tok = Previous; 654 while (Tok && Tok->is(tok::comment)) 655 Tok = Tok->Previous; 656 return Tok; 657 } 658 659 /// Returns the next token ignoring comments. 660 const FormatToken *getNextNonComment() const { 661 const FormatToken *Tok = Next; 662 while (Tok && Tok->is(tok::comment)) 663 Tok = Tok->Next; 664 return Tok; 665 } 666 667 /// Returns \c true if this tokens starts a block-type list, i.e. a 668 /// list that should be indented with a block indent. 669 bool opensBlockOrBlockTypeList(const FormatStyle &Style) const { 670 // C# Does not indent object initialisers as continuations. 671 if (is(tok::l_brace) && getBlockKind() == BK_BracedInit && Style.isCSharp()) 672 return true; 673 if (is(TT_TemplateString) && opensScope()) 674 return true; 675 return is(TT_ArrayInitializerLSquare) || is(TT_ProtoExtensionLSquare) || 676 (is(tok::l_brace) && 677 (getBlockKind() == BK_Block || is(TT_DictLiteral) || 678 (!Style.Cpp11BracedListStyle && NestingLevel == 0))) || 679 (is(tok::less) && (Style.Language == FormatStyle::LK_Proto || 680 Style.Language == FormatStyle::LK_TextProto)); 681 } 682 683 /// Returns whether the token is the left square bracket of a C++ 684 /// structured binding declaration. 685 bool isCppStructuredBinding(const FormatStyle &Style) const { 686 if (!Style.isCpp() || isNot(tok::l_square)) 687 return false; 688 const FormatToken *T = this; 689 do { 690 T = T->getPreviousNonComment(); 691 } while (T && T->isOneOf(tok::kw_const, tok::kw_volatile, tok::amp, 692 tok::ampamp)); 693 return T && T->is(tok::kw_auto); 694 } 695 696 /// Same as opensBlockOrBlockTypeList, but for the closing token. 697 bool closesBlockOrBlockTypeList(const FormatStyle &Style) const { 698 if (is(TT_TemplateString) && closesScope()) 699 return true; 700 return MatchingParen && MatchingParen->opensBlockOrBlockTypeList(Style); 701 } 702 703 /// Return the actual namespace token, if this token starts a namespace 704 /// block. 705 const FormatToken *getNamespaceToken() const { 706 const FormatToken *NamespaceTok = this; 707 if (is(tok::comment)) 708 NamespaceTok = NamespaceTok->getNextNonComment(); 709 // Detect "(inline|export)? namespace" in the beginning of a line. 710 if (NamespaceTok && NamespaceTok->isOneOf(tok::kw_inline, tok::kw_export)) 711 NamespaceTok = NamespaceTok->getNextNonComment(); 712 return NamespaceTok && 713 NamespaceTok->isOneOf(tok::kw_namespace, TT_NamespaceMacro) 714 ? NamespaceTok 715 : nullptr; 716 } 717 718 void copyFrom(const FormatToken &Tok) { *this = Tok; } 719 720 private: 721 // Only allow copying via the explicit copyFrom method. 722 FormatToken(const FormatToken &) = delete; 723 FormatToken &operator=(const FormatToken &) = default; 724 725 template <typename A, typename... Ts> 726 bool startsSequenceInternal(A K1, Ts... Tokens) const { 727 if (is(tok::comment) && Next) 728 return Next->startsSequenceInternal(K1, Tokens...); 729 return is(K1) && Next && Next->startsSequenceInternal(Tokens...); 730 } 731 732 template <typename A> bool startsSequenceInternal(A K1) const { 733 if (is(tok::comment) && Next) 734 return Next->startsSequenceInternal(K1); 735 return is(K1); 736 } 737 738 template <typename A, typename... Ts> bool endsSequenceInternal(A K1) const { 739 if (is(tok::comment) && Previous) 740 return Previous->endsSequenceInternal(K1); 741 return is(K1); 742 } 743 744 template <typename A, typename... Ts> 745 bool endsSequenceInternal(A K1, Ts... Tokens) const { 746 if (is(tok::comment) && Previous) 747 return Previous->endsSequenceInternal(K1, Tokens...); 748 return is(K1) && Previous && Previous->endsSequenceInternal(Tokens...); 749 } 750 }; 751 752 class ContinuationIndenter; 753 struct LineState; 754 755 class TokenRole { 756 public: 757 TokenRole(const FormatStyle &Style) : Style(Style) {} 758 virtual ~TokenRole(); 759 760 /// After the \c TokenAnnotator has finished annotating all the tokens, 761 /// this function precomputes required information for formatting. 762 virtual void precomputeFormattingInfos(const FormatToken *Token); 763 764 /// Apply the special formatting that the given role demands. 765 /// 766 /// Assumes that the token having this role is already formatted. 767 /// 768 /// Continues formatting from \p State leaving indentation to \p Indenter and 769 /// returns the total penalty that this formatting incurs. 770 virtual unsigned formatFromToken(LineState &State, 771 ContinuationIndenter *Indenter, 772 bool DryRun) { 773 return 0; 774 } 775 776 /// Same as \c formatFromToken, but assumes that the first token has 777 /// already been set thereby deciding on the first line break. 778 virtual unsigned formatAfterToken(LineState &State, 779 ContinuationIndenter *Indenter, 780 bool DryRun) { 781 return 0; 782 } 783 784 /// Notifies the \c Role that a comma was found. 785 virtual void CommaFound(const FormatToken *Token) {} 786 787 virtual const FormatToken *lastComma() { return nullptr; } 788 789 protected: 790 const FormatStyle &Style; 791 }; 792 793 class CommaSeparatedList : public TokenRole { 794 public: 795 CommaSeparatedList(const FormatStyle &Style) 796 : TokenRole(Style), HasNestedBracedList(false) {} 797 798 void precomputeFormattingInfos(const FormatToken *Token) override; 799 800 unsigned formatAfterToken(LineState &State, ContinuationIndenter *Indenter, 801 bool DryRun) override; 802 803 unsigned formatFromToken(LineState &State, ContinuationIndenter *Indenter, 804 bool DryRun) override; 805 806 /// Adds \p Token as the next comma to the \c CommaSeparated list. 807 void CommaFound(const FormatToken *Token) override { 808 Commas.push_back(Token); 809 } 810 811 const FormatToken *lastComma() override { 812 if (Commas.empty()) 813 return nullptr; 814 return Commas.back(); 815 } 816 817 private: 818 /// A struct that holds information on how to format a given list with 819 /// a specific number of columns. 820 struct ColumnFormat { 821 /// The number of columns to use. 822 unsigned Columns; 823 824 /// The total width in characters. 825 unsigned TotalWidth; 826 827 /// The number of lines required for this format. 828 unsigned LineCount; 829 830 /// The size of each column in characters. 831 SmallVector<unsigned, 8> ColumnSizes; 832 }; 833 834 /// Calculate which \c ColumnFormat fits best into 835 /// \p RemainingCharacters. 836 const ColumnFormat *getColumnFormat(unsigned RemainingCharacters) const; 837 838 /// The ordered \c FormatTokens making up the commas of this list. 839 SmallVector<const FormatToken *, 8> Commas; 840 841 /// The length of each of the list's items in characters including the 842 /// trailing comma. 843 SmallVector<unsigned, 8> ItemLengths; 844 845 /// Precomputed formats that can be used for this list. 846 SmallVector<ColumnFormat, 4> Formats; 847 848 bool HasNestedBracedList; 849 }; 850 851 /// Encapsulates keywords that are context sensitive or for languages not 852 /// properly supported by Clang's lexer. 853 struct AdditionalKeywords { 854 AdditionalKeywords(IdentifierTable &IdentTable) { 855 kw_final = &IdentTable.get("final"); 856 kw_override = &IdentTable.get("override"); 857 kw_in = &IdentTable.get("in"); 858 kw_of = &IdentTable.get("of"); 859 kw_CF_CLOSED_ENUM = &IdentTable.get("CF_CLOSED_ENUM"); 860 kw_CF_ENUM = &IdentTable.get("CF_ENUM"); 861 kw_CF_OPTIONS = &IdentTable.get("CF_OPTIONS"); 862 kw_NS_CLOSED_ENUM = &IdentTable.get("NS_CLOSED_ENUM"); 863 kw_NS_ENUM = &IdentTable.get("NS_ENUM"); 864 kw_NS_OPTIONS = &IdentTable.get("NS_OPTIONS"); 865 866 kw_as = &IdentTable.get("as"); 867 kw_async = &IdentTable.get("async"); 868 kw_await = &IdentTable.get("await"); 869 kw_declare = &IdentTable.get("declare"); 870 kw_finally = &IdentTable.get("finally"); 871 kw_from = &IdentTable.get("from"); 872 kw_function = &IdentTable.get("function"); 873 kw_get = &IdentTable.get("get"); 874 kw_import = &IdentTable.get("import"); 875 kw_infer = &IdentTable.get("infer"); 876 kw_is = &IdentTable.get("is"); 877 kw_let = &IdentTable.get("let"); 878 kw_module = &IdentTable.get("module"); 879 kw_readonly = &IdentTable.get("readonly"); 880 kw_set = &IdentTable.get("set"); 881 kw_type = &IdentTable.get("type"); 882 kw_typeof = &IdentTable.get("typeof"); 883 kw_var = &IdentTable.get("var"); 884 kw_yield = &IdentTable.get("yield"); 885 886 kw_abstract = &IdentTable.get("abstract"); 887 kw_assert = &IdentTable.get("assert"); 888 kw_extends = &IdentTable.get("extends"); 889 kw_implements = &IdentTable.get("implements"); 890 kw_instanceof = &IdentTable.get("instanceof"); 891 kw_interface = &IdentTable.get("interface"); 892 kw_native = &IdentTable.get("native"); 893 kw_package = &IdentTable.get("package"); 894 kw_synchronized = &IdentTable.get("synchronized"); 895 kw_throws = &IdentTable.get("throws"); 896 kw___except = &IdentTable.get("__except"); 897 kw___has_include = &IdentTable.get("__has_include"); 898 kw___has_include_next = &IdentTable.get("__has_include_next"); 899 900 kw_mark = &IdentTable.get("mark"); 901 902 kw_extend = &IdentTable.get("extend"); 903 kw_option = &IdentTable.get("option"); 904 kw_optional = &IdentTable.get("optional"); 905 kw_repeated = &IdentTable.get("repeated"); 906 kw_required = &IdentTable.get("required"); 907 kw_returns = &IdentTable.get("returns"); 908 909 kw_signals = &IdentTable.get("signals"); 910 kw_qsignals = &IdentTable.get("Q_SIGNALS"); 911 kw_slots = &IdentTable.get("slots"); 912 kw_qslots = &IdentTable.get("Q_SLOTS"); 913 914 // C# keywords 915 kw_dollar = &IdentTable.get("dollar"); 916 kw_base = &IdentTable.get("base"); 917 kw_byte = &IdentTable.get("byte"); 918 kw_checked = &IdentTable.get("checked"); 919 kw_decimal = &IdentTable.get("decimal"); 920 kw_delegate = &IdentTable.get("delegate"); 921 kw_event = &IdentTable.get("event"); 922 kw_fixed = &IdentTable.get("fixed"); 923 kw_foreach = &IdentTable.get("foreach"); 924 kw_implicit = &IdentTable.get("implicit"); 925 kw_internal = &IdentTable.get("internal"); 926 kw_lock = &IdentTable.get("lock"); 927 kw_null = &IdentTable.get("null"); 928 kw_object = &IdentTable.get("object"); 929 kw_out = &IdentTable.get("out"); 930 kw_params = &IdentTable.get("params"); 931 kw_ref = &IdentTable.get("ref"); 932 kw_string = &IdentTable.get("string"); 933 kw_stackalloc = &IdentTable.get("stackalloc"); 934 kw_sbyte = &IdentTable.get("sbyte"); 935 kw_sealed = &IdentTable.get("sealed"); 936 kw_uint = &IdentTable.get("uint"); 937 kw_ulong = &IdentTable.get("ulong"); 938 kw_unchecked = &IdentTable.get("unchecked"); 939 kw_unsafe = &IdentTable.get("unsafe"); 940 kw_ushort = &IdentTable.get("ushort"); 941 kw_when = &IdentTable.get("when"); 942 kw_where = &IdentTable.get("where"); 943 944 // Keep this at the end of the constructor to make sure everything here 945 // is 946 // already initialized. 947 JsExtraKeywords = std::unordered_set<IdentifierInfo *>( 948 {kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from, 949 kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_override, 950 kw_readonly, kw_set, kw_type, kw_typeof, kw_var, kw_yield, 951 // Keywords from the Java section. 952 kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface}); 953 954 CSharpExtraKeywords = std::unordered_set<IdentifierInfo *>( 955 {kw_base, kw_byte, kw_checked, kw_decimal, kw_delegate, kw_event, 956 kw_fixed, kw_foreach, kw_implicit, kw_in, kw_interface, kw_internal, 957 kw_is, kw_lock, kw_null, kw_object, kw_out, kw_override, kw_params, 958 kw_readonly, kw_ref, kw_string, kw_stackalloc, kw_sbyte, kw_sealed, 959 kw_uint, kw_ulong, kw_unchecked, kw_unsafe, kw_ushort, kw_when, 960 kw_where, 961 // Keywords from the JavaScript section. 962 kw_as, kw_async, kw_await, kw_declare, kw_finally, kw_from, 963 kw_function, kw_get, kw_import, kw_is, kw_let, kw_module, kw_readonly, 964 kw_set, kw_type, kw_typeof, kw_var, kw_yield, 965 // Keywords from the Java section. 966 kw_abstract, kw_extends, kw_implements, kw_instanceof, kw_interface}); 967 } 968 969 // Context sensitive keywords. 970 IdentifierInfo *kw_final; 971 IdentifierInfo *kw_override; 972 IdentifierInfo *kw_in; 973 IdentifierInfo *kw_of; 974 IdentifierInfo *kw_CF_CLOSED_ENUM; 975 IdentifierInfo *kw_CF_ENUM; 976 IdentifierInfo *kw_CF_OPTIONS; 977 IdentifierInfo *kw_NS_CLOSED_ENUM; 978 IdentifierInfo *kw_NS_ENUM; 979 IdentifierInfo *kw_NS_OPTIONS; 980 IdentifierInfo *kw___except; 981 IdentifierInfo *kw___has_include; 982 IdentifierInfo *kw___has_include_next; 983 984 // JavaScript keywords. 985 IdentifierInfo *kw_as; 986 IdentifierInfo *kw_async; 987 IdentifierInfo *kw_await; 988 IdentifierInfo *kw_declare; 989 IdentifierInfo *kw_finally; 990 IdentifierInfo *kw_from; 991 IdentifierInfo *kw_function; 992 IdentifierInfo *kw_get; 993 IdentifierInfo *kw_import; 994 IdentifierInfo *kw_infer; 995 IdentifierInfo *kw_is; 996 IdentifierInfo *kw_let; 997 IdentifierInfo *kw_module; 998 IdentifierInfo *kw_readonly; 999 IdentifierInfo *kw_set; 1000 IdentifierInfo *kw_type; 1001 IdentifierInfo *kw_typeof; 1002 IdentifierInfo *kw_var; 1003 IdentifierInfo *kw_yield; 1004 1005 // Java keywords. 1006 IdentifierInfo *kw_abstract; 1007 IdentifierInfo *kw_assert; 1008 IdentifierInfo *kw_extends; 1009 IdentifierInfo *kw_implements; 1010 IdentifierInfo *kw_instanceof; 1011 IdentifierInfo *kw_interface; 1012 IdentifierInfo *kw_native; 1013 IdentifierInfo *kw_package; 1014 IdentifierInfo *kw_synchronized; 1015 IdentifierInfo *kw_throws; 1016 1017 // Pragma keywords. 1018 IdentifierInfo *kw_mark; 1019 1020 // Proto keywords. 1021 IdentifierInfo *kw_extend; 1022 IdentifierInfo *kw_option; 1023 IdentifierInfo *kw_optional; 1024 IdentifierInfo *kw_repeated; 1025 IdentifierInfo *kw_required; 1026 IdentifierInfo *kw_returns; 1027 1028 // QT keywords. 1029 IdentifierInfo *kw_signals; 1030 IdentifierInfo *kw_qsignals; 1031 IdentifierInfo *kw_slots; 1032 IdentifierInfo *kw_qslots; 1033 1034 // C# keywords 1035 IdentifierInfo *kw_dollar; 1036 IdentifierInfo *kw_base; 1037 IdentifierInfo *kw_byte; 1038 IdentifierInfo *kw_checked; 1039 IdentifierInfo *kw_decimal; 1040 IdentifierInfo *kw_delegate; 1041 IdentifierInfo *kw_event; 1042 IdentifierInfo *kw_fixed; 1043 IdentifierInfo *kw_foreach; 1044 IdentifierInfo *kw_implicit; 1045 IdentifierInfo *kw_internal; 1046 1047 IdentifierInfo *kw_lock; 1048 IdentifierInfo *kw_null; 1049 IdentifierInfo *kw_object; 1050 IdentifierInfo *kw_out; 1051 1052 IdentifierInfo *kw_params; 1053 1054 IdentifierInfo *kw_ref; 1055 IdentifierInfo *kw_string; 1056 IdentifierInfo *kw_stackalloc; 1057 IdentifierInfo *kw_sbyte; 1058 IdentifierInfo *kw_sealed; 1059 IdentifierInfo *kw_uint; 1060 IdentifierInfo *kw_ulong; 1061 IdentifierInfo *kw_unchecked; 1062 IdentifierInfo *kw_unsafe; 1063 IdentifierInfo *kw_ushort; 1064 IdentifierInfo *kw_when; 1065 IdentifierInfo *kw_where; 1066 1067 /// Returns \c true if \p Tok is a true JavaScript identifier, returns 1068 /// \c false if it is a keyword or a pseudo keyword. 1069 /// If \c AcceptIdentifierName is true, returns true not only for keywords, 1070 // but also for IdentifierName tokens (aka pseudo-keywords), such as 1071 // ``yield``. 1072 bool IsJavaScriptIdentifier(const FormatToken &Tok, 1073 bool AcceptIdentifierName = true) const { 1074 // Based on the list of JavaScript & TypeScript keywords here: 1075 // https://github.com/microsoft/TypeScript/blob/main/src/compiler/scanner.ts#L74 1076 switch (Tok.Tok.getKind()) { 1077 case tok::kw_break: 1078 case tok::kw_case: 1079 case tok::kw_catch: 1080 case tok::kw_class: 1081 case tok::kw_continue: 1082 case tok::kw_const: 1083 case tok::kw_default: 1084 case tok::kw_delete: 1085 case tok::kw_do: 1086 case tok::kw_else: 1087 case tok::kw_enum: 1088 case tok::kw_export: 1089 case tok::kw_false: 1090 case tok::kw_for: 1091 case tok::kw_if: 1092 case tok::kw_import: 1093 case tok::kw_module: 1094 case tok::kw_new: 1095 case tok::kw_private: 1096 case tok::kw_protected: 1097 case tok::kw_public: 1098 case tok::kw_return: 1099 case tok::kw_static: 1100 case tok::kw_switch: 1101 case tok::kw_this: 1102 case tok::kw_throw: 1103 case tok::kw_true: 1104 case tok::kw_try: 1105 case tok::kw_typeof: 1106 case tok::kw_void: 1107 case tok::kw_while: 1108 // These are JS keywords that are lexed by LLVM/clang as keywords. 1109 return false; 1110 case tok::identifier: { 1111 // For identifiers, make sure they are true identifiers, excluding the 1112 // JavaScript pseudo-keywords (not lexed by LLVM/clang as keywords). 1113 bool IsPseudoKeyword = 1114 JsExtraKeywords.find(Tok.Tok.getIdentifierInfo()) != 1115 JsExtraKeywords.end(); 1116 return AcceptIdentifierName || !IsPseudoKeyword; 1117 } 1118 default: 1119 // Other keywords are handled in the switch below, to avoid problems due 1120 // to duplicate case labels when using the #include trick. 1121 break; 1122 } 1123 1124 switch (Tok.Tok.getKind()) { 1125 // Handle C++ keywords not included above: these are all JS identifiers. 1126 #define KEYWORD(X, Y) case tok::kw_##X: 1127 #include "clang/Basic/TokenKinds.def" 1128 // #undef KEYWORD is not needed -- it's #undef-ed at the end of 1129 // TokenKinds.def 1130 return true; 1131 default: 1132 // All other tokens (punctuation etc) are not JS identifiers. 1133 return false; 1134 } 1135 } 1136 1137 /// Returns \c true if \p Tok is a C# keyword, returns 1138 /// \c false if it is a anything else. 1139 bool isCSharpKeyword(const FormatToken &Tok) const { 1140 switch (Tok.Tok.getKind()) { 1141 case tok::kw_bool: 1142 case tok::kw_break: 1143 case tok::kw_case: 1144 case tok::kw_catch: 1145 case tok::kw_char: 1146 case tok::kw_class: 1147 case tok::kw_const: 1148 case tok::kw_continue: 1149 case tok::kw_default: 1150 case tok::kw_do: 1151 case tok::kw_double: 1152 case tok::kw_else: 1153 case tok::kw_enum: 1154 case tok::kw_explicit: 1155 case tok::kw_extern: 1156 case tok::kw_false: 1157 case tok::kw_float: 1158 case tok::kw_for: 1159 case tok::kw_goto: 1160 case tok::kw_if: 1161 case tok::kw_int: 1162 case tok::kw_long: 1163 case tok::kw_namespace: 1164 case tok::kw_new: 1165 case tok::kw_operator: 1166 case tok::kw_private: 1167 case tok::kw_protected: 1168 case tok::kw_public: 1169 case tok::kw_return: 1170 case tok::kw_short: 1171 case tok::kw_sizeof: 1172 case tok::kw_static: 1173 case tok::kw_struct: 1174 case tok::kw_switch: 1175 case tok::kw_this: 1176 case tok::kw_throw: 1177 case tok::kw_true: 1178 case tok::kw_try: 1179 case tok::kw_typeof: 1180 case tok::kw_using: 1181 case tok::kw_virtual: 1182 case tok::kw_void: 1183 case tok::kw_volatile: 1184 case tok::kw_while: 1185 return true; 1186 default: 1187 return Tok.is(tok::identifier) && 1188 CSharpExtraKeywords.find(Tok.Tok.getIdentifierInfo()) == 1189 CSharpExtraKeywords.end(); 1190 } 1191 } 1192 1193 private: 1194 /// The JavaScript keywords beyond the C++ keyword set. 1195 std::unordered_set<IdentifierInfo *> JsExtraKeywords; 1196 1197 /// The C# keywords beyond the C++ keyword set 1198 std::unordered_set<IdentifierInfo *> CSharpExtraKeywords; 1199 }; 1200 1201 } // namespace format 1202 } // namespace clang 1203 1204 #endif 1205