1 //===--- SourceCode.h - Manipulating source code as strings -----*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 #include "SourceCode.h" 9 10 #include "Context.h" 11 #include "FuzzyMatch.h" 12 #include "Logger.h" 13 #include "Protocol.h" 14 #include "clang/AST/ASTContext.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "clang/Basic/TokenKinds.h" 17 #include "clang/Format/Format.h" 18 #include "clang/Lex/Lexer.h" 19 #include "clang/Lex/Preprocessor.h" 20 #include "llvm/ADT/None.h" 21 #include "llvm/ADT/StringExtras.h" 22 #include "llvm/ADT/StringRef.h" 23 #include "llvm/Support/Compiler.h" 24 #include "llvm/Support/Errc.h" 25 #include "llvm/Support/Error.h" 26 #include "llvm/Support/ErrorHandling.h" 27 #include "llvm/Support/Path.h" 28 #include <algorithm> 29 30 namespace clang { 31 namespace clangd { 32 33 // Here be dragons. LSP positions use columns measured in *UTF-16 code units*! 34 // Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial. 35 36 // Iterates over unicode codepoints in the (UTF-8) string. For each, 37 // invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true. 38 // Returns true if CB returned true, false if we hit the end of string. 39 template <typename Callback> 40 static bool iterateCodepoints(llvm::StringRef U8, const Callback &CB) { 41 // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). 42 // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx. 43 for (size_t I = 0; I < U8.size();) { 44 unsigned char C = static_cast<unsigned char>(U8[I]); 45 if (LLVM_LIKELY(!(C & 0x80))) { // ASCII character. 46 if (CB(1, 1)) 47 return true; 48 ++I; 49 continue; 50 } 51 // This convenient property of UTF-8 holds for all non-ASCII characters. 52 size_t UTF8Length = llvm::countLeadingOnes(C); 53 // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here. 54 // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug. 55 assert((UTF8Length >= 2 && UTF8Length <= 4) && 56 "Invalid UTF-8, or transcoding bug?"); 57 I += UTF8Length; // Skip over all trailing bytes. 58 // A codepoint takes two UTF-16 code unit if it's astral (outside BMP). 59 // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...) 60 if (CB(UTF8Length, UTF8Length == 4 ? 2 : 1)) 61 return true; 62 } 63 return false; 64 } 65 66 // Returns the byte offset into the string that is an offset of \p Units in 67 // the specified encoding. 68 // Conceptually, this converts to the encoding, truncates to CodeUnits, 69 // converts back to UTF-8, and returns the length in bytes. 70 static size_t measureUnits(llvm::StringRef U8, int Units, OffsetEncoding Enc, 71 bool &Valid) { 72 Valid = Units >= 0; 73 if (Units <= 0) 74 return 0; 75 size_t Result = 0; 76 switch (Enc) { 77 case OffsetEncoding::UTF8: 78 Result = Units; 79 break; 80 case OffsetEncoding::UTF16: 81 Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) { 82 Result += U8Len; 83 Units -= U16Len; 84 return Units <= 0; 85 }); 86 if (Units < 0) // Offset in the middle of a surrogate pair. 87 Valid = false; 88 break; 89 case OffsetEncoding::UTF32: 90 Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) { 91 Result += U8Len; 92 Units--; 93 return Units <= 0; 94 }); 95 break; 96 case OffsetEncoding::UnsupportedEncoding: 97 llvm_unreachable("unsupported encoding"); 98 } 99 // Don't return an out-of-range index if we overran. 100 if (Result > U8.size()) { 101 Valid = false; 102 return U8.size(); 103 } 104 return Result; 105 } 106 107 Key<OffsetEncoding> kCurrentOffsetEncoding; 108 static OffsetEncoding lspEncoding() { 109 auto *Enc = Context::current().get(kCurrentOffsetEncoding); 110 return Enc ? *Enc : OffsetEncoding::UTF16; 111 } 112 113 // Like most strings in clangd, the input is UTF-8 encoded. 114 size_t lspLength(llvm::StringRef Code) { 115 size_t Count = 0; 116 switch (lspEncoding()) { 117 case OffsetEncoding::UTF8: 118 Count = Code.size(); 119 break; 120 case OffsetEncoding::UTF16: 121 iterateCodepoints(Code, [&](int U8Len, int U16Len) { 122 Count += U16Len; 123 return false; 124 }); 125 break; 126 case OffsetEncoding::UTF32: 127 iterateCodepoints(Code, [&](int U8Len, int U16Len) { 128 ++Count; 129 return false; 130 }); 131 break; 132 case OffsetEncoding::UnsupportedEncoding: 133 llvm_unreachable("unsupported encoding"); 134 } 135 return Count; 136 } 137 138 llvm::Expected<size_t> positionToOffset(llvm::StringRef Code, Position P, 139 bool AllowColumnsBeyondLineLength) { 140 if (P.line < 0) 141 return llvm::make_error<llvm::StringError>( 142 llvm::formatv("Line value can't be negative ({0})", P.line), 143 llvm::errc::invalid_argument); 144 if (P.character < 0) 145 return llvm::make_error<llvm::StringError>( 146 llvm::formatv("Character value can't be negative ({0})", P.character), 147 llvm::errc::invalid_argument); 148 size_t StartOfLine = 0; 149 for (int I = 0; I != P.line; ++I) { 150 size_t NextNL = Code.find('\n', StartOfLine); 151 if (NextNL == llvm::StringRef::npos) 152 return llvm::make_error<llvm::StringError>( 153 llvm::formatv("Line value is out of range ({0})", P.line), 154 llvm::errc::invalid_argument); 155 StartOfLine = NextNL + 1; 156 } 157 StringRef Line = 158 Code.substr(StartOfLine).take_until([](char C) { return C == '\n'; }); 159 160 // P.character may be in UTF-16, transcode if necessary. 161 bool Valid; 162 size_t ByteInLine = measureUnits(Line, P.character, lspEncoding(), Valid); 163 if (!Valid && !AllowColumnsBeyondLineLength) 164 return llvm::make_error<llvm::StringError>( 165 llvm::formatv("{0} offset {1} is invalid for line {2}", lspEncoding(), 166 P.character, P.line), 167 llvm::errc::invalid_argument); 168 return StartOfLine + ByteInLine; 169 } 170 171 Position offsetToPosition(llvm::StringRef Code, size_t Offset) { 172 Offset = std::min(Code.size(), Offset); 173 llvm::StringRef Before = Code.substr(0, Offset); 174 int Lines = Before.count('\n'); 175 size_t PrevNL = Before.rfind('\n'); 176 size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1); 177 Position Pos; 178 Pos.line = Lines; 179 Pos.character = lspLength(Before.substr(StartOfLine)); 180 return Pos; 181 } 182 183 Position sourceLocToPosition(const SourceManager &SM, SourceLocation Loc) { 184 // We use the SourceManager's line tables, but its column number is in bytes. 185 FileID FID; 186 unsigned Offset; 187 std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc); 188 Position P; 189 P.line = static_cast<int>(SM.getLineNumber(FID, Offset)) - 1; 190 bool Invalid = false; 191 llvm::StringRef Code = SM.getBufferData(FID, &Invalid); 192 if (!Invalid) { 193 auto ColumnInBytes = SM.getColumnNumber(FID, Offset) - 1; 194 auto LineSoFar = Code.substr(Offset - ColumnInBytes, ColumnInBytes); 195 P.character = lspLength(LineSoFar); 196 } 197 return P; 198 } 199 200 llvm::Optional<Range> getTokenRange(const SourceManager &SM, 201 const LangOptions &LangOpts, 202 SourceLocation TokLoc) { 203 if (!TokLoc.isValid()) 204 return llvm::None; 205 SourceLocation End = Lexer::getLocForEndOfToken(TokLoc, 0, SM, LangOpts); 206 if (!End.isValid()) 207 return llvm::None; 208 return halfOpenToRange(SM, CharSourceRange::getCharRange(TokLoc, End)); 209 } 210 211 bool isValidFileRange(const SourceManager &Mgr, SourceRange R) { 212 if (!R.getBegin().isValid() || !R.getEnd().isValid()) 213 return false; 214 215 FileID BeginFID; 216 size_t BeginOffset = 0; 217 std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin()); 218 219 FileID EndFID; 220 size_t EndOffset = 0; 221 std::tie(EndFID, EndOffset) = Mgr.getDecomposedLoc(R.getEnd()); 222 223 return BeginFID.isValid() && BeginFID == EndFID && BeginOffset <= EndOffset; 224 } 225 226 bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R, 227 SourceLocation L) { 228 assert(isValidFileRange(Mgr, R)); 229 230 FileID BeginFID; 231 size_t BeginOffset = 0; 232 std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin()); 233 size_t EndOffset = Mgr.getFileOffset(R.getEnd()); 234 235 FileID LFid; 236 size_t LOffset; 237 std::tie(LFid, LOffset) = Mgr.getDecomposedLoc(L); 238 return BeginFID == LFid && BeginOffset <= LOffset && LOffset < EndOffset; 239 } 240 241 bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R, 242 SourceLocation L) { 243 return L == R.getEnd() || halfOpenRangeContains(Mgr, R, L); 244 } 245 246 llvm::Optional<SourceRange> toHalfOpenFileRange(const SourceManager &Mgr, 247 const LangOptions &LangOpts, 248 SourceRange R) { 249 auto Begin = Mgr.getFileLoc(R.getBegin()); 250 if (Begin.isInvalid()) 251 return llvm::None; 252 auto End = Mgr.getFileLoc(R.getEnd()); 253 if (End.isInvalid()) 254 return llvm::None; 255 End = Lexer::getLocForEndOfToken(End, 0, Mgr, LangOpts); 256 257 SourceRange Result(Begin, End); 258 if (!isValidFileRange(Mgr, Result)) 259 return llvm::None; 260 return Result; 261 } 262 263 llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R) { 264 assert(isValidFileRange(SM, R)); 265 bool Invalid = false; 266 auto *Buf = SM.getBuffer(SM.getFileID(R.getBegin()), &Invalid); 267 assert(!Invalid); 268 269 size_t BeginOffset = SM.getFileOffset(R.getBegin()); 270 size_t EndOffset = SM.getFileOffset(R.getEnd()); 271 return Buf->getBuffer().substr(BeginOffset, EndOffset - BeginOffset); 272 } 273 274 llvm::Expected<SourceLocation> sourceLocationInMainFile(const SourceManager &SM, 275 Position P) { 276 llvm::StringRef Code = SM.getBuffer(SM.getMainFileID())->getBuffer(); 277 auto Offset = 278 positionToOffset(Code, P, /*AllowColumnBeyondLineLength=*/false); 279 if (!Offset) 280 return Offset.takeError(); 281 return SM.getLocForStartOfFile(SM.getMainFileID()).getLocWithOffset(*Offset); 282 } 283 284 Range halfOpenToRange(const SourceManager &SM, CharSourceRange R) { 285 // Clang is 1-based, LSP uses 0-based indexes. 286 Position Begin = sourceLocToPosition(SM, R.getBegin()); 287 Position End = sourceLocToPosition(SM, R.getEnd()); 288 289 return {Begin, End}; 290 } 291 292 std::pair<size_t, size_t> offsetToClangLineColumn(llvm::StringRef Code, 293 size_t Offset) { 294 Offset = std::min(Code.size(), Offset); 295 llvm::StringRef Before = Code.substr(0, Offset); 296 int Lines = Before.count('\n'); 297 size_t PrevNL = Before.rfind('\n'); 298 size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1); 299 return {Lines + 1, Offset - StartOfLine + 1}; 300 } 301 302 std::pair<StringRef, StringRef> splitQualifiedName(StringRef QName) { 303 size_t Pos = QName.rfind("::"); 304 if (Pos == llvm::StringRef::npos) 305 return {llvm::StringRef(), QName}; 306 return {QName.substr(0, Pos + 2), QName.substr(Pos + 2)}; 307 } 308 309 TextEdit replacementToEdit(llvm::StringRef Code, 310 const tooling::Replacement &R) { 311 Range ReplacementRange = { 312 offsetToPosition(Code, R.getOffset()), 313 offsetToPosition(Code, R.getOffset() + R.getLength())}; 314 return {ReplacementRange, R.getReplacementText()}; 315 } 316 317 std::vector<TextEdit> replacementsToEdits(llvm::StringRef Code, 318 const tooling::Replacements &Repls) { 319 std::vector<TextEdit> Edits; 320 for (const auto &R : Repls) 321 Edits.push_back(replacementToEdit(Code, R)); 322 return Edits; 323 } 324 325 llvm::Optional<std::string> getCanonicalPath(const FileEntry *F, 326 const SourceManager &SourceMgr) { 327 if (!F) 328 return None; 329 330 llvm::SmallString<128> FilePath = F->getName(); 331 if (!llvm::sys::path::is_absolute(FilePath)) { 332 if (auto EC = 333 SourceMgr.getFileManager().getVirtualFileSystem().makeAbsolute( 334 FilePath)) { 335 elog("Could not turn relative path '{0}' to absolute: {1}", FilePath, 336 EC.message()); 337 return None; 338 } 339 } 340 341 // Handle the symbolic link path case where the current working directory 342 // (getCurrentWorkingDirectory) is a symlink./ We always want to the real 343 // file path (instead of the symlink path) for the C++ symbols. 344 // 345 // Consider the following example: 346 // 347 // src dir: /project/src/foo.h 348 // current working directory (symlink): /tmp/build -> /project/src/ 349 // 350 // The file path of Symbol is "/project/src/foo.h" instead of 351 // "/tmp/build/foo.h" 352 if (const DirectoryEntry *Dir = SourceMgr.getFileManager().getDirectory( 353 llvm::sys::path::parent_path(FilePath))) { 354 llvm::SmallString<128> RealPath; 355 llvm::StringRef DirName = SourceMgr.getFileManager().getCanonicalName(Dir); 356 llvm::sys::path::append(RealPath, DirName, 357 llvm::sys::path::filename(FilePath)); 358 return RealPath.str().str(); 359 } 360 361 return FilePath.str().str(); 362 } 363 364 TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M, 365 const LangOptions &L) { 366 TextEdit Result; 367 Result.range = 368 halfOpenToRange(M, Lexer::makeFileCharRange(FixIt.RemoveRange, M, L)); 369 Result.newText = FixIt.CodeToInsert; 370 return Result; 371 } 372 373 bool isRangeConsecutive(const Range &Left, const Range &Right) { 374 return Left.end.line == Right.start.line && 375 Left.end.character == Right.start.character; 376 } 377 378 FileDigest digest(llvm::StringRef Content) { 379 return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()}); 380 } 381 382 llvm::Optional<FileDigest> digestFile(const SourceManager &SM, FileID FID) { 383 bool Invalid = false; 384 llvm::StringRef Content = SM.getBufferData(FID, &Invalid); 385 if (Invalid) 386 return None; 387 return digest(Content); 388 } 389 390 format::FormatStyle getFormatStyleForFile(llvm::StringRef File, 391 llvm::StringRef Content, 392 llvm::vfs::FileSystem *FS) { 393 auto Style = format::getStyle(format::DefaultFormatStyle, File, 394 format::DefaultFallbackStyle, Content, FS); 395 if (!Style) { 396 log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.", File, 397 Style.takeError()); 398 Style = format::getLLVMStyle(); 399 } 400 return *Style; 401 } 402 403 llvm::Expected<tooling::Replacements> 404 cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces, 405 const format::FormatStyle &Style) { 406 auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style); 407 if (!CleanReplaces) 408 return CleanReplaces; 409 return formatReplacements(Code, std::move(*CleanReplaces), Style); 410 } 411 412 template <typename Action> 413 static void lex(llvm::StringRef Code, const format::FormatStyle &Style, 414 Action A) { 415 // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated! 416 std::string NullTerminatedCode = Code.str(); 417 SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode); 418 auto &SM = FileSM.get(); 419 auto FID = SM.getMainFileID(); 420 Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style)); 421 Token Tok; 422 423 while (!Lex.LexFromRawLexer(Tok)) 424 A(Tok); 425 } 426 427 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content, 428 const format::FormatStyle &Style) { 429 llvm::StringMap<unsigned> Identifiers; 430 lex(Content, Style, [&](const clang::Token &Tok) { 431 switch (Tok.getKind()) { 432 case tok::identifier: 433 ++Identifiers[Tok.getIdentifierInfo()->getName()]; 434 break; 435 case tok::raw_identifier: 436 ++Identifiers[Tok.getRawIdentifier()]; 437 break; 438 default: 439 break; 440 } 441 }); 442 return Identifiers; 443 } 444 445 namespace { 446 enum NamespaceEvent { 447 BeginNamespace, // namespace <ns> {. Payload is resolved <ns>. 448 EndNamespace, // } // namespace <ns>. Payload is resolved *outer* namespace. 449 UsingDirective // using namespace <ns>. Payload is unresolved <ns>. 450 }; 451 // Scans C++ source code for constructs that change the visible namespaces. 452 void parseNamespaceEvents( 453 llvm::StringRef Code, const format::FormatStyle &Style, 454 llvm::function_ref<void(NamespaceEvent, llvm::StringRef)> Callback) { 455 456 // Stack of enclosing namespaces, e.g. {"clang", "clangd"} 457 std::vector<std::string> Enclosing; // Contains e.g. "clang", "clangd" 458 // Stack counts open braces. true if the brace opened a namespace. 459 std::vector<bool> BraceStack; 460 461 enum { 462 Default, 463 Namespace, // just saw 'namespace' 464 NamespaceName, // just saw 'namespace' NSName 465 Using, // just saw 'using' 466 UsingNamespace, // just saw 'using namespace' 467 UsingNamespaceName, // just saw 'using namespace' NSName 468 } State = Default; 469 std::string NSName; 470 471 lex(Code, Style, [&](const clang::Token &Tok) { 472 switch(Tok.getKind()) { 473 case tok::raw_identifier: 474 // In raw mode, this could be a keyword or a name. 475 switch (State) { 476 case UsingNamespace: 477 case UsingNamespaceName: 478 NSName.append(Tok.getRawIdentifier()); 479 State = UsingNamespaceName; 480 break; 481 case Namespace: 482 case NamespaceName: 483 NSName.append(Tok.getRawIdentifier()); 484 State = NamespaceName; 485 break; 486 case Using: 487 State = 488 (Tok.getRawIdentifier() == "namespace") ? UsingNamespace : Default; 489 break; 490 case Default: 491 NSName.clear(); 492 if (Tok.getRawIdentifier() == "namespace") 493 State = Namespace; 494 else if (Tok.getRawIdentifier() == "using") 495 State = Using; 496 break; 497 } 498 break; 499 case tok::coloncolon: 500 // This can come at the beginning or in the middle of a namespace name. 501 switch (State) { 502 case UsingNamespace: 503 case UsingNamespaceName: 504 NSName.append("::"); 505 State = UsingNamespaceName; 506 break; 507 case NamespaceName: 508 NSName.append("::"); 509 State = NamespaceName; 510 break; 511 case Namespace: // Not legal here. 512 case Using: 513 case Default: 514 State = Default; 515 break; 516 } 517 break; 518 case tok::l_brace: 519 // Record which { started a namespace, so we know when } ends one. 520 if (State == NamespaceName) { 521 // Parsed: namespace <name> { 522 BraceStack.push_back(true); 523 Enclosing.push_back(NSName); 524 Callback(BeginNamespace, llvm::join(Enclosing, "::")); 525 } else { 526 // This case includes anonymous namespaces (State = Namespace). 527 // For our purposes, they're not namespaces and we ignore them. 528 BraceStack.push_back(false); 529 } 530 State = Default; 531 break; 532 case tok::r_brace: 533 // If braces are unmatched, we're going to be confused, but don't crash. 534 if (!BraceStack.empty()) { 535 if (BraceStack.back()) { 536 // Parsed: } // namespace 537 Enclosing.pop_back(); 538 Callback(EndNamespace, llvm::join(Enclosing, "::")); 539 } 540 BraceStack.pop_back(); 541 } 542 break; 543 case tok::semi: 544 if (State == UsingNamespaceName) 545 // Parsed: using namespace <name> ; 546 Callback(UsingDirective, llvm::StringRef(NSName)); 547 State = Default; 548 break; 549 default: 550 State = Default; 551 break; 552 } 553 }); 554 } 555 556 // Returns the prefix namespaces of NS: {"" ... NS}. 557 llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) { 558 llvm::SmallVector<llvm::StringRef, 8> Results; 559 Results.push_back(NS.take_front(0)); 560 NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false); 561 for (llvm::StringRef &R : Results) 562 R = NS.take_front(R.end() - NS.begin()); 563 return Results; 564 } 565 566 } // namespace 567 568 std::vector<std::string> visibleNamespaces(llvm::StringRef Code, 569 const format::FormatStyle &Style) { 570 std::string Current; 571 // Map from namespace to (resolved) namespaces introduced via using directive. 572 llvm::StringMap<llvm::StringSet<>> UsingDirectives; 573 574 parseNamespaceEvents(Code, Style, 575 [&](NamespaceEvent Event, llvm::StringRef NS) { 576 switch (Event) { 577 case BeginNamespace: 578 case EndNamespace: 579 Current = NS; 580 break; 581 case UsingDirective: 582 if (NS.consume_front("::")) 583 UsingDirectives[Current].insert(NS); 584 else { 585 for (llvm::StringRef Enclosing : 586 ancestorNamespaces(Current)) { 587 if (Enclosing.empty()) 588 UsingDirectives[Current].insert(NS); 589 else 590 UsingDirectives[Current].insert( 591 (Enclosing + "::" + NS).str()); 592 } 593 } 594 break; 595 } 596 }); 597 598 std::vector<std::string> Found; 599 for (llvm::StringRef Enclosing : ancestorNamespaces(Current)) { 600 Found.push_back(Enclosing); 601 auto It = UsingDirectives.find(Enclosing); 602 if (It != UsingDirectives.end()) 603 for (const auto& Used : It->second) 604 Found.push_back(Used.getKey()); 605 } 606 607 608 llvm::sort(Found, [&](const std::string &LHS, const std::string &RHS) { 609 if (Current == RHS) 610 return false; 611 if (Current == LHS) 612 return true; 613 return LHS < RHS; 614 }); 615 Found.erase(std::unique(Found.begin(), Found.end()), Found.end()); 616 return Found; 617 } 618 619 llvm::StringSet<> collectWords(llvm::StringRef Content) { 620 // We assume short words are not significant. 621 // We may want to consider other stopwords, e.g. language keywords. 622 // (A very naive implementation showed no benefit, but lexing might do better) 623 static constexpr int MinWordLength = 4; 624 625 std::vector<CharRole> Roles(Content.size()); 626 calculateRoles(Content, Roles); 627 628 llvm::StringSet<> Result; 629 llvm::SmallString<256> Word; 630 auto Flush = [&] { 631 if (Word.size() >= MinWordLength) { 632 for (char &C : Word) 633 C = llvm::toLower(C); 634 Result.insert(Word); 635 } 636 Word.clear(); 637 }; 638 for (unsigned I = 0; I < Content.size(); ++I) { 639 switch (Roles[I]) { 640 case Head: 641 Flush(); 642 LLVM_FALLTHROUGH; 643 case Tail: 644 Word.push_back(Content[I]); 645 break; 646 case Unknown: 647 case Separator: 648 Flush(); 649 break; 650 } 651 } 652 Flush(); 653 654 return Result; 655 } 656 657 llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc, 658 Preprocessor &PP) { 659 const auto &SM = PP.getSourceManager(); 660 const auto &LangOpts = PP.getLangOpts(); 661 Token Result; 662 if (Lexer::getRawToken(SM.getSpellingLoc(Loc), Result, SM, LangOpts, false)) 663 return None; 664 if (Result.is(tok::raw_identifier)) 665 PP.LookUpIdentifierInfo(Result); 666 IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo(); 667 if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition()) 668 return None; 669 670 std::pair<FileID, unsigned int> DecLoc = SM.getDecomposedExpansionLoc(Loc); 671 // Get the definition just before the searched location so that a macro 672 // referenced in a '#undef MACRO' can still be found. 673 SourceLocation BeforeSearchedLocation = 674 SM.getMacroArgExpandedLocation(SM.getLocForStartOfFile(DecLoc.first) 675 .getLocWithOffset(DecLoc.second - 1)); 676 MacroDefinition MacroDef = 677 PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation); 678 if (auto *MI = MacroDef.getMacroInfo()) 679 return DefinedMacro{IdentifierInfo->getName(), MI}; 680 return None; 681 } 682 683 } // namespace clangd 684 } // namespace clang 685