1 //===--- SourceCode.h - Manipulating source code as strings -----*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 #include "SourceCode.h"
9 
10 #include "Context.h"
11 #include "FuzzyMatch.h"
12 #include "Logger.h"
13 #include "Protocol.h"
14 #include "clang/AST/ASTContext.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Basic/TokenKinds.h"
17 #include "clang/Format/Format.h"
18 #include "clang/Lex/Lexer.h"
19 #include "clang/Lex/Preprocessor.h"
20 #include "llvm/ADT/None.h"
21 #include "llvm/ADT/StringExtras.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/Support/Compiler.h"
24 #include "llvm/Support/Errc.h"
25 #include "llvm/Support/Error.h"
26 #include "llvm/Support/ErrorHandling.h"
27 #include "llvm/Support/Path.h"
28 #include <algorithm>
29 
30 namespace clang {
31 namespace clangd {
32 
33 // Here be dragons. LSP positions use columns measured in *UTF-16 code units*!
34 // Clangd uses UTF-8 and byte-offsets internally, so conversion is nontrivial.
35 
36 // Iterates over unicode codepoints in the (UTF-8) string. For each,
37 // invokes CB(UTF-8 length, UTF-16 length), and breaks if it returns true.
38 // Returns true if CB returned true, false if we hit the end of string.
39 template <typename Callback>
40 static bool iterateCodepoints(llvm::StringRef U8, const Callback &CB) {
41   // A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
42   // Astral codepoints are encoded as 4 bytes in UTF-8, starting with 11110xxx.
43   for (size_t I = 0; I < U8.size();) {
44     unsigned char C = static_cast<unsigned char>(U8[I]);
45     if (LLVM_LIKELY(!(C & 0x80))) { // ASCII character.
46       if (CB(1, 1))
47         return true;
48       ++I;
49       continue;
50     }
51     // This convenient property of UTF-8 holds for all non-ASCII characters.
52     size_t UTF8Length = llvm::countLeadingOnes(C);
53     // 0xxx is ASCII, handled above. 10xxx is a trailing byte, invalid here.
54     // 11111xxx is not valid UTF-8 at all. Assert because it's probably our bug.
55     assert((UTF8Length >= 2 && UTF8Length <= 4) &&
56            "Invalid UTF-8, or transcoding bug?");
57     I += UTF8Length; // Skip over all trailing bytes.
58     // A codepoint takes two UTF-16 code unit if it's astral (outside BMP).
59     // Astral codepoints are encoded as 4 bytes in UTF-8 (11110xxx ...)
60     if (CB(UTF8Length, UTF8Length == 4 ? 2 : 1))
61       return true;
62   }
63   return false;
64 }
65 
66 // Returns the byte offset into the string that is an offset of \p Units in
67 // the specified encoding.
68 // Conceptually, this converts to the encoding, truncates to CodeUnits,
69 // converts back to UTF-8, and returns the length in bytes.
70 static size_t measureUnits(llvm::StringRef U8, int Units, OffsetEncoding Enc,
71                            bool &Valid) {
72   Valid = Units >= 0;
73   if (Units <= 0)
74     return 0;
75   size_t Result = 0;
76   switch (Enc) {
77   case OffsetEncoding::UTF8:
78     Result = Units;
79     break;
80   case OffsetEncoding::UTF16:
81     Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) {
82       Result += U8Len;
83       Units -= U16Len;
84       return Units <= 0;
85     });
86     if (Units < 0) // Offset in the middle of a surrogate pair.
87       Valid = false;
88     break;
89   case OffsetEncoding::UTF32:
90     Valid = iterateCodepoints(U8, [&](int U8Len, int U16Len) {
91       Result += U8Len;
92       Units--;
93       return Units <= 0;
94     });
95     break;
96   case OffsetEncoding::UnsupportedEncoding:
97     llvm_unreachable("unsupported encoding");
98   }
99   // Don't return an out-of-range index if we overran.
100   if (Result > U8.size()) {
101     Valid = false;
102     return U8.size();
103   }
104   return Result;
105 }
106 
107 Key<OffsetEncoding> kCurrentOffsetEncoding;
108 static OffsetEncoding lspEncoding() {
109   auto *Enc = Context::current().get(kCurrentOffsetEncoding);
110   return Enc ? *Enc : OffsetEncoding::UTF16;
111 }
112 
113 // Like most strings in clangd, the input is UTF-8 encoded.
114 size_t lspLength(llvm::StringRef Code) {
115   size_t Count = 0;
116   switch (lspEncoding()) {
117   case OffsetEncoding::UTF8:
118     Count = Code.size();
119     break;
120   case OffsetEncoding::UTF16:
121     iterateCodepoints(Code, [&](int U8Len, int U16Len) {
122       Count += U16Len;
123       return false;
124     });
125     break;
126   case OffsetEncoding::UTF32:
127     iterateCodepoints(Code, [&](int U8Len, int U16Len) {
128       ++Count;
129       return false;
130     });
131     break;
132   case OffsetEncoding::UnsupportedEncoding:
133     llvm_unreachable("unsupported encoding");
134   }
135   return Count;
136 }
137 
138 llvm::Expected<size_t> positionToOffset(llvm::StringRef Code, Position P,
139                                         bool AllowColumnsBeyondLineLength) {
140   if (P.line < 0)
141     return llvm::make_error<llvm::StringError>(
142         llvm::formatv("Line value can't be negative ({0})", P.line),
143         llvm::errc::invalid_argument);
144   if (P.character < 0)
145     return llvm::make_error<llvm::StringError>(
146         llvm::formatv("Character value can't be negative ({0})", P.character),
147         llvm::errc::invalid_argument);
148   size_t StartOfLine = 0;
149   for (int I = 0; I != P.line; ++I) {
150     size_t NextNL = Code.find('\n', StartOfLine);
151     if (NextNL == llvm::StringRef::npos)
152       return llvm::make_error<llvm::StringError>(
153           llvm::formatv("Line value is out of range ({0})", P.line),
154           llvm::errc::invalid_argument);
155     StartOfLine = NextNL + 1;
156   }
157   StringRef Line =
158       Code.substr(StartOfLine).take_until([](char C) { return C == '\n'; });
159 
160   // P.character may be in UTF-16, transcode if necessary.
161   bool Valid;
162   size_t ByteInLine = measureUnits(Line, P.character, lspEncoding(), Valid);
163   if (!Valid && !AllowColumnsBeyondLineLength)
164     return llvm::make_error<llvm::StringError>(
165         llvm::formatv("{0} offset {1} is invalid for line {2}", lspEncoding(),
166                       P.character, P.line),
167         llvm::errc::invalid_argument);
168   return StartOfLine + ByteInLine;
169 }
170 
171 Position offsetToPosition(llvm::StringRef Code, size_t Offset) {
172   Offset = std::min(Code.size(), Offset);
173   llvm::StringRef Before = Code.substr(0, Offset);
174   int Lines = Before.count('\n');
175   size_t PrevNL = Before.rfind('\n');
176   size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1);
177   Position Pos;
178   Pos.line = Lines;
179   Pos.character = lspLength(Before.substr(StartOfLine));
180   return Pos;
181 }
182 
183 Position sourceLocToPosition(const SourceManager &SM, SourceLocation Loc) {
184   // We use the SourceManager's line tables, but its column number is in bytes.
185   FileID FID;
186   unsigned Offset;
187   std::tie(FID, Offset) = SM.getDecomposedSpellingLoc(Loc);
188   Position P;
189   P.line = static_cast<int>(SM.getLineNumber(FID, Offset)) - 1;
190   bool Invalid = false;
191   llvm::StringRef Code = SM.getBufferData(FID, &Invalid);
192   if (!Invalid) {
193     auto ColumnInBytes = SM.getColumnNumber(FID, Offset) - 1;
194     auto LineSoFar = Code.substr(Offset - ColumnInBytes, ColumnInBytes);
195     P.character = lspLength(LineSoFar);
196   }
197   return P;
198 }
199 
200 llvm::Optional<Range> getTokenRange(const SourceManager &SM,
201                                     const LangOptions &LangOpts,
202                                     SourceLocation TokLoc) {
203   if (!TokLoc.isValid())
204     return llvm::None;
205   SourceLocation End = Lexer::getLocForEndOfToken(TokLoc, 0, SM, LangOpts);
206   if (!End.isValid())
207     return llvm::None;
208   return halfOpenToRange(SM, CharSourceRange::getCharRange(TokLoc, End));
209 }
210 
211 bool isValidFileRange(const SourceManager &Mgr, SourceRange R) {
212   if (!R.getBegin().isValid() || !R.getEnd().isValid())
213     return false;
214 
215   FileID BeginFID;
216   size_t BeginOffset = 0;
217   std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin());
218 
219   FileID EndFID;
220   size_t EndOffset = 0;
221   std::tie(EndFID, EndOffset) = Mgr.getDecomposedLoc(R.getEnd());
222 
223   return BeginFID.isValid() && BeginFID == EndFID && BeginOffset <= EndOffset;
224 }
225 
226 bool halfOpenRangeContains(const SourceManager &Mgr, SourceRange R,
227                            SourceLocation L) {
228   assert(isValidFileRange(Mgr, R));
229 
230   FileID BeginFID;
231   size_t BeginOffset = 0;
232   std::tie(BeginFID, BeginOffset) = Mgr.getDecomposedLoc(R.getBegin());
233   size_t EndOffset = Mgr.getFileOffset(R.getEnd());
234 
235   FileID LFid;
236   size_t LOffset;
237   std::tie(LFid, LOffset) = Mgr.getDecomposedLoc(L);
238   return BeginFID == LFid && BeginOffset <= LOffset && LOffset < EndOffset;
239 }
240 
241 bool halfOpenRangeTouches(const SourceManager &Mgr, SourceRange R,
242                           SourceLocation L) {
243   return L == R.getEnd() || halfOpenRangeContains(Mgr, R, L);
244 }
245 
246 llvm::Optional<SourceRange> toHalfOpenFileRange(const SourceManager &Mgr,
247                                                 const LangOptions &LangOpts,
248                                                 SourceRange R) {
249   auto Begin = Mgr.getFileLoc(R.getBegin());
250   if (Begin.isInvalid())
251     return llvm::None;
252   auto End = Mgr.getFileLoc(R.getEnd());
253   if (End.isInvalid())
254     return llvm::None;
255   End = Lexer::getLocForEndOfToken(End, 0, Mgr, LangOpts);
256 
257   SourceRange Result(Begin, End);
258   if (!isValidFileRange(Mgr, Result))
259     return llvm::None;
260   return Result;
261 }
262 
263 llvm::StringRef toSourceCode(const SourceManager &SM, SourceRange R) {
264   assert(isValidFileRange(SM, R));
265   bool Invalid = false;
266   auto *Buf = SM.getBuffer(SM.getFileID(R.getBegin()), &Invalid);
267   assert(!Invalid);
268 
269   size_t BeginOffset = SM.getFileOffset(R.getBegin());
270   size_t EndOffset = SM.getFileOffset(R.getEnd());
271   return Buf->getBuffer().substr(BeginOffset, EndOffset - BeginOffset);
272 }
273 
274 llvm::Expected<SourceLocation> sourceLocationInMainFile(const SourceManager &SM,
275                                                         Position P) {
276   llvm::StringRef Code = SM.getBuffer(SM.getMainFileID())->getBuffer();
277   auto Offset =
278       positionToOffset(Code, P, /*AllowColumnBeyondLineLength=*/false);
279   if (!Offset)
280     return Offset.takeError();
281   return SM.getLocForStartOfFile(SM.getMainFileID()).getLocWithOffset(*Offset);
282 }
283 
284 Range halfOpenToRange(const SourceManager &SM, CharSourceRange R) {
285   // Clang is 1-based, LSP uses 0-based indexes.
286   Position Begin = sourceLocToPosition(SM, R.getBegin());
287   Position End = sourceLocToPosition(SM, R.getEnd());
288 
289   return {Begin, End};
290 }
291 
292 std::pair<size_t, size_t> offsetToClangLineColumn(llvm::StringRef Code,
293                                                   size_t Offset) {
294   Offset = std::min(Code.size(), Offset);
295   llvm::StringRef Before = Code.substr(0, Offset);
296   int Lines = Before.count('\n');
297   size_t PrevNL = Before.rfind('\n');
298   size_t StartOfLine = (PrevNL == llvm::StringRef::npos) ? 0 : (PrevNL + 1);
299   return {Lines + 1, Offset - StartOfLine + 1};
300 }
301 
302 std::pair<StringRef, StringRef> splitQualifiedName(StringRef QName) {
303   size_t Pos = QName.rfind("::");
304   if (Pos == llvm::StringRef::npos)
305     return {llvm::StringRef(), QName};
306   return {QName.substr(0, Pos + 2), QName.substr(Pos + 2)};
307 }
308 
309 TextEdit replacementToEdit(llvm::StringRef Code,
310                            const tooling::Replacement &R) {
311   Range ReplacementRange = {
312       offsetToPosition(Code, R.getOffset()),
313       offsetToPosition(Code, R.getOffset() + R.getLength())};
314   return {ReplacementRange, R.getReplacementText()};
315 }
316 
317 std::vector<TextEdit> replacementsToEdits(llvm::StringRef Code,
318                                           const tooling::Replacements &Repls) {
319   std::vector<TextEdit> Edits;
320   for (const auto &R : Repls)
321     Edits.push_back(replacementToEdit(Code, R));
322   return Edits;
323 }
324 
325 llvm::Optional<std::string> getCanonicalPath(const FileEntry *F,
326                                              const SourceManager &SourceMgr) {
327   if (!F)
328     return None;
329 
330   llvm::SmallString<128> FilePath = F->getName();
331   if (!llvm::sys::path::is_absolute(FilePath)) {
332     if (auto EC =
333             SourceMgr.getFileManager().getVirtualFileSystem().makeAbsolute(
334                 FilePath)) {
335       elog("Could not turn relative path '{0}' to absolute: {1}", FilePath,
336            EC.message());
337       return None;
338     }
339   }
340 
341   // Handle the symbolic link path case where the current working directory
342   // (getCurrentWorkingDirectory) is a symlink./ We always want to the real
343   // file path (instead of the symlink path) for the  C++ symbols.
344   //
345   // Consider the following example:
346   //
347   //   src dir: /project/src/foo.h
348   //   current working directory (symlink): /tmp/build -> /project/src/
349   //
350   //  The file path of Symbol is "/project/src/foo.h" instead of
351   //  "/tmp/build/foo.h"
352   if (const DirectoryEntry *Dir = SourceMgr.getFileManager().getDirectory(
353           llvm::sys::path::parent_path(FilePath))) {
354     llvm::SmallString<128> RealPath;
355     llvm::StringRef DirName = SourceMgr.getFileManager().getCanonicalName(Dir);
356     llvm::sys::path::append(RealPath, DirName,
357                             llvm::sys::path::filename(FilePath));
358     return RealPath.str().str();
359   }
360 
361   return FilePath.str().str();
362 }
363 
364 TextEdit toTextEdit(const FixItHint &FixIt, const SourceManager &M,
365                     const LangOptions &L) {
366   TextEdit Result;
367   Result.range =
368       halfOpenToRange(M, Lexer::makeFileCharRange(FixIt.RemoveRange, M, L));
369   Result.newText = FixIt.CodeToInsert;
370   return Result;
371 }
372 
373 bool isRangeConsecutive(const Range &Left, const Range &Right) {
374   return Left.end.line == Right.start.line &&
375          Left.end.character == Right.start.character;
376 }
377 
378 FileDigest digest(llvm::StringRef Content) {
379   return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
380 }
381 
382 llvm::Optional<FileDigest> digestFile(const SourceManager &SM, FileID FID) {
383   bool Invalid = false;
384   llvm::StringRef Content = SM.getBufferData(FID, &Invalid);
385   if (Invalid)
386     return None;
387   return digest(Content);
388 }
389 
390 format::FormatStyle getFormatStyleForFile(llvm::StringRef File,
391                                           llvm::StringRef Content,
392                                           llvm::vfs::FileSystem *FS) {
393   auto Style = format::getStyle(format::DefaultFormatStyle, File,
394                                 format::DefaultFallbackStyle, Content, FS);
395   if (!Style) {
396     log("getStyle() failed for file {0}: {1}. Fallback is LLVM style.", File,
397         Style.takeError());
398     Style = format::getLLVMStyle();
399   }
400   return *Style;
401 }
402 
403 llvm::Expected<tooling::Replacements>
404 cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces,
405                  const format::FormatStyle &Style) {
406   auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style);
407   if (!CleanReplaces)
408     return CleanReplaces;
409   return formatReplacements(Code, std::move(*CleanReplaces), Style);
410 }
411 
412 template <typename Action>
413 static void lex(llvm::StringRef Code, const format::FormatStyle &Style,
414                 Action A) {
415   // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated!
416   std::string NullTerminatedCode = Code.str();
417   SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode);
418   auto &SM = FileSM.get();
419   auto FID = SM.getMainFileID();
420   Lexer Lex(FID, SM.getBuffer(FID), SM, format::getFormattingLangOpts(Style));
421   Token Tok;
422 
423   while (!Lex.LexFromRawLexer(Tok))
424     A(Tok);
425 }
426 
427 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
428                                              const format::FormatStyle &Style) {
429   llvm::StringMap<unsigned> Identifiers;
430   lex(Content, Style, [&](const clang::Token &Tok) {
431     switch (Tok.getKind()) {
432     case tok::identifier:
433       ++Identifiers[Tok.getIdentifierInfo()->getName()];
434       break;
435     case tok::raw_identifier:
436       ++Identifiers[Tok.getRawIdentifier()];
437       break;
438     default:
439       break;
440     }
441   });
442   return Identifiers;
443 }
444 
445 namespace {
446 enum NamespaceEvent {
447   BeginNamespace, // namespace <ns> {.     Payload is resolved <ns>.
448   EndNamespace,   // } // namespace <ns>.  Payload is resolved *outer* namespace.
449   UsingDirective  // using namespace <ns>. Payload is unresolved <ns>.
450 };
451 // Scans C++ source code for constructs that change the visible namespaces.
452 void parseNamespaceEvents(
453     llvm::StringRef Code, const format::FormatStyle &Style,
454     llvm::function_ref<void(NamespaceEvent, llvm::StringRef)> Callback) {
455 
456   // Stack of enclosing namespaces, e.g. {"clang", "clangd"}
457   std::vector<std::string> Enclosing; // Contains e.g. "clang", "clangd"
458   // Stack counts open braces. true if the brace opened a namespace.
459   std::vector<bool> BraceStack;
460 
461   enum {
462     Default,
463     Namespace,          // just saw 'namespace'
464     NamespaceName,      // just saw 'namespace' NSName
465     Using,              // just saw 'using'
466     UsingNamespace,     // just saw 'using namespace'
467     UsingNamespaceName, // just saw 'using namespace' NSName
468   } State = Default;
469   std::string NSName;
470 
471   lex(Code, Style, [&](const clang::Token &Tok) {
472     switch(Tok.getKind()) {
473     case tok::raw_identifier:
474       // In raw mode, this could be a keyword or a name.
475       switch (State) {
476       case UsingNamespace:
477       case UsingNamespaceName:
478         NSName.append(Tok.getRawIdentifier());
479         State = UsingNamespaceName;
480         break;
481       case Namespace:
482       case NamespaceName:
483         NSName.append(Tok.getRawIdentifier());
484         State = NamespaceName;
485         break;
486       case Using:
487         State =
488             (Tok.getRawIdentifier() == "namespace") ? UsingNamespace : Default;
489         break;
490       case Default:
491         NSName.clear();
492         if (Tok.getRawIdentifier() == "namespace")
493           State = Namespace;
494         else if (Tok.getRawIdentifier() == "using")
495           State = Using;
496         break;
497       }
498       break;
499     case tok::coloncolon:
500       // This can come at the beginning or in the middle of a namespace name.
501       switch (State) {
502       case UsingNamespace:
503       case UsingNamespaceName:
504         NSName.append("::");
505         State = UsingNamespaceName;
506         break;
507       case NamespaceName:
508         NSName.append("::");
509         State = NamespaceName;
510         break;
511       case Namespace: // Not legal here.
512       case Using:
513       case Default:
514         State = Default;
515         break;
516       }
517       break;
518     case tok::l_brace:
519       // Record which { started a namespace, so we know when } ends one.
520       if (State == NamespaceName) {
521         // Parsed: namespace <name> {
522         BraceStack.push_back(true);
523         Enclosing.push_back(NSName);
524         Callback(BeginNamespace, llvm::join(Enclosing, "::"));
525       } else {
526         // This case includes anonymous namespaces (State = Namespace).
527         // For our purposes, they're not namespaces and we ignore them.
528         BraceStack.push_back(false);
529       }
530       State = Default;
531       break;
532     case tok::r_brace:
533       // If braces are unmatched, we're going to be confused, but don't crash.
534       if (!BraceStack.empty()) {
535         if (BraceStack.back()) {
536           // Parsed: } // namespace
537           Enclosing.pop_back();
538           Callback(EndNamespace, llvm::join(Enclosing, "::"));
539         }
540         BraceStack.pop_back();
541       }
542       break;
543     case tok::semi:
544       if (State == UsingNamespaceName)
545         // Parsed: using namespace <name> ;
546         Callback(UsingDirective, llvm::StringRef(NSName));
547       State = Default;
548       break;
549     default:
550       State = Default;
551       break;
552     }
553   });
554 }
555 
556 // Returns the prefix namespaces of NS: {"" ... NS}.
557 llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) {
558   llvm::SmallVector<llvm::StringRef, 8> Results;
559   Results.push_back(NS.take_front(0));
560   NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
561   for (llvm::StringRef &R : Results)
562     R = NS.take_front(R.end() - NS.begin());
563   return Results;
564 }
565 
566 } // namespace
567 
568 std::vector<std::string> visibleNamespaces(llvm::StringRef Code,
569                                            const format::FormatStyle &Style) {
570   std::string Current;
571   // Map from namespace to (resolved) namespaces introduced via using directive.
572   llvm::StringMap<llvm::StringSet<>> UsingDirectives;
573 
574   parseNamespaceEvents(Code, Style,
575                        [&](NamespaceEvent Event, llvm::StringRef NS) {
576                          switch (Event) {
577                          case BeginNamespace:
578                          case EndNamespace:
579                            Current = NS;
580                            break;
581                          case UsingDirective:
582                            if (NS.consume_front("::"))
583                              UsingDirectives[Current].insert(NS);
584                            else {
585                              for (llvm::StringRef Enclosing :
586                                   ancestorNamespaces(Current)) {
587                                if (Enclosing.empty())
588                                  UsingDirectives[Current].insert(NS);
589                                else
590                                  UsingDirectives[Current].insert(
591                                      (Enclosing + "::" + NS).str());
592                              }
593                            }
594                            break;
595                          }
596                        });
597 
598   std::vector<std::string> Found;
599   for (llvm::StringRef Enclosing : ancestorNamespaces(Current)) {
600     Found.push_back(Enclosing);
601     auto It = UsingDirectives.find(Enclosing);
602     if (It != UsingDirectives.end())
603       for (const auto& Used : It->second)
604         Found.push_back(Used.getKey());
605   }
606 
607 
608   llvm::sort(Found, [&](const std::string &LHS, const std::string &RHS) {
609     if (Current == RHS)
610       return false;
611     if (Current == LHS)
612       return true;
613     return LHS < RHS;
614   });
615   Found.erase(std::unique(Found.begin(), Found.end()), Found.end());
616   return Found;
617 }
618 
619 llvm::StringSet<> collectWords(llvm::StringRef Content) {
620   // We assume short words are not significant.
621   // We may want to consider other stopwords, e.g. language keywords.
622   // (A very naive implementation showed no benefit, but lexing might do better)
623   static constexpr int MinWordLength = 4;
624 
625   std::vector<CharRole> Roles(Content.size());
626   calculateRoles(Content, Roles);
627 
628   llvm::StringSet<> Result;
629   llvm::SmallString<256> Word;
630   auto Flush = [&] {
631     if (Word.size() >= MinWordLength) {
632       for (char &C : Word)
633         C = llvm::toLower(C);
634       Result.insert(Word);
635     }
636     Word.clear();
637   };
638   for (unsigned I = 0; I < Content.size(); ++I) {
639     switch (Roles[I]) {
640     case Head:
641       Flush();
642       LLVM_FALLTHROUGH;
643     case Tail:
644       Word.push_back(Content[I]);
645       break;
646     case Unknown:
647     case Separator:
648       Flush();
649       break;
650     }
651   }
652   Flush();
653 
654   return Result;
655 }
656 
657 llvm::Optional<DefinedMacro> locateMacroAt(SourceLocation Loc,
658                                            Preprocessor &PP) {
659   const auto &SM = PP.getSourceManager();
660   const auto &LangOpts = PP.getLangOpts();
661   Token Result;
662   if (Lexer::getRawToken(SM.getSpellingLoc(Loc), Result, SM, LangOpts, false))
663     return None;
664   if (Result.is(tok::raw_identifier))
665     PP.LookUpIdentifierInfo(Result);
666   IdentifierInfo *IdentifierInfo = Result.getIdentifierInfo();
667   if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition())
668     return None;
669 
670   std::pair<FileID, unsigned int> DecLoc = SM.getDecomposedExpansionLoc(Loc);
671   // Get the definition just before the searched location so that a macro
672   // referenced in a '#undef MACRO' can still be found.
673   SourceLocation BeforeSearchedLocation =
674       SM.getMacroArgExpandedLocation(SM.getLocForStartOfFile(DecLoc.first)
675                                          .getLocWithOffset(DecLoc.second - 1));
676   MacroDefinition MacroDef =
677       PP.getMacroDefinitionAtLoc(IdentifierInfo, BeforeSearchedLocation);
678   if (auto *MI = MacroDef.getMacroInfo())
679     return DefinedMacro{IdentifierInfo->getName(), MI};
680   return None;
681 }
682 
683 } // namespace clangd
684 } // namespace clang
685