1 //===--- Check.cpp - clangd self-diagnostics ------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Many basic problems can occur processing a file in clangd, e.g.:
10 //  - system includes are not found
11 //  - crash when indexing its AST
12 // clangd --check provides a simplified, isolated way to reproduce these,
13 // with no editor, LSP, threads, background indexing etc to contend with.
14 //
15 // One important use case is gathering information for bug reports.
16 // Another is reproducing crashes, and checking which setting prevent them.
17 //
18 // It simulates opening a file (determining compile command, parsing, indexing)
19 // and then running features at many locations.
20 //
21 // Currently it adds some basic logging of progress and results.
22 // We should consider extending it to also recognize common symptoms and
23 // recommend solutions (e.g. standard library installation issues).
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "ClangdLSPServer.h"
28 #include "CodeComplete.h"
29 #include "CompileCommands.h"
30 #include "Config.h"
31 #include "GlobalCompilationDatabase.h"
32 #include "Hover.h"
33 #include "InlayHints.h"
34 #include "ParsedAST.h"
35 #include "Preamble.h"
36 #include "Protocol.h"
37 #include "SourceCode.h"
38 #include "XRefs.h"
39 #include "index/CanonicalIncludes.h"
40 #include "index/FileIndex.h"
41 #include "refactor/Tweak.h"
42 #include "support/ThreadsafeFS.h"
43 #include "support/Trace.h"
44 #include "clang/AST/ASTContext.h"
45 #include "clang/Format/Format.h"
46 #include "clang/Frontend/CompilerInvocation.h"
47 #include "clang/Tooling/CompilationDatabase.h"
48 #include "llvm/ADT/ArrayRef.h"
49 #include "llvm/ADT/Optional.h"
50 #include "llvm/Support/Path.h"
51 
52 namespace clang {
53 namespace clangd {
54 namespace {
55 
56 // Print (and count) the error-level diagnostics (warnings are ignored).
showErrors(llvm::ArrayRef<Diag> Diags)57 unsigned showErrors(llvm::ArrayRef<Diag> Diags) {
58   unsigned ErrCount = 0;
59   for (const auto &D : Diags) {
60     if (D.Severity >= DiagnosticsEngine::Error) {
61       elog("[{0}] Line {1}: {2}", D.Name, D.Range.start.line + 1, D.Message);
62       ++ErrCount;
63     }
64   }
65   return ErrCount;
66 }
67 
68 // This class is just a linear pipeline whose functions get called in sequence.
69 // Each exercises part of clangd's logic on our test file and logs results.
70 // Later steps depend on state built in earlier ones (such as the AST).
71 // Many steps can fatally fail (return false), then subsequent ones cannot run.
72 // Nonfatal failures are logged and tracked in ErrCount.
73 class Checker {
74   // from constructor
75   std::string File;
76   ClangdLSPServer::Options Opts;
77   // from buildCommand
78   tooling::CompileCommand Cmd;
79   // from buildInvocation
80   ParseInputs Inputs;
81   std::unique_ptr<CompilerInvocation> Invocation;
82   format::FormatStyle Style;
83   // from buildAST
84   std::shared_ptr<const PreambleData> Preamble;
85   llvm::Optional<ParsedAST> AST;
86   FileIndex Index;
87 
88 public:
89   // Number of non-fatal errors seen.
90   unsigned ErrCount = 0;
91 
Checker(llvm::StringRef File,const ClangdLSPServer::Options & Opts)92   Checker(llvm::StringRef File, const ClangdLSPServer::Options &Opts)
93       : File(File), Opts(Opts) {}
94 
95   // Read compilation database and choose a compile command for the file.
buildCommand(const ThreadsafeFS & TFS)96   bool buildCommand(const ThreadsafeFS &TFS) {
97     log("Loading compilation database...");
98     DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS);
99     CDBOpts.CompileCommandsDir =
100         Config::current().CompileFlags.CDBSearch.FixedCDBPath;
101     std::unique_ptr<GlobalCompilationDatabase> BaseCDB =
102         std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts);
103     BaseCDB = getQueryDriverDatabase(llvm::makeArrayRef(Opts.QueryDriverGlobs),
104                                      std::move(BaseCDB));
105     auto Mangler = CommandMangler::detect();
106     if (Opts.ResourceDir)
107       Mangler.ResourceDir = *Opts.ResourceDir;
108     auto CDB = std::make_unique<OverlayCDB>(
109         BaseCDB.get(), std::vector<std::string>{},
110         tooling::ArgumentsAdjuster(std::move(Mangler)));
111 
112     if (auto TrueCmd = CDB->getCompileCommand(File)) {
113       Cmd = std::move(*TrueCmd);
114       log("Compile command {0} is: {1}",
115           Cmd.Heuristic.empty() ? "from CDB" : Cmd.Heuristic,
116           printArgv(Cmd.CommandLine));
117     } else {
118       Cmd = CDB->getFallbackCommand(File);
119       log("Generic fallback command is: {0}", printArgv(Cmd.CommandLine));
120     }
121 
122     return true;
123   }
124 
125   // Prepare inputs and build CompilerInvocation (parsed compile command).
buildInvocation(const ThreadsafeFS & TFS,llvm::Optional<std::string> Contents)126   bool buildInvocation(const ThreadsafeFS &TFS,
127                        llvm::Optional<std::string> Contents) {
128     StoreDiags CaptureInvocationDiags;
129     std::vector<std::string> CC1Args;
130     Inputs.CompileCommand = Cmd;
131     Inputs.TFS = &TFS;
132     Inputs.ClangTidyProvider = Opts.ClangTidyProvider;
133     Inputs.Opts.PreambleParseForwardingFunctions =
134         Opts.PreambleParseForwardingFunctions;
135     if (Contents) {
136       Inputs.Contents = *Contents;
137       log("Imaginary source file contents:\n{0}", Inputs.Contents);
138     } else {
139       if (auto Contents = TFS.view(llvm::None)->getBufferForFile(File)) {
140         Inputs.Contents = Contents->get()->getBuffer().str();
141       } else {
142         elog("Couldn't read {0}: {1}", File, Contents.getError().message());
143         return false;
144       }
145     }
146     log("Parsing command...");
147     Invocation =
148         buildCompilerInvocation(Inputs, CaptureInvocationDiags, &CC1Args);
149     auto InvocationDiags = CaptureInvocationDiags.take();
150     ErrCount += showErrors(InvocationDiags);
151     log("internal (cc1) args are: {0}", printArgv(CC1Args));
152     if (!Invocation) {
153       elog("Failed to parse command line");
154       return false;
155     }
156 
157     // FIXME: Check that resource-dir/built-in-headers exist?
158 
159     Style = getFormatStyleForFile(File, Inputs.Contents, TFS);
160 
161     return true;
162   }
163 
164   // Build preamble and AST, and index them.
buildAST()165   bool buildAST() {
166     log("Building preamble...");
167     Preamble = buildPreamble(File, *Invocation, Inputs, /*StoreInMemory=*/true,
168                              [&](ASTContext &Ctx, Preprocessor &PP,
169                                  const CanonicalIncludes &Includes) {
170                                if (!Opts.BuildDynamicSymbolIndex)
171                                  return;
172                                log("Indexing headers...");
173                                Index.updatePreamble(File, /*Version=*/"null",
174                                                     Ctx, PP, Includes);
175                              });
176     if (!Preamble) {
177       elog("Failed to build preamble");
178       return false;
179     }
180     ErrCount += showErrors(Preamble->Diags);
181 
182     log("Building AST...");
183     AST = ParsedAST::build(File, Inputs, std::move(Invocation),
184                            /*InvocationDiags=*/std::vector<Diag>{}, Preamble);
185     if (!AST) {
186       elog("Failed to build AST");
187       return false;
188     }
189     ErrCount += showErrors(llvm::makeArrayRef(*AST->getDiagnostics())
190                                .drop_front(Preamble->Diags.size()));
191 
192     if (Opts.BuildDynamicSymbolIndex) {
193       log("Indexing AST...");
194       Index.updateMain(File, *AST);
195     }
196     return true;
197   }
198 
199   // Build Inlay Hints for the entire AST or the specified range
buildInlayHints(llvm::Optional<Range> LineRange)200   void buildInlayHints(llvm::Optional<Range> LineRange) {
201     log("Building inlay hints");
202     auto Hints = inlayHints(*AST, LineRange);
203 
204     for (const auto &Hint : Hints) {
205       vlog("  {0} {1} {2}", Hint.kind, Hint.position, Hint.label);
206     }
207   }
208 
209   // Run AST-based features at each token in the file.
testLocationFeatures(llvm::Optional<Range> LineRange,const bool EnableCodeCompletion)210   void testLocationFeatures(llvm::Optional<Range> LineRange,
211                             const bool EnableCodeCompletion) {
212     trace::Span Trace("testLocationFeatures");
213     log("Testing features at each token (may be slow in large files)");
214     auto &SM = AST->getSourceManager();
215     auto SpelledTokens = AST->getTokens().spelledTokens(SM.getMainFileID());
216 
217     CodeCompleteOptions CCOpts = Opts.CodeComplete;
218     CCOpts.Index = &Index;
219 
220     for (const auto &Tok : SpelledTokens) {
221       unsigned Start = AST->getSourceManager().getFileOffset(Tok.location());
222       unsigned End = Start + Tok.length();
223       Position Pos = offsetToPosition(Inputs.Contents, Start);
224 
225       if (LineRange && !LineRange->contains(Pos))
226         continue;
227 
228       trace::Span Trace("Token");
229       SPAN_ATTACH(Trace, "pos", Pos);
230       SPAN_ATTACH(Trace, "text", Tok.text(AST->getSourceManager()));
231 
232       // FIXME: dumping the tokens may leak sensitive code into bug reports.
233       // Add an option to turn this off, once we decide how options work.
234       vlog("  {0} {1}", Pos, Tok.text(AST->getSourceManager()));
235       auto Tree = SelectionTree::createRight(AST->getASTContext(),
236                                              AST->getTokens(), Start, End);
237       Tweak::Selection Selection(&Index, *AST, Start, End, std::move(Tree),
238                                  nullptr);
239       // FS is only populated when applying a tweak, not during prepare as
240       // prepare should not do any I/O to be fast.
241       auto Tweaks =
242           prepareTweaks(Selection, Opts.TweakFilter, Opts.FeatureModules);
243       Selection.FS =
244           &AST->getSourceManager().getFileManager().getVirtualFileSystem();
245       for (const auto &T : Tweaks) {
246         auto Result = T->apply(Selection);
247         if (!Result) {
248           elog("    tweak: {0} ==> FAIL: {1}", T->id(), Result.takeError());
249           ++ErrCount;
250         } else {
251           vlog("    tweak: {0}", T->id());
252         }
253       }
254       unsigned Definitions = locateSymbolAt(*AST, Pos, &Index).size();
255       vlog("    definition: {0}", Definitions);
256 
257       auto Hover = getHover(*AST, Pos, Style, &Index);
258       vlog("    hover: {0}", Hover.has_value());
259 
260       unsigned DocHighlights = findDocumentHighlights(*AST, Pos).size();
261       vlog("    documentHighlight: {0}", DocHighlights);
262 
263       if (EnableCodeCompletion) {
264         Position EndPos = offsetToPosition(Inputs.Contents, End);
265         auto CC = codeComplete(File, EndPos, Preamble.get(), Inputs, CCOpts);
266         vlog("    code completion: {0}",
267              CC.Completions.empty() ? "<empty>" : CC.Completions[0].Name);
268       }
269     }
270   }
271 };
272 
273 } // namespace
274 
check(llvm::StringRef File,llvm::Optional<Range> LineRange,const ThreadsafeFS & TFS,const ClangdLSPServer::Options & Opts,bool EnableCodeCompletion)275 bool check(llvm::StringRef File, llvm::Optional<Range> LineRange,
276            const ThreadsafeFS &TFS, const ClangdLSPServer::Options &Opts,
277            bool EnableCodeCompletion) {
278   llvm::SmallString<0> FakeFile;
279   llvm::Optional<std::string> Contents;
280   if (File.empty()) {
281     llvm::sys::path::system_temp_directory(false, FakeFile);
282     llvm::sys::path::append(FakeFile, "test.cc");
283     File = FakeFile;
284     Contents = R"cpp(
285       #include <stddef.h>
286       #include <string>
287 
288       size_t N = 50;
289       auto xxx = std::string(N, 'x');
290     )cpp";
291   }
292   log("Testing on source file {0}", File);
293 
294   auto ContextProvider = ClangdServer::createConfiguredContextProvider(
295       Opts.ConfigProvider, nullptr);
296   WithContext Ctx(ContextProvider(
297       FakeFile.empty()
298           ? File
299           : /*Don't turn on local configs for an arbitrary temp path.*/ ""));
300   Checker C(File, Opts);
301   if (!C.buildCommand(TFS) || !C.buildInvocation(TFS, Contents) ||
302       !C.buildAST())
303     return false;
304   C.buildInlayHints(LineRange);
305   C.testLocationFeatures(LineRange, EnableCodeCompletion);
306 
307   log("All checks completed, {0} errors", C.ErrCount);
308   return C.ErrCount == 0;
309 }
310 
311 } // namespace clangd
312 } // namespace clang
313