1 //===--- Check.cpp - clangd self-diagnostics ------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Many basic problems can occur processing a file in clangd, e.g.:
10 //  - system includes are not found
11 //  - crash when indexing its AST
12 // clangd --check provides a simplified, isolated way to reproduce these,
13 // with no editor, LSP, threads, background indexing etc to contend with.
14 //
15 // One important use case is gathering information for bug reports.
16 // Another is reproducing crashes, and checking which setting prevent them.
17 //
18 // It simulates opening a file (determining compile command, parsing, indexing)
19 // and then running features at many locations.
20 //
21 // Currently it adds some basic logging of progress and results.
22 // We should consider extending it to also recognize common symptoms and
23 // recommend solutions (e.g. standard library installation issues).
24 //
25 //===----------------------------------------------------------------------===//
26 
27 #include "ClangdLSPServer.h"
28 #include "CodeComplete.h"
29 #include "Config.h"
30 #include "GlobalCompilationDatabase.h"
31 #include "Hover.h"
32 #include "ParsedAST.h"
33 #include "Preamble.h"
34 #include "SourceCode.h"
35 #include "XRefs.h"
36 #include "index/CanonicalIncludes.h"
37 #include "index/FileIndex.h"
38 #include "refactor/Tweak.h"
39 #include "support/ThreadsafeFS.h"
40 #include "support/Trace.h"
41 #include "clang/AST/ASTContext.h"
42 #include "clang/Basic/DiagnosticIDs.h"
43 #include "clang/Format/Format.h"
44 #include "clang/Frontend/CompilerInvocation.h"
45 #include "clang/Tooling/CompilationDatabase.h"
46 #include "llvm/ADT/ArrayRef.h"
47 #include "llvm/ADT/Optional.h"
48 #include "llvm/ADT/StringExtras.h"
49 #include "llvm/Support/Path.h"
50 
51 namespace clang {
52 namespace clangd {
53 namespace {
54 
55 // Print (and count) the error-level diagnostics (warnings are ignored).
56 unsigned showErrors(llvm::ArrayRef<Diag> Diags) {
57   unsigned ErrCount = 0;
58   for (const auto &D : Diags) {
59     if (D.Severity >= DiagnosticsEngine::Error) {
60       elog("[{0}] Line {1}: {2}", D.Name, D.Range.start.line + 1, D.Message);
61       ++ErrCount;
62     }
63   }
64   return ErrCount;
65 }
66 
67 // This class is just a linear pipeline whose functions get called in sequence.
68 // Each exercises part of clangd's logic on our test file and logs results.
69 // Later steps depend on state built in earlier ones (such as the AST).
70 // Many steps can fatally fail (return false), then subsequent ones cannot run.
71 // Nonfatal failures are logged and tracked in ErrCount.
72 class Checker {
73   // from constructor
74   std::string File;
75   ClangdLSPServer::Options Opts;
76   // from buildCommand
77   tooling::CompileCommand Cmd;
78   // from buildInvocation
79   ParseInputs Inputs;
80   std::unique_ptr<CompilerInvocation> Invocation;
81   format::FormatStyle Style;
82   // from buildAST
83   std::shared_ptr<const PreambleData> Preamble;
84   llvm::Optional<ParsedAST> AST;
85   FileIndex Index;
86 
87 public:
88   // Number of non-fatal errors seen.
89   unsigned ErrCount = 0;
90 
91   Checker(llvm::StringRef File, const ClangdLSPServer::Options &Opts)
92       : File(File), Opts(Opts) {}
93 
94   // Read compilation database and choose a compile command for the file.
95   bool buildCommand(const ThreadsafeFS &TFS) {
96     log("Loading compilation database...");
97     DirectoryBasedGlobalCompilationDatabase::Options CDBOpts(TFS);
98     CDBOpts.CompileCommandsDir =
99         Config::current().CompileFlags.CDBSearch.FixedCDBPath;
100     std::unique_ptr<GlobalCompilationDatabase> BaseCDB =
101         std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts);
102     BaseCDB = getQueryDriverDatabase(llvm::makeArrayRef(Opts.QueryDriverGlobs),
103                                      std::move(BaseCDB));
104     auto Mangler = CommandMangler::detect();
105     if (Opts.ResourceDir)
106       Mangler.ResourceDir = *Opts.ResourceDir;
107     auto CDB = std::make_unique<OverlayCDB>(
108         BaseCDB.get(), std::vector<std::string>{},
109         tooling::ArgumentsAdjuster(std::move(Mangler)));
110 
111     if (auto TrueCmd = CDB->getCompileCommand(File)) {
112       Cmd = std::move(*TrueCmd);
113       log("Compile command from CDB is: {0}", printArgv(Cmd.CommandLine));
114     } else {
115       Cmd = CDB->getFallbackCommand(File);
116       log("Generic fallback command is: {0}", printArgv(Cmd.CommandLine));
117     }
118 
119     return true;
120   }
121 
122   // Prepare inputs and build CompilerInvocation (parsed compile command).
123   bool buildInvocation(const ThreadsafeFS &TFS,
124                        llvm::Optional<std::string> Contents) {
125     StoreDiags CaptureInvocationDiags;
126     std::vector<std::string> CC1Args;
127     Inputs.CompileCommand = Cmd;
128     Inputs.TFS = &TFS;
129     Inputs.ClangTidyProvider = Opts.ClangTidyProvider;
130     if (Contents.hasValue()) {
131       Inputs.Contents = *Contents;
132       log("Imaginary source file contents:\n{0}", Inputs.Contents);
133     } else {
134       if (auto Contents = TFS.view(llvm::None)->getBufferForFile(File)) {
135         Inputs.Contents = Contents->get()->getBuffer().str();
136       } else {
137         elog("Couldn't read {0}: {1}", File, Contents.getError().message());
138         return false;
139       }
140     }
141     log("Parsing command...");
142     Invocation =
143         buildCompilerInvocation(Inputs, CaptureInvocationDiags, &CC1Args);
144     auto InvocationDiags = CaptureInvocationDiags.take();
145     ErrCount += showErrors(InvocationDiags);
146     log("internal (cc1) args are: {0}", printArgv(CC1Args));
147     if (!Invocation) {
148       elog("Failed to parse command line");
149       return false;
150     }
151 
152     // FIXME: Check that resource-dir/built-in-headers exist?
153 
154     Style = getFormatStyleForFile(File, Inputs.Contents, TFS);
155 
156     return true;
157   }
158 
159   // Build preamble and AST, and index them.
160   bool buildAST() {
161     log("Building preamble...");
162     Preamble = buildPreamble(File, *Invocation, Inputs, /*StoreInMemory=*/true,
163                              [&](ASTContext &Ctx, Preprocessor &PP,
164                                  const CanonicalIncludes &Includes) {
165                                if (!Opts.BuildDynamicSymbolIndex)
166                                  return;
167                                log("Indexing headers...");
168                                Index.updatePreamble(File, /*Version=*/"null",
169                                                     Ctx, PP, Includes);
170                              });
171     if (!Preamble) {
172       elog("Failed to build preamble");
173       return false;
174     }
175     ErrCount += showErrors(Preamble->Diags);
176 
177     log("Building AST...");
178     AST = ParsedAST::build(File, Inputs, std::move(Invocation),
179                            /*InvocationDiags=*/std::vector<Diag>{}, Preamble);
180     if (!AST) {
181       elog("Failed to build AST");
182       return false;
183     }
184     ErrCount += showErrors(llvm::makeArrayRef(*AST->getDiagnostics())
185                                .drop_front(Preamble->Diags.size()));
186 
187     if (Opts.BuildDynamicSymbolIndex) {
188       log("Indexing AST...");
189       Index.updateMain(File, *AST);
190     }
191     return true;
192   }
193 
194   // Run AST-based features at each token in the file.
195   void testLocationFeatures(
196       llvm::function_ref<bool(const Position &)> ShouldCheckLine,
197       const bool EnableCodeCompletion) {
198     trace::Span Trace("testLocationFeatures");
199     log("Testing features at each token (may be slow in large files)");
200     auto &SM = AST->getSourceManager();
201     auto SpelledTokens = AST->getTokens().spelledTokens(SM.getMainFileID());
202 
203     CodeCompleteOptions CCOpts = Opts.CodeComplete;
204     CCOpts.Index = &Index;
205 
206     for (const auto &Tok : SpelledTokens) {
207       unsigned Start = AST->getSourceManager().getFileOffset(Tok.location());
208       unsigned End = Start + Tok.length();
209       Position Pos = offsetToPosition(Inputs.Contents, Start);
210 
211       if (!ShouldCheckLine(Pos))
212         continue;
213 
214       trace::Span Trace("Token");
215       SPAN_ATTACH(Trace, "pos", Pos);
216       SPAN_ATTACH(Trace, "text", Tok.text(AST->getSourceManager()));
217 
218       // FIXME: dumping the tokens may leak sensitive code into bug reports.
219       // Add an option to turn this off, once we decide how options work.
220       vlog("  {0} {1}", Pos, Tok.text(AST->getSourceManager()));
221       auto Tree = SelectionTree::createRight(AST->getASTContext(),
222                                              AST->getTokens(), Start, End);
223       Tweak::Selection Selection(&Index, *AST, Start, End, std::move(Tree),
224                                  nullptr);
225       // FS is only populated when applying a tweak, not during prepare as
226       // prepare should not do any I/O to be fast.
227       auto Tweaks =
228           prepareTweaks(Selection, Opts.TweakFilter, Opts.FeatureModules);
229       Selection.FS =
230           &AST->getSourceManager().getFileManager().getVirtualFileSystem();
231       for (const auto &T : Tweaks) {
232         auto Result = T->apply(Selection);
233         if (!Result) {
234           elog("    tweak: {0} ==> FAIL: {1}", T->id(), Result.takeError());
235           ++ErrCount;
236         } else {
237           vlog("    tweak: {0}", T->id());
238         }
239       }
240       unsigned Definitions = locateSymbolAt(*AST, Pos, &Index).size();
241       vlog("    definition: {0}", Definitions);
242 
243       auto Hover = getHover(*AST, Pos, Style, &Index);
244       vlog("    hover: {0}", Hover.hasValue());
245 
246       if (EnableCodeCompletion) {
247         Position EndPos = offsetToPosition(Inputs.Contents, End);
248         auto CC = codeComplete(File, EndPos, Preamble.get(), Inputs, CCOpts);
249         vlog("    code completion: {0}",
250              CC.Completions.empty() ? "<empty>" : CC.Completions[0].Name);
251       }
252     }
253   }
254 };
255 
256 } // namespace
257 
258 bool check(llvm::StringRef File,
259            llvm::function_ref<bool(const Position &)> ShouldCheckLine,
260            const ThreadsafeFS &TFS, const ClangdLSPServer::Options &Opts,
261            bool EnableCodeCompletion) {
262   llvm::SmallString<0> FakeFile;
263   llvm::Optional<std::string> Contents;
264   if (File.empty()) {
265     llvm::sys::path::system_temp_directory(false, FakeFile);
266     llvm::sys::path::append(FakeFile, "test.cc");
267     File = FakeFile;
268     Contents = R"cpp(
269       #include <stddef.h>
270       #include <string>
271 
272       size_t N = 50;
273       auto xxx = std::string(N, 'x');
274     )cpp";
275   }
276   log("Testing on source file {0}", File);
277 
278   auto ContextProvider = ClangdServer::createConfiguredContextProvider(
279       Opts.ConfigProvider, nullptr);
280   WithContext Ctx(ContextProvider(
281       FakeFile.empty()
282           ? File
283           : /*Don't turn on local configs for an arbitrary temp path.*/ ""));
284   Checker C(File, Opts);
285   if (!C.buildCommand(TFS) || !C.buildInvocation(TFS, Contents) ||
286       !C.buildAST())
287     return false;
288   C.testLocationFeatures(ShouldCheckLine, EnableCodeCompletion);
289 
290   log("All checks completed, {0} errors", C.ErrCount);
291   return C.ErrCount == 0;
292 }
293 
294 } // namespace clangd
295 } // namespace clang
296