1 //===-- Reproducer.cpp ----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "lldb/Utility/Reproducer.h"
10 #include "lldb/Utility/LLDBAssert.h"
11 #include "lldb/Utility/ReproducerProvider.h"
12 #include "lldb/Utility/Timer.h"
13 
14 #include "llvm/Support/FileSystem.h"
15 #include "llvm/Support/Threading.h"
16 #include "llvm/Support/raw_ostream.h"
17 
18 using namespace lldb_private;
19 using namespace lldb_private::repro;
20 using namespace llvm;
21 using namespace llvm::yaml;
22 
23 static llvm::Optional<bool> GetEnv(const char *var) {
24   std::string val = llvm::StringRef(getenv(var)).lower();
25   if (val == "0" || val == "off")
26     return false;
27   if (val == "1" || val == "on")
28     return true;
29   return {};
30 }
31 
32 Reproducer &Reproducer::Instance() { return *InstanceImpl(); }
33 
34 llvm::Error Reproducer::Initialize(ReproducerMode mode,
35                                    llvm::Optional<FileSpec> root) {
36   lldbassert(!InstanceImpl() && "Already initialized.");
37   InstanceImpl().emplace();
38 
39   // The environment can override the capture mode.
40   if (mode != ReproducerMode::Replay) {
41     if (llvm::Optional<bool> override = GetEnv("LLDB_CAPTURE_REPRODUCER")) {
42       if (*override)
43         mode = ReproducerMode::Capture;
44       else
45         mode = ReproducerMode::Off;
46     }
47   }
48 
49   switch (mode) {
50   case ReproducerMode::Capture: {
51     if (!root) {
52       SmallString<128> repro_dir;
53       auto ec = sys::fs::createUniqueDirectory("reproducer", repro_dir);
54       if (ec)
55         return make_error<StringError>(
56             "unable to create unique reproducer directory", ec);
57       root.emplace(repro_dir);
58     } else {
59       auto ec = sys::fs::create_directory(root->GetPath());
60       if (ec)
61         return make_error<StringError>("unable to create reproducer directory",
62                                        ec);
63     }
64     return Instance().SetCapture(root);
65   } break;
66   case ReproducerMode::Replay:
67     return Instance().SetReplay(root, /*passive*/ false);
68   case ReproducerMode::PassiveReplay:
69     return Instance().SetReplay(root, /*passive*/ true);
70   case ReproducerMode::Off:
71     break;
72   };
73 
74   return Error::success();
75 }
76 
77 void Reproducer::Initialize() {
78   llvm::cantFail(Initialize(repro::ReproducerMode::Off, llvm::None));
79 }
80 
81 bool Reproducer::Initialized() { return InstanceImpl().operator bool(); }
82 
83 void Reproducer::Terminate() {
84   lldbassert(InstanceImpl() && "Already terminated.");
85   InstanceImpl().reset();
86 }
87 
88 Optional<Reproducer> &Reproducer::InstanceImpl() {
89   static Optional<Reproducer> g_reproducer;
90   return g_reproducer;
91 }
92 
93 const Generator *Reproducer::GetGenerator() const {
94   std::lock_guard<std::mutex> guard(m_mutex);
95   if (m_generator)
96     return &(*m_generator);
97   return nullptr;
98 }
99 
100 const Loader *Reproducer::GetLoader() const {
101   std::lock_guard<std::mutex> guard(m_mutex);
102   if (m_loader)
103     return &(*m_loader);
104   return nullptr;
105 }
106 
107 Generator *Reproducer::GetGenerator() {
108   std::lock_guard<std::mutex> guard(m_mutex);
109   if (m_generator)
110     return &(*m_generator);
111   return nullptr;
112 }
113 
114 Loader *Reproducer::GetLoader() {
115   std::lock_guard<std::mutex> guard(m_mutex);
116   if (m_loader)
117     return &(*m_loader);
118   return nullptr;
119 }
120 
121 llvm::Error Reproducer::SetCapture(llvm::Optional<FileSpec> root) {
122   std::lock_guard<std::mutex> guard(m_mutex);
123 
124   if (root && m_loader)
125     return make_error<StringError>(
126         "cannot generate a reproducer when replay one",
127         inconvertibleErrorCode());
128 
129   if (!root) {
130     m_generator.reset();
131     return Error::success();
132   }
133 
134   m_generator.emplace(*root);
135   return Error::success();
136 }
137 
138 llvm::Error Reproducer::SetReplay(llvm::Optional<FileSpec> root, bool passive) {
139   std::lock_guard<std::mutex> guard(m_mutex);
140 
141   if (root && m_generator)
142     return make_error<StringError>(
143         "cannot replay a reproducer when generating one",
144         inconvertibleErrorCode());
145 
146   if (!root) {
147     m_loader.reset();
148     return Error::success();
149   }
150 
151   m_loader.emplace(*root, passive);
152   if (auto e = m_loader->LoadIndex())
153     return e;
154 
155   return Error::success();
156 }
157 
158 FileSpec Reproducer::GetReproducerPath() const {
159   if (auto g = GetGenerator())
160     return g->GetRoot();
161   if (auto l = GetLoader())
162     return l->GetRoot();
163   return {};
164 }
165 
166 static FileSpec MakeAbsolute(const FileSpec &file_spec) {
167   SmallString<128> path;
168   file_spec.GetPath(path, false);
169   llvm::sys::fs::make_absolute(path);
170   return FileSpec(path, file_spec.GetPathStyle());
171 }
172 
173 Generator::Generator(FileSpec root) : m_root(MakeAbsolute(std::move(root))) {
174   GetOrCreate<repro::WorkingDirectoryProvider>();
175   GetOrCreate<repro::HomeDirectoryProvider>();
176 }
177 
178 Generator::~Generator() {
179   if (!m_done) {
180     if (m_auto_generate) {
181       Keep();
182       llvm::cantFail(Finalize(GetRoot()));
183     } else {
184       Discard();
185     }
186   }
187 }
188 
189 ProviderBase *Generator::Register(std::unique_ptr<ProviderBase> provider) {
190   std::lock_guard<std::mutex> lock(m_providers_mutex);
191   std::pair<const void *, std::unique_ptr<ProviderBase>> key_value(
192       provider->DynamicClassID(), std::move(provider));
193   auto e = m_providers.insert(std::move(key_value));
194   return e.first->getSecond().get();
195 }
196 
197 void Generator::Keep() {
198   LLDB_SCOPED_TIMER();
199   assert(!m_done);
200   m_done = true;
201 
202   for (auto &provider : m_providers)
203     provider.second->Keep();
204 
205   AddProvidersToIndex();
206 }
207 
208 void Generator::Discard() {
209   LLDB_SCOPED_TIMER();
210   assert(!m_done);
211   m_done = true;
212 
213   for (auto &provider : m_providers)
214     provider.second->Discard();
215 
216   llvm::sys::fs::remove_directories(m_root.GetPath());
217 }
218 
219 void Generator::SetAutoGenerate(bool b) { m_auto_generate = b; }
220 
221 bool Generator::IsAutoGenerate() const { return m_auto_generate; }
222 
223 const FileSpec &Generator::GetRoot() const { return m_root; }
224 
225 void Generator::AddProvidersToIndex() {
226   FileSpec index = m_root;
227   index.AppendPathComponent("index.yaml");
228 
229   std::error_code EC;
230   auto strm = std::make_unique<raw_fd_ostream>(index.GetPath(), EC,
231                                                sys::fs::OpenFlags::OF_None);
232   yaml::Output yout(*strm);
233 
234   std::vector<std::string> files;
235   files.reserve(m_providers.size());
236   for (auto &provider : m_providers) {
237     files.emplace_back(provider.second->GetFile());
238   }
239 
240   yout << files;
241 }
242 
243 Loader::Loader(FileSpec root, bool passive)
244     : m_root(MakeAbsolute(std::move(root))), m_loaded(false),
245       m_passive_replay(passive) {}
246 
247 llvm::Error Loader::LoadIndex() {
248   if (m_loaded)
249     return llvm::Error::success();
250 
251   FileSpec index = m_root.CopyByAppendingPathComponent("index.yaml");
252 
253   auto error_or_file = MemoryBuffer::getFile(index.GetPath());
254   if (auto err = error_or_file.getError())
255     return make_error<StringError>("unable to load reproducer index", err);
256 
257   yaml::Input yin((*error_or_file)->getBuffer());
258   yin >> m_files;
259   if (auto err = yin.error())
260     return make_error<StringError>("unable to read reproducer index", err);
261 
262   // Sort files to speed up search.
263   llvm::sort(m_files);
264 
265   // Remember that we've loaded the index.
266   m_loaded = true;
267 
268   return llvm::Error::success();
269 }
270 
271 bool Loader::HasFile(StringRef file) {
272   assert(m_loaded);
273   auto it = std::lower_bound(m_files.begin(), m_files.end(), file.str());
274   return (it != m_files.end()) && (*it == file);
275 }
276 
277 void Verifier::Verify(
278     llvm::function_ref<void(llvm::StringRef)> error_callback,
279     llvm::function_ref<void(llvm::StringRef)> warning_callback,
280     llvm::function_ref<void(llvm::StringRef)> note_callack) const {
281   if (!m_loader) {
282     error_callback("invalid loader");
283     return;
284   }
285 
286   FileSpec vfs_mapping = m_loader->GetFile<FileProvider::Info>();
287   ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
288       vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath());
289   if (!buffer) {
290     error_callback("unable to read files: " + buffer.getError().message());
291     return;
292   }
293 
294   IntrusiveRefCntPtr<vfs::FileSystem> vfs = vfs::getVFSFromYAML(
295       std::move(buffer.get()), nullptr, vfs_mapping.GetPath());
296   if (!vfs) {
297     error_callback("unable to initialize the virtual file system");
298     return;
299   }
300 
301   auto &redirecting_vfs = static_cast<vfs::RedirectingFileSystem &>(*vfs);
302   redirecting_vfs.setFallthrough(false);
303 
304   {
305     llvm::Expected<std::string> working_dir =
306         GetDirectoryFrom<WorkingDirectoryProvider>(m_loader);
307     if (working_dir) {
308       if (!vfs->exists(*working_dir))
309         warning_callback("working directory '" + *working_dir + "' not in VFS");
310       vfs->setCurrentWorkingDirectory(*working_dir);
311     } else {
312       warning_callback("no working directory in reproducer: " +
313                        toString(working_dir.takeError()));
314     }
315   }
316 
317   {
318     llvm::Expected<std::string> home_dir =
319         GetDirectoryFrom<HomeDirectoryProvider>(m_loader);
320     if (home_dir) {
321       if (!vfs->exists(*home_dir))
322         warning_callback("home directory '" + *home_dir + "' not in VFS");
323     } else {
324       warning_callback("no home directory in reproducer: " +
325                        toString(home_dir.takeError()));
326     }
327   }
328 
329   {
330     Expected<std::string> symbol_files =
331         m_loader->LoadBuffer<SymbolFileProvider>();
332     if (symbol_files) {
333       std::vector<SymbolFileProvider::Entry> entries;
334       llvm::yaml::Input yin(*symbol_files);
335       yin >> entries;
336       for (const auto &entry : entries) {
337         if (!entry.module_path.empty() && !vfs->exists(entry.module_path)) {
338           warning_callback("'" + entry.module_path + "': module path for " +
339                            entry.uuid + " not in VFS");
340         }
341         if (!entry.symbol_path.empty() && !vfs->exists(entry.symbol_path)) {
342           warning_callback("'" + entry.symbol_path + "': symbol path for " +
343                            entry.uuid + " not in VFS");
344         }
345       }
346     } else {
347       llvm::consumeError(symbol_files.takeError());
348     }
349   }
350 
351   // Missing files in the VFS are notes rather than warnings. Because the VFS
352   // is a snapshot, temporary files could have been removed between when they
353   // were recorded and when the reproducer was generated.
354   std::vector<llvm::StringRef> roots = redirecting_vfs.getRoots();
355   for (llvm::StringRef root : roots) {
356     std::error_code ec;
357     vfs::recursive_directory_iterator iter(*vfs, root, ec);
358     vfs::recursive_directory_iterator end;
359     for (; iter != end && !ec; iter.increment(ec)) {
360       ErrorOr<vfs::Status> status = vfs->status(iter->path());
361       if (!status)
362         note_callack("'" + iter->path().str() +
363                      "': " + status.getError().message());
364     }
365   }
366 }
367 
368 static llvm::Error addPaths(StringRef path,
369                             function_ref<void(StringRef)> callback) {
370   auto buffer = llvm::MemoryBuffer::getFile(path);
371   if (!buffer)
372     return errorCodeToError(buffer.getError());
373 
374   SmallVector<StringRef, 0> paths;
375   (*buffer)->getBuffer().split(paths, '\0');
376   for (StringRef p : paths) {
377     if (!p.empty())
378       callback(p);
379   }
380 
381   return errorCodeToError(llvm::sys::fs::remove(path));
382 }
383 
384 llvm::Error repro::Finalize(Loader *loader) {
385   if (!loader)
386     return make_error<StringError>("invalid loader",
387                                    llvm::inconvertibleErrorCode());
388 
389   FileSpec reproducer_root = loader->GetRoot();
390   std::string files_path =
391       reproducer_root.CopyByAppendingPathComponent("files.txt").GetPath();
392   std::string dirs_path =
393       reproducer_root.CopyByAppendingPathComponent("dirs.txt").GetPath();
394 
395   FileCollector collector(
396       reproducer_root.CopyByAppendingPathComponent("root").GetPath(),
397       reproducer_root.GetPath());
398 
399   if (Error e =
400           addPaths(files_path, [&](StringRef p) { collector.addFile(p); }))
401     return e;
402 
403   if (Error e =
404           addPaths(dirs_path, [&](StringRef p) { collector.addDirectory(p); }))
405     return e;
406 
407   FileSpec mapping =
408       reproducer_root.CopyByAppendingPathComponent(FileProvider::Info::file);
409   if (auto ec = collector.copyFiles(/*stop_on_error=*/false))
410     return errorCodeToError(ec);
411   collector.writeMapping(mapping.GetPath());
412 
413   return llvm::Error::success();
414 }
415 
416 llvm::Error repro::Finalize(const FileSpec &root) {
417   Loader loader(root);
418   if (Error e = loader.LoadIndex())
419     return e;
420   return Finalize(&loader);
421 }
422