1 //===-- Reproducer.cpp ----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "lldb/Utility/Reproducer.h"
10 #include "lldb/Utility/LLDBAssert.h"
11 #include "lldb/Utility/ReproducerProvider.h"
12 #include "lldb/Utility/Timer.h"
13 
14 #include "llvm/Support/FileSystem.h"
15 #include "llvm/Support/Threading.h"
16 #include "llvm/Support/raw_ostream.h"
17 
18 using namespace lldb_private;
19 using namespace lldb_private::repro;
20 using namespace llvm;
21 using namespace llvm::yaml;
22 
23 Reproducer &Reproducer::Instance() { return *InstanceImpl(); }
24 
25 llvm::Error Reproducer::Initialize(ReproducerMode mode,
26                                    llvm::Optional<FileSpec> root) {
27   lldbassert(!InstanceImpl() && "Already initialized.");
28   InstanceImpl().emplace();
29 
30   switch (mode) {
31   case ReproducerMode::Capture: {
32     if (!root) {
33       SmallString<128> repro_dir;
34       auto ec = sys::fs::createUniqueDirectory("reproducer", repro_dir);
35       if (ec)
36         return make_error<StringError>(
37             "unable to create unique reproducer directory", ec);
38       root.emplace(repro_dir);
39     } else {
40       auto ec = sys::fs::create_directory(root->GetPath());
41       if (ec)
42         return make_error<StringError>("unable to create reproducer directory",
43                                        ec);
44     }
45     return Instance().SetCapture(root);
46   } break;
47   case ReproducerMode::Off:
48     break;
49   };
50 
51   return Error::success();
52 }
53 
54 void Reproducer::Initialize() {
55   llvm::cantFail(Initialize(repro::ReproducerMode::Off, llvm::None));
56 }
57 
58 bool Reproducer::Initialized() { return InstanceImpl().operator bool(); }
59 
60 void Reproducer::Terminate() {
61   lldbassert(InstanceImpl() && "Already terminated.");
62   InstanceImpl().reset();
63 }
64 
65 Optional<Reproducer> &Reproducer::InstanceImpl() {
66   static Optional<Reproducer> g_reproducer;
67   return g_reproducer;
68 }
69 
70 const Generator *Reproducer::GetGenerator() const {
71   std::lock_guard<std::mutex> guard(m_mutex);
72   if (m_generator)
73     return &(*m_generator);
74   return nullptr;
75 }
76 
77 const Loader *Reproducer::GetLoader() const {
78   std::lock_guard<std::mutex> guard(m_mutex);
79   if (m_loader)
80     return &(*m_loader);
81   return nullptr;
82 }
83 
84 Generator *Reproducer::GetGenerator() {
85   std::lock_guard<std::mutex> guard(m_mutex);
86   if (m_generator)
87     return &(*m_generator);
88   return nullptr;
89 }
90 
91 Loader *Reproducer::GetLoader() {
92   std::lock_guard<std::mutex> guard(m_mutex);
93   if (m_loader)
94     return &(*m_loader);
95   return nullptr;
96 }
97 
98 llvm::Error Reproducer::SetCapture(llvm::Optional<FileSpec> root) {
99   std::lock_guard<std::mutex> guard(m_mutex);
100 
101   if (root && m_loader)
102     return make_error<StringError>(
103         "cannot generate a reproducer when replay one",
104         inconvertibleErrorCode());
105 
106   if (!root) {
107     m_generator.reset();
108     return Error::success();
109   }
110 
111   m_generator.emplace(*root);
112   return Error::success();
113 }
114 
115 FileSpec Reproducer::GetReproducerPath() const {
116   if (auto g = GetGenerator())
117     return g->GetRoot();
118   if (auto l = GetLoader())
119     return l->GetRoot();
120   return {};
121 }
122 
123 static FileSpec MakeAbsolute(const FileSpec &file_spec) {
124   SmallString<128> path;
125   file_spec.GetPath(path, false);
126   llvm::sys::fs::make_absolute(path);
127   return FileSpec(path, file_spec.GetPathStyle());
128 }
129 
130 Generator::Generator(FileSpec root) : m_root(MakeAbsolute(std::move(root))) {
131   GetOrCreate<repro::WorkingDirectoryProvider>();
132   GetOrCreate<repro::HomeDirectoryProvider>();
133 }
134 
135 Generator::~Generator() {
136   if (!m_done) {
137     if (m_auto_generate) {
138       Keep();
139       llvm::cantFail(Finalize(GetRoot()));
140     } else {
141       Discard();
142     }
143   }
144 }
145 
146 ProviderBase *Generator::Register(std::unique_ptr<ProviderBase> provider) {
147   std::lock_guard<std::mutex> lock(m_providers_mutex);
148   std::pair<const void *, std::unique_ptr<ProviderBase>> key_value(
149       provider->DynamicClassID(), std::move(provider));
150   auto e = m_providers.insert(std::move(key_value));
151   return e.first->getSecond().get();
152 }
153 
154 void Generator::Keep() {
155   LLDB_SCOPED_TIMER();
156   assert(!m_done);
157   m_done = true;
158 
159   for (auto &provider : m_providers)
160     provider.second->Keep();
161 
162   AddProvidersToIndex();
163 }
164 
165 void Generator::Discard() {
166   LLDB_SCOPED_TIMER();
167   assert(!m_done);
168   m_done = true;
169 
170   for (auto &provider : m_providers)
171     provider.second->Discard();
172 
173   llvm::sys::fs::remove_directories(m_root.GetPath());
174 }
175 
176 void Generator::SetAutoGenerate(bool b) { m_auto_generate = b; }
177 
178 bool Generator::IsAutoGenerate() const { return m_auto_generate; }
179 
180 const FileSpec &Generator::GetRoot() const { return m_root; }
181 
182 void Generator::AddProvidersToIndex() {
183   FileSpec index = m_root;
184   index.AppendPathComponent("index.yaml");
185 
186   std::error_code EC;
187   auto strm = std::make_unique<raw_fd_ostream>(index.GetPath(), EC,
188                                                sys::fs::OpenFlags::OF_None);
189   yaml::Output yout(*strm);
190 
191   std::vector<std::string> files;
192   files.reserve(m_providers.size());
193   for (auto &provider : m_providers) {
194     files.emplace_back(provider.second->GetFile());
195   }
196 
197   yout << files;
198 }
199 
200 Loader::Loader(FileSpec root, bool passive)
201     : m_root(MakeAbsolute(std::move(root))), m_loaded(false) {}
202 
203 llvm::Error Loader::LoadIndex() {
204   if (m_loaded)
205     return llvm::Error::success();
206 
207   FileSpec index = m_root.CopyByAppendingPathComponent("index.yaml");
208 
209   auto error_or_file = MemoryBuffer::getFile(index.GetPath());
210   if (auto err = error_or_file.getError())
211     return make_error<StringError>("unable to load reproducer index", err);
212 
213   yaml::Input yin((*error_or_file)->getBuffer());
214   yin >> m_files;
215   if (auto err = yin.error())
216     return make_error<StringError>("unable to read reproducer index", err);
217 
218   // Sort files to speed up search.
219   llvm::sort(m_files);
220 
221   // Remember that we've loaded the index.
222   m_loaded = true;
223 
224   return llvm::Error::success();
225 }
226 
227 bool Loader::HasFile(StringRef file) {
228   assert(m_loaded);
229   auto it = std::lower_bound(m_files.begin(), m_files.end(), file.str());
230   return (it != m_files.end()) && (*it == file);
231 }
232 
233 void Verifier::Verify(
234     llvm::function_ref<void(llvm::StringRef)> error_callback,
235     llvm::function_ref<void(llvm::StringRef)> warning_callback,
236     llvm::function_ref<void(llvm::StringRef)> note_callack) const {
237   if (!m_loader) {
238     error_callback("invalid loader");
239     return;
240   }
241 
242   FileSpec vfs_mapping = m_loader->GetFile<FileProvider::Info>();
243   ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
244       vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath());
245   if (!buffer) {
246     error_callback("unable to read files: " + buffer.getError().message());
247     return;
248   }
249 
250   IntrusiveRefCntPtr<vfs::FileSystem> vfs = vfs::getVFSFromYAML(
251       std::move(buffer.get()), nullptr, vfs_mapping.GetPath());
252   if (!vfs) {
253     error_callback("unable to initialize the virtual file system");
254     return;
255   }
256 
257   auto &redirecting_vfs = static_cast<vfs::RedirectingFileSystem &>(*vfs);
258   redirecting_vfs.setFallthrough(false);
259 
260   {
261     llvm::Expected<std::string> working_dir =
262         GetDirectoryFrom<WorkingDirectoryProvider>(m_loader);
263     if (working_dir) {
264       if (!vfs->exists(*working_dir))
265         warning_callback("working directory '" + *working_dir + "' not in VFS");
266       vfs->setCurrentWorkingDirectory(*working_dir);
267     } else {
268       warning_callback("no working directory in reproducer: " +
269                        toString(working_dir.takeError()));
270     }
271   }
272 
273   {
274     llvm::Expected<std::string> home_dir =
275         GetDirectoryFrom<HomeDirectoryProvider>(m_loader);
276     if (home_dir) {
277       if (!vfs->exists(*home_dir))
278         warning_callback("home directory '" + *home_dir + "' not in VFS");
279     } else {
280       warning_callback("no home directory in reproducer: " +
281                        toString(home_dir.takeError()));
282     }
283   }
284 
285   {
286     Expected<std::string> symbol_files =
287         m_loader->LoadBuffer<SymbolFileProvider>();
288     if (symbol_files) {
289       std::vector<SymbolFileProvider::Entry> entries;
290       llvm::yaml::Input yin(*symbol_files);
291       yin >> entries;
292       for (const auto &entry : entries) {
293         if (!entry.module_path.empty() && !vfs->exists(entry.module_path)) {
294           warning_callback("'" + entry.module_path + "': module path for " +
295                            entry.uuid + " not in VFS");
296         }
297         if (!entry.symbol_path.empty() && !vfs->exists(entry.symbol_path)) {
298           warning_callback("'" + entry.symbol_path + "': symbol path for " +
299                            entry.uuid + " not in VFS");
300         }
301       }
302     } else {
303       llvm::consumeError(symbol_files.takeError());
304     }
305   }
306 
307   // Missing files in the VFS are notes rather than warnings. Because the VFS
308   // is a snapshot, temporary files could have been removed between when they
309   // were recorded and when the reproducer was generated.
310   std::vector<llvm::StringRef> roots = redirecting_vfs.getRoots();
311   for (llvm::StringRef root : roots) {
312     std::error_code ec;
313     vfs::recursive_directory_iterator iter(*vfs, root, ec);
314     vfs::recursive_directory_iterator end;
315     for (; iter != end && !ec; iter.increment(ec)) {
316       ErrorOr<vfs::Status> status = vfs->status(iter->path());
317       if (!status)
318         note_callack("'" + iter->path().str() +
319                      "': " + status.getError().message());
320     }
321   }
322 }
323 
324 static llvm::Error addPaths(StringRef path,
325                             function_ref<void(StringRef)> callback) {
326   auto buffer = llvm::MemoryBuffer::getFile(path);
327   if (!buffer)
328     return errorCodeToError(buffer.getError());
329 
330   SmallVector<StringRef, 0> paths;
331   (*buffer)->getBuffer().split(paths, '\0');
332   for (StringRef p : paths) {
333     if (!p.empty() && llvm::sys::fs::exists(p))
334       callback(p);
335   }
336 
337   return errorCodeToError(llvm::sys::fs::remove(path));
338 }
339 
340 llvm::Error repro::Finalize(Loader *loader) {
341   if (!loader)
342     return make_error<StringError>("invalid loader",
343                                    llvm::inconvertibleErrorCode());
344 
345   FileSpec reproducer_root = loader->GetRoot();
346   std::string files_path =
347       reproducer_root.CopyByAppendingPathComponent("files.txt").GetPath();
348   std::string dirs_path =
349       reproducer_root.CopyByAppendingPathComponent("dirs.txt").GetPath();
350 
351   FileCollector collector(
352       reproducer_root.CopyByAppendingPathComponent("root").GetPath(),
353       reproducer_root.GetPath());
354 
355   if (Error e =
356           addPaths(files_path, [&](StringRef p) { collector.addFile(p); }))
357     return e;
358 
359   if (Error e =
360           addPaths(dirs_path, [&](StringRef p) { collector.addDirectory(p); }))
361     return e;
362 
363   FileSpec mapping =
364       reproducer_root.CopyByAppendingPathComponent(FileProvider::Info::file);
365   if (auto ec = collector.copyFiles(/*stop_on_error=*/false))
366     return errorCodeToError(ec);
367   collector.writeMapping(mapping.GetPath());
368 
369   return llvm::Error::success();
370 }
371 
372 llvm::Error repro::Finalize(const FileSpec &root) {
373   Loader loader(root);
374   if (Error e = loader.LoadIndex())
375     return e;
376   return Finalize(&loader);
377 }
378