1 //===-- Reproducer.cpp ----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "lldb/Utility/Reproducer.h"
10 #include "lldb/Utility/LLDBAssert.h"
11 #include "lldb/Utility/ReproducerProvider.h"
12 #include "lldb/Utility/Timer.h"
13 
14 #include "llvm/Support/FileSystem.h"
15 #include "llvm/Support/Threading.h"
16 #include "llvm/Support/raw_ostream.h"
17 
18 using namespace lldb_private;
19 using namespace lldb_private::repro;
20 using namespace llvm;
21 using namespace llvm::yaml;
22 
23 Reproducer &Reproducer::Instance() { return *InstanceImpl(); }
24 
25 llvm::Error Reproducer::Initialize(ReproducerMode mode,
26                                    llvm::Optional<FileSpec> root) {
27   lldbassert(!InstanceImpl() && "Already initialized.");
28   InstanceImpl().emplace();
29 
30   switch (mode) {
31   case ReproducerMode::Capture: {
32     if (!root) {
33       SmallString<128> repro_dir;
34       auto ec = sys::fs::createUniqueDirectory("reproducer", repro_dir);
35       if (ec)
36         return make_error<StringError>(
37             "unable to create unique reproducer directory", ec);
38       root.emplace(repro_dir);
39     } else {
40       auto ec = sys::fs::create_directory(root->GetPath());
41       if (ec)
42         return make_error<StringError>("unable to create reproducer directory",
43                                        ec);
44     }
45     return Instance().SetCapture(root);
46   } break;
47   case ReproducerMode::Replay:
48     return Instance().SetReplay(root, /*passive*/ false);
49   case ReproducerMode::PassiveReplay:
50     return Instance().SetReplay(root, /*passive*/ true);
51   case ReproducerMode::Off:
52     break;
53   };
54 
55   return Error::success();
56 }
57 
58 void Reproducer::Initialize() {
59   llvm::cantFail(Initialize(repro::ReproducerMode::Off, llvm::None));
60 }
61 
62 bool Reproducer::Initialized() { return InstanceImpl().operator bool(); }
63 
64 void Reproducer::Terminate() {
65   lldbassert(InstanceImpl() && "Already terminated.");
66   InstanceImpl().reset();
67 }
68 
69 Optional<Reproducer> &Reproducer::InstanceImpl() {
70   static Optional<Reproducer> g_reproducer;
71   return g_reproducer;
72 }
73 
74 const Generator *Reproducer::GetGenerator() const {
75   std::lock_guard<std::mutex> guard(m_mutex);
76   if (m_generator)
77     return &(*m_generator);
78   return nullptr;
79 }
80 
81 const Loader *Reproducer::GetLoader() const {
82   std::lock_guard<std::mutex> guard(m_mutex);
83   if (m_loader)
84     return &(*m_loader);
85   return nullptr;
86 }
87 
88 Generator *Reproducer::GetGenerator() {
89   std::lock_guard<std::mutex> guard(m_mutex);
90   if (m_generator)
91     return &(*m_generator);
92   return nullptr;
93 }
94 
95 Loader *Reproducer::GetLoader() {
96   std::lock_guard<std::mutex> guard(m_mutex);
97   if (m_loader)
98     return &(*m_loader);
99   return nullptr;
100 }
101 
102 llvm::Error Reproducer::SetCapture(llvm::Optional<FileSpec> root) {
103   std::lock_guard<std::mutex> guard(m_mutex);
104 
105   if (root && m_loader)
106     return make_error<StringError>(
107         "cannot generate a reproducer when replay one",
108         inconvertibleErrorCode());
109 
110   if (!root) {
111     m_generator.reset();
112     return Error::success();
113   }
114 
115   m_generator.emplace(*root);
116   return Error::success();
117 }
118 
119 llvm::Error Reproducer::SetReplay(llvm::Optional<FileSpec> root, bool passive) {
120   std::lock_guard<std::mutex> guard(m_mutex);
121 
122   if (root && m_generator)
123     return make_error<StringError>(
124         "cannot replay a reproducer when generating one",
125         inconvertibleErrorCode());
126 
127   if (!root) {
128     m_loader.reset();
129     return Error::success();
130   }
131 
132   m_loader.emplace(*root, passive);
133   if (auto e = m_loader->LoadIndex())
134     return e;
135 
136   return Error::success();
137 }
138 
139 FileSpec Reproducer::GetReproducerPath() const {
140   if (auto g = GetGenerator())
141     return g->GetRoot();
142   if (auto l = GetLoader())
143     return l->GetRoot();
144   return {};
145 }
146 
147 static FileSpec MakeAbsolute(const FileSpec &file_spec) {
148   SmallString<128> path;
149   file_spec.GetPath(path, false);
150   llvm::sys::fs::make_absolute(path);
151   return FileSpec(path, file_spec.GetPathStyle());
152 }
153 
154 Generator::Generator(FileSpec root) : m_root(MakeAbsolute(std::move(root))) {
155   GetOrCreate<repro::WorkingDirectoryProvider>();
156   GetOrCreate<repro::HomeDirectoryProvider>();
157 }
158 
159 Generator::~Generator() {
160   if (!m_done) {
161     if (m_auto_generate) {
162       Keep();
163       llvm::cantFail(Finalize(GetRoot()));
164     } else {
165       Discard();
166     }
167   }
168 }
169 
170 ProviderBase *Generator::Register(std::unique_ptr<ProviderBase> provider) {
171   std::lock_guard<std::mutex> lock(m_providers_mutex);
172   std::pair<const void *, std::unique_ptr<ProviderBase>> key_value(
173       provider->DynamicClassID(), std::move(provider));
174   auto e = m_providers.insert(std::move(key_value));
175   return e.first->getSecond().get();
176 }
177 
178 void Generator::Keep() {
179   LLDB_SCOPED_TIMER();
180   assert(!m_done);
181   m_done = true;
182 
183   for (auto &provider : m_providers)
184     provider.second->Keep();
185 
186   AddProvidersToIndex();
187 }
188 
189 void Generator::Discard() {
190   LLDB_SCOPED_TIMER();
191   assert(!m_done);
192   m_done = true;
193 
194   for (auto &provider : m_providers)
195     provider.second->Discard();
196 
197   llvm::sys::fs::remove_directories(m_root.GetPath());
198 }
199 
200 void Generator::SetAutoGenerate(bool b) { m_auto_generate = b; }
201 
202 bool Generator::IsAutoGenerate() const { return m_auto_generate; }
203 
204 const FileSpec &Generator::GetRoot() const { return m_root; }
205 
206 void Generator::AddProvidersToIndex() {
207   FileSpec index = m_root;
208   index.AppendPathComponent("index.yaml");
209 
210   std::error_code EC;
211   auto strm = std::make_unique<raw_fd_ostream>(index.GetPath(), EC,
212                                                sys::fs::OpenFlags::OF_None);
213   yaml::Output yout(*strm);
214 
215   std::vector<std::string> files;
216   files.reserve(m_providers.size());
217   for (auto &provider : m_providers) {
218     files.emplace_back(provider.second->GetFile());
219   }
220 
221   yout << files;
222 }
223 
224 Loader::Loader(FileSpec root, bool passive)
225     : m_root(MakeAbsolute(std::move(root))), m_loaded(false),
226       m_passive_replay(passive) {}
227 
228 llvm::Error Loader::LoadIndex() {
229   if (m_loaded)
230     return llvm::Error::success();
231 
232   FileSpec index = m_root.CopyByAppendingPathComponent("index.yaml");
233 
234   auto error_or_file = MemoryBuffer::getFile(index.GetPath());
235   if (auto err = error_or_file.getError())
236     return make_error<StringError>("unable to load reproducer index", err);
237 
238   yaml::Input yin((*error_or_file)->getBuffer());
239   yin >> m_files;
240   if (auto err = yin.error())
241     return make_error<StringError>("unable to read reproducer index", err);
242 
243   // Sort files to speed up search.
244   llvm::sort(m_files);
245 
246   // Remember that we've loaded the index.
247   m_loaded = true;
248 
249   return llvm::Error::success();
250 }
251 
252 bool Loader::HasFile(StringRef file) {
253   assert(m_loaded);
254   auto it = std::lower_bound(m_files.begin(), m_files.end(), file.str());
255   return (it != m_files.end()) && (*it == file);
256 }
257 
258 void Verifier::Verify(
259     llvm::function_ref<void(llvm::StringRef)> error_callback,
260     llvm::function_ref<void(llvm::StringRef)> warning_callback,
261     llvm::function_ref<void(llvm::StringRef)> note_callack) const {
262   if (!m_loader) {
263     error_callback("invalid loader");
264     return;
265   }
266 
267   FileSpec vfs_mapping = m_loader->GetFile<FileProvider::Info>();
268   ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
269       vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath());
270   if (!buffer) {
271     error_callback("unable to read files: " + buffer.getError().message());
272     return;
273   }
274 
275   IntrusiveRefCntPtr<vfs::FileSystem> vfs = vfs::getVFSFromYAML(
276       std::move(buffer.get()), nullptr, vfs_mapping.GetPath());
277   if (!vfs) {
278     error_callback("unable to initialize the virtual file system");
279     return;
280   }
281 
282   auto &redirecting_vfs = static_cast<vfs::RedirectingFileSystem &>(*vfs);
283   redirecting_vfs.setFallthrough(false);
284 
285   {
286     llvm::Expected<std::string> working_dir =
287         GetDirectoryFrom<WorkingDirectoryProvider>(m_loader);
288     if (working_dir) {
289       if (!vfs->exists(*working_dir))
290         warning_callback("working directory '" + *working_dir + "' not in VFS");
291       vfs->setCurrentWorkingDirectory(*working_dir);
292     } else {
293       warning_callback("no working directory in reproducer: " +
294                        toString(working_dir.takeError()));
295     }
296   }
297 
298   {
299     llvm::Expected<std::string> home_dir =
300         GetDirectoryFrom<HomeDirectoryProvider>(m_loader);
301     if (home_dir) {
302       if (!vfs->exists(*home_dir))
303         warning_callback("home directory '" + *home_dir + "' not in VFS");
304     } else {
305       warning_callback("no home directory in reproducer: " +
306                        toString(home_dir.takeError()));
307     }
308   }
309 
310   {
311     Expected<std::string> symbol_files =
312         m_loader->LoadBuffer<SymbolFileProvider>();
313     if (symbol_files) {
314       std::vector<SymbolFileProvider::Entry> entries;
315       llvm::yaml::Input yin(*symbol_files);
316       yin >> entries;
317       for (const auto &entry : entries) {
318         if (!entry.module_path.empty() && !vfs->exists(entry.module_path)) {
319           warning_callback("'" + entry.module_path + "': module path for " +
320                            entry.uuid + " not in VFS");
321         }
322         if (!entry.symbol_path.empty() && !vfs->exists(entry.symbol_path)) {
323           warning_callback("'" + entry.symbol_path + "': symbol path for " +
324                            entry.uuid + " not in VFS");
325         }
326       }
327     } else {
328       llvm::consumeError(symbol_files.takeError());
329     }
330   }
331 
332   // Missing files in the VFS are notes rather than warnings. Because the VFS
333   // is a snapshot, temporary files could have been removed between when they
334   // were recorded and when the reproducer was generated.
335   std::vector<llvm::StringRef> roots = redirecting_vfs.getRoots();
336   for (llvm::StringRef root : roots) {
337     std::error_code ec;
338     vfs::recursive_directory_iterator iter(*vfs, root, ec);
339     vfs::recursive_directory_iterator end;
340     for (; iter != end && !ec; iter.increment(ec)) {
341       ErrorOr<vfs::Status> status = vfs->status(iter->path());
342       if (!status)
343         note_callack("'" + iter->path().str() +
344                      "': " + status.getError().message());
345     }
346   }
347 }
348 
349 static llvm::Error addPaths(StringRef path,
350                             function_ref<void(StringRef)> callback) {
351   auto buffer = llvm::MemoryBuffer::getFile(path);
352   if (!buffer)
353     return errorCodeToError(buffer.getError());
354 
355   SmallVector<StringRef, 0> paths;
356   (*buffer)->getBuffer().split(paths, '\0');
357   for (StringRef p : paths) {
358     if (!p.empty() && llvm::sys::fs::exists(p))
359       callback(p);
360   }
361 
362   return errorCodeToError(llvm::sys::fs::remove(path));
363 }
364 
365 llvm::Error repro::Finalize(Loader *loader) {
366   if (!loader)
367     return make_error<StringError>("invalid loader",
368                                    llvm::inconvertibleErrorCode());
369 
370   FileSpec reproducer_root = loader->GetRoot();
371   std::string files_path =
372       reproducer_root.CopyByAppendingPathComponent("files.txt").GetPath();
373   std::string dirs_path =
374       reproducer_root.CopyByAppendingPathComponent("dirs.txt").GetPath();
375 
376   FileCollector collector(
377       reproducer_root.CopyByAppendingPathComponent("root").GetPath(),
378       reproducer_root.GetPath());
379 
380   if (Error e =
381           addPaths(files_path, [&](StringRef p) { collector.addFile(p); }))
382     return e;
383 
384   if (Error e =
385           addPaths(dirs_path, [&](StringRef p) { collector.addDirectory(p); }))
386     return e;
387 
388   FileSpec mapping =
389       reproducer_root.CopyByAppendingPathComponent(FileProvider::Info::file);
390   if (auto ec = collector.copyFiles(/*stop_on_error=*/false))
391     return errorCodeToError(ec);
392   collector.writeMapping(mapping.GetPath());
393 
394   return llvm::Error::success();
395 }
396 
397 llvm::Error repro::Finalize(const FileSpec &root) {
398   Loader loader(root);
399   if (Error e = loader.LoadIndex())
400     return e;
401   return Finalize(&loader);
402 }
403