1 //===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Helper class to build precompiled preamble.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "clang/Frontend/PrecompiledPreamble.h"
15 #include "clang/AST/DeclObjC.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/VirtualFileSystem.h"
18 #include "clang/Frontend/CompilerInstance.h"
19 #include "clang/Frontend/CompilerInvocation.h"
20 #include "clang/Frontend/FrontendActions.h"
21 #include "clang/Frontend/FrontendOptions.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/PreprocessorOptions.h"
24 #include "clang/Serialization/ASTWriter.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringSet.h"
27 #include "llvm/Support/CrashRecoveryContext.h"
28 #include "llvm/Support/FileSystem.h"
29 #include "llvm/Support/Mutex.h"
30 #include "llvm/Support/MutexGuard.h"
31 
32 using namespace clang;
33 
34 namespace {
35 
36 /// Keeps a track of files to be deleted in destructor.
37 class TemporaryFiles {
38 public:
39   // A static instance to be used by all clients.
40   static TemporaryFiles &getInstance();
41 
42 private:
43   // Disallow constructing the class directly.
44   TemporaryFiles() = default;
45   // Disallow copy.
46   TemporaryFiles(const TemporaryFiles &) = delete;
47 
48 public:
49   ~TemporaryFiles();
50 
51   /// Adds \p File to a set of tracked files.
52   void addFile(StringRef File);
53 
54   /// Remove \p File from disk and from the set of tracked files.
55   void removeFile(StringRef File);
56 
57 private:
58   llvm::sys::SmartMutex<false> Mutex;
59   llvm::StringSet<> Files;
60 };
61 
62 TemporaryFiles &TemporaryFiles::getInstance() {
63   static TemporaryFiles Instance;
64   return Instance;
65 }
66 
67 TemporaryFiles::~TemporaryFiles() {
68   llvm::MutexGuard Guard(Mutex);
69   for (const auto &File : Files)
70     llvm::sys::fs::remove(File.getKey());
71 }
72 
73 void TemporaryFiles::addFile(StringRef File) {
74   llvm::MutexGuard Guard(Mutex);
75   auto IsInserted = Files.insert(File).second;
76   (void)IsInserted;
77   assert(IsInserted && "File has already been added");
78 }
79 
80 void TemporaryFiles::removeFile(StringRef File) {
81   llvm::MutexGuard Guard(Mutex);
82   auto WasPresent = Files.erase(File);
83   (void)WasPresent;
84   assert(WasPresent && "File was not tracked");
85   llvm::sys::fs::remove(File);
86 }
87 
88 class PreambleMacroCallbacks : public PPCallbacks {
89 public:
90   PreambleMacroCallbacks(PreambleCallbacks &Callbacks) : Callbacks(Callbacks) {}
91 
92   void MacroDefined(const Token &MacroNameTok,
93                     const MacroDirective *MD) override {
94     Callbacks.HandleMacroDefined(MacroNameTok, MD);
95   }
96 
97 private:
98   PreambleCallbacks &Callbacks;
99 };
100 
101 class PrecompilePreambleAction : public ASTFrontendAction {
102 public:
103   PrecompilePreambleAction(PreambleCallbacks &Callbacks)
104       : Callbacks(Callbacks) {}
105 
106   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
107                                                  StringRef InFile) override;
108 
109   bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
110 
111   void setEmittedPreamblePCH(ASTWriter &Writer) {
112     this->HasEmittedPreamblePCH = true;
113     Callbacks.AfterPCHEmitted(Writer);
114   }
115 
116   bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
117   bool hasCodeCompletionSupport() const override { return false; }
118   bool hasASTFileSupport() const override { return false; }
119   TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
120 
121 private:
122   friend class PrecompilePreambleConsumer;
123 
124   bool HasEmittedPreamblePCH = false;
125   PreambleCallbacks &Callbacks;
126 };
127 
128 class PrecompilePreambleConsumer : public PCHGenerator {
129 public:
130   PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
131                              const Preprocessor &PP, StringRef isysroot,
132                              std::unique_ptr<raw_ostream> Out)
133       : PCHGenerator(PP, "", isysroot, std::make_shared<PCHBuffer>(),
134                      ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
135                      /*AllowASTWithErrors=*/true),
136         Action(Action), Out(std::move(Out)) {}
137 
138   bool HandleTopLevelDecl(DeclGroupRef DG) override {
139     Action.Callbacks.HandleTopLevelDecl(DG);
140     return true;
141   }
142 
143   void HandleTranslationUnit(ASTContext &Ctx) override {
144     PCHGenerator::HandleTranslationUnit(Ctx);
145     if (!hasEmittedPCH())
146       return;
147 
148     // Write the generated bitstream to "Out".
149     *Out << getPCH();
150     // Make sure it hits disk now.
151     Out->flush();
152     // Free the buffer.
153     llvm::SmallVector<char, 0> Empty;
154     getPCH() = std::move(Empty);
155 
156     Action.setEmittedPreamblePCH(getWriter());
157   }
158 
159 private:
160   PrecompilePreambleAction &Action;
161   std::unique_ptr<raw_ostream> Out;
162 };
163 
164 std::unique_ptr<ASTConsumer>
165 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
166 
167                                             StringRef InFile) {
168   std::string Sysroot;
169   std::string OutputFile;
170   std::unique_ptr<raw_ostream> OS =
171       GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
172                                                      OutputFile);
173   if (!OS)
174     return nullptr;
175 
176   if (!CI.getFrontendOpts().RelocatablePCH)
177     Sysroot.clear();
178 
179   CI.getPreprocessor().addPPCallbacks(
180       llvm::make_unique<PreambleMacroCallbacks>(Callbacks));
181   return llvm::make_unique<PrecompilePreambleConsumer>(
182       *this, CI.getPreprocessor(), Sysroot, std::move(OS));
183 }
184 
185 template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
186   if (!Val)
187     return false;
188   Output = std::move(*Val);
189   return true;
190 }
191 
192 } // namespace
193 
194 PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
195                                             llvm::MemoryBuffer *Buffer,
196                                             unsigned MaxLines) {
197   auto Pre = Lexer::ComputePreamble(Buffer->getBuffer(), LangOpts, MaxLines);
198   return PreambleBounds(Pre.first, Pre.second);
199 }
200 
201 llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
202     const CompilerInvocation &Invocation,
203     const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
204     DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
205     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
206     PreambleCallbacks &Callbacks) {
207   assert(VFS && "VFS is null");
208 
209   if (!Bounds.Size)
210     return BuildPreambleError::PreambleIsEmpty;
211 
212   auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
213   FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
214   PreprocessorOptions &PreprocessorOpts =
215       PreambleInvocation->getPreprocessorOpts();
216 
217   // Create a temporary file for the precompiled preamble. In rare
218   // circumstances, this can fail.
219   llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
220       PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
221   if (!PreamblePCHFile)
222     return BuildPreambleError::CouldntCreateTempFile;
223 
224   // Save the preamble text for later; we'll need to compare against it for
225   // subsequent reparses.
226   std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
227                                   MainFileBuffer->getBufferStart() +
228                                       Bounds.Size);
229   bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
230 
231   // Tell the compiler invocation to generate a temporary precompiled header.
232   FrontendOpts.ProgramAction = frontend::GeneratePCH;
233   // FIXME: Generate the precompiled header into memory?
234   FrontendOpts.OutputFile = PreamblePCHFile->getFilePath();
235   PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
236   PreprocessorOpts.PrecompiledPreambleBytes.second = false;
237 
238   // Create the compiler instance to use for building the precompiled preamble.
239   std::unique_ptr<CompilerInstance> Clang(
240       new CompilerInstance(std::move(PCHContainerOps)));
241 
242   // Recover resources if we crash before exiting this method.
243   llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
244       Clang.get());
245 
246   Clang->setInvocation(std::move(PreambleInvocation));
247   Clang->setDiagnostics(&Diagnostics);
248 
249   // Create the target instance.
250   Clang->setTarget(TargetInfo::CreateTargetInfo(
251       Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
252   if (!Clang->hasTarget())
253     return BuildPreambleError::CouldntCreateTargetInfo;
254 
255   // Inform the target of the language options.
256   //
257   // FIXME: We shouldn't need to do this, the target should be immutable once
258   // created. This complexity should be lifted elsewhere.
259   Clang->getTarget().adjust(Clang->getLangOpts());
260 
261   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
262          "Invocation must have exactly one source file!");
263   assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() ==
264              InputKind::Source &&
265          "FIXME: AST inputs not yet supported here!");
266   assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() !=
267              InputKind::LLVM_IR &&
268          "IR inputs not support here!");
269 
270   // Clear out old caches and data.
271   Diagnostics.Reset();
272   ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
273 
274   VFS =
275       createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
276   if (!VFS)
277     return BuildPreambleError::CouldntCreateVFSOverlay;
278 
279   // Create a file manager object to provide access to and cache the filesystem.
280   Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
281 
282   // Create the source manager.
283   Clang->setSourceManager(
284       new SourceManager(Diagnostics, Clang->getFileManager()));
285 
286   auto PreambleDepCollector = std::make_shared<DependencyCollector>();
287   Clang->addDependencyCollector(PreambleDepCollector);
288 
289   // Remap the main source file to the preamble buffer.
290   StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
291   auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
292       MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
293   if (PreprocessorOpts.RetainRemappedFileBuffers) {
294     // MainFileBuffer will be deleted by unique_ptr after leaving the method.
295     PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
296   } else {
297     // In that case, remapped buffer will be deleted by CompilerInstance on
298     // BeginSourceFile, so we call release() to avoid double deletion.
299     PreprocessorOpts.addRemappedFile(MainFilePath,
300                                      PreambleInputBuffer.release());
301   }
302 
303   std::unique_ptr<PrecompilePreambleAction> Act;
304   Act.reset(new PrecompilePreambleAction(Callbacks));
305   if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
306     return BuildPreambleError::BeginSourceFileFailed;
307 
308   Act->Execute();
309 
310   // Run the callbacks.
311   Callbacks.AfterExecute(*Clang);
312 
313   Act->EndSourceFile();
314 
315   if (!Act->hasEmittedPreamblePCH())
316     return BuildPreambleError::CouldntEmitPCH;
317 
318   // Keep track of all of the files that the source manager knows about,
319   // so we can verify whether they have changed or not.
320   llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
321 
322   SourceManager &SourceMgr = Clang->getSourceManager();
323   for (auto &Filename : PreambleDepCollector->getDependencies()) {
324     const FileEntry *File = Clang->getFileManager().getFile(Filename);
325     if (!File || File == SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()))
326       continue;
327     if (time_t ModTime = File->getModificationTime()) {
328       FilesInPreamble[File->getName()] =
329           PrecompiledPreamble::PreambleFileHash::createForFile(File->getSize(),
330                                                                ModTime);
331     } else {
332       llvm::MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File);
333       FilesInPreamble[File->getName()] =
334           PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
335     }
336   }
337 
338   return PrecompiledPreamble(
339       std::move(*PreamblePCHFile), std::move(PreambleBytes),
340       PreambleEndsAtStartOfLine, std::move(FilesInPreamble));
341 }
342 
343 PreambleBounds PrecompiledPreamble::getBounds() const {
344   return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
345 }
346 
347 bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
348                                    const llvm::MemoryBuffer *MainFileBuffer,
349                                    PreambleBounds Bounds,
350                                    vfs::FileSystem *VFS) const {
351 
352   assert(
353       Bounds.Size <= MainFileBuffer->getBufferSize() &&
354       "Buffer is too large. Bounds were calculated from a different buffer?");
355 
356   auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
357   PreprocessorOptions &PreprocessorOpts =
358       PreambleInvocation->getPreprocessorOpts();
359 
360   if (!Bounds.Size)
361     return false;
362 
363   // We've previously computed a preamble. Check whether we have the same
364   // preamble now that we did before, and that there's enough space in
365   // the main-file buffer within the precompiled preamble to fit the
366   // new main file.
367   if (PreambleBytes.size() != Bounds.Size ||
368       PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
369       memcmp(PreambleBytes.data(), MainFileBuffer->getBufferStart(),
370              Bounds.Size) != 0)
371     return false;
372   // The preamble has not changed. We may be able to re-use the precompiled
373   // preamble.
374 
375   // Check that none of the files used by the preamble have changed.
376   // First, make a record of those files that have been overridden via
377   // remapping or unsaved_files.
378   std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
379   for (const auto &R : PreprocessorOpts.RemappedFiles) {
380     vfs::Status Status;
381     if (!moveOnNoError(VFS->status(R.second), Status)) {
382       // If we can't stat the file we're remapping to, assume that something
383       // horrible happened.
384       return false;
385     }
386 
387     OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
388         Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
389   }
390 
391   for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
392     vfs::Status Status;
393     if (!moveOnNoError(VFS->status(RB.first), Status))
394       return false;
395 
396     OverriddenFiles[Status.getUniqueID()] =
397         PreambleFileHash::createForMemoryBuffer(RB.second);
398   }
399 
400   // Check whether anything has changed.
401   for (const auto &F : FilesInPreamble) {
402     vfs::Status Status;
403     if (!moveOnNoError(VFS->status(F.first()), Status)) {
404       // If we can't stat the file, assume that something horrible happened.
405       return false;
406     }
407 
408     std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
409         OverriddenFiles.find(Status.getUniqueID());
410     if (Overridden != OverriddenFiles.end()) {
411       // This file was remapped; check whether the newly-mapped file
412       // matches up with the previous mapping.
413       if (Overridden->second != F.second)
414         return false;
415       continue;
416     }
417 
418     // The file was not remapped; check whether it has changed on disk.
419     if (Status.getSize() != uint64_t(F.second.Size) ||
420         llvm::sys::toTimeT(Status.getLastModificationTime()) !=
421             F.second.ModTime)
422       return false;
423   }
424   return true;
425 }
426 
427 void PrecompiledPreamble::AddImplicitPreamble(
428     CompilerInvocation &CI, llvm::MemoryBuffer *MainFileBuffer) const {
429   auto &PreprocessorOpts = CI.getPreprocessorOpts();
430 
431   // Configure ImpicitPCHInclude.
432   PreprocessorOpts.PrecompiledPreambleBytes.first = PreambleBytes.size();
433   PreprocessorOpts.PrecompiledPreambleBytes.second = PreambleEndsAtStartOfLine;
434   PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
435   PreprocessorOpts.DisablePCHValidation = true;
436 
437   // Remap main file to point to MainFileBuffer.
438   auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
439   PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
440 }
441 
442 PrecompiledPreamble::PrecompiledPreamble(
443     TempPCHFile PCHFile, std::vector<char> PreambleBytes,
444     bool PreambleEndsAtStartOfLine,
445     llvm::StringMap<PreambleFileHash> FilesInPreamble)
446     : PCHFile(std::move(PCHFile)), FilesInPreamble(FilesInPreamble),
447       PreambleBytes(std::move(PreambleBytes)),
448       PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {}
449 
450 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
451 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
452   // FIXME: This is a hack so that we can override the preamble file during
453   // crash-recovery testing, which is the only case where the preamble files
454   // are not necessarily cleaned up.
455   const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE");
456   if (TmpFile)
457     return TempPCHFile::createFromCustomPath(TmpFile);
458   return TempPCHFile::createInSystemTempDir("preamble", "pch");
459 }
460 
461 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
462 PrecompiledPreamble::TempPCHFile::createInSystemTempDir(const Twine &Prefix,
463                                                         StringRef Suffix) {
464   llvm::SmallString<64> File;
465   auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, /*ref*/ File);
466   if (EC)
467     return EC;
468   return TempPCHFile(std::move(File).str());
469 }
470 
471 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
472 PrecompiledPreamble::TempPCHFile::createFromCustomPath(const Twine &Path) {
473   return TempPCHFile(Path.str());
474 }
475 
476 PrecompiledPreamble::TempPCHFile::TempPCHFile(std::string FilePath)
477     : FilePath(std::move(FilePath)) {
478   TemporaryFiles::getInstance().addFile(*this->FilePath);
479 }
480 
481 PrecompiledPreamble::TempPCHFile::TempPCHFile(TempPCHFile &&Other) {
482   FilePath = std::move(Other.FilePath);
483   Other.FilePath = None;
484 }
485 
486 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::TempPCHFile::
487 operator=(TempPCHFile &&Other) {
488   RemoveFileIfPresent();
489 
490   FilePath = std::move(Other.FilePath);
491   Other.FilePath = None;
492   return *this;
493 }
494 
495 PrecompiledPreamble::TempPCHFile::~TempPCHFile() { RemoveFileIfPresent(); }
496 
497 void PrecompiledPreamble::TempPCHFile::RemoveFileIfPresent() {
498   if (FilePath) {
499     TemporaryFiles::getInstance().removeFile(*FilePath);
500     FilePath = None;
501   }
502 }
503 
504 llvm::StringRef PrecompiledPreamble::TempPCHFile::getFilePath() const {
505   assert(FilePath && "TempPCHFile doesn't have a FilePath. Had it been moved?");
506   return *FilePath;
507 }
508 
509 PrecompiledPreamble::PreambleFileHash
510 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
511                                                      time_t ModTime) {
512   PreambleFileHash Result;
513   Result.Size = Size;
514   Result.ModTime = ModTime;
515   Result.MD5 = {};
516   return Result;
517 }
518 
519 PrecompiledPreamble::PreambleFileHash
520 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
521     const llvm::MemoryBuffer *Buffer) {
522   PreambleFileHash Result;
523   Result.Size = Buffer->getBufferSize();
524   Result.ModTime = 0;
525 
526   llvm::MD5 MD5Ctx;
527   MD5Ctx.update(Buffer->getBuffer().data());
528   MD5Ctx.final(Result.MD5);
529 
530   return Result;
531 }
532 
533 void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
534 void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
535 void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
536 void PreambleCallbacks::HandleMacroDefined(const Token &MacroNameTok,
537                                            const MacroDirective *MD) {}
538 
539 std::error_code clang::make_error_code(BuildPreambleError Error) {
540   return std::error_code(static_cast<int>(Error), BuildPreambleErrorCategory());
541 }
542 
543 const char *BuildPreambleErrorCategory::name() const noexcept {
544   return "build-preamble.error";
545 }
546 
547 std::string BuildPreambleErrorCategory::message(int condition) const {
548   switch (static_cast<BuildPreambleError>(condition)) {
549   case BuildPreambleError::PreambleIsEmpty:
550     return "Preamble is empty";
551   case BuildPreambleError::CouldntCreateTempFile:
552     return "Could not create temporary file for PCH";
553   case BuildPreambleError::CouldntCreateTargetInfo:
554     return "CreateTargetInfo() return null";
555   case BuildPreambleError::CouldntCreateVFSOverlay:
556     return "Could not create VFS Overlay";
557   case BuildPreambleError::BeginSourceFileFailed:
558     return "BeginSourceFile() return an error";
559   case BuildPreambleError::CouldntEmitPCH:
560     return "Could not emit PCH";
561   }
562   llvm_unreachable("unexpected BuildPreambleError");
563 }
564