1 //===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 //  This file implements the CrossTranslationUnit interface.
10 //
11 //===----------------------------------------------------------------------===//
12 #include "clang/CrossTU/CrossTranslationUnit.h"
13 #include "clang/AST/ASTImporter.h"
14 #include "clang/AST/Decl.h"
15 #include "clang/Basic/TargetInfo.h"
16 #include "clang/CrossTU/CrossTUDiagnostic.h"
17 #include "clang/Frontend/ASTUnit.h"
18 #include "clang/Frontend/CompilerInstance.h"
19 #include "clang/Frontend/TextDiagnosticPrinter.h"
20 #include "clang/Index/USRGeneration.h"
21 #include "llvm/ADT/Optional.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/ADT/Triple.h"
24 #include "llvm/Option/ArgList.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/ManagedStatic.h"
27 #include "llvm/Support/Path.h"
28 #include "llvm/Support/YAMLParser.h"
29 #include "llvm/Support/raw_ostream.h"
30 #include <algorithm>
31 #include <fstream>
32 #include <sstream>
33 #include <tuple>
34 
35 namespace clang {
36 namespace cross_tu {
37 
38 namespace {
39 
40 #define DEBUG_TYPE "CrossTranslationUnit"
41 STATISTIC(NumGetCTUCalled, "The # of getCTUDefinition function called");
42 STATISTIC(
43     NumNotInOtherTU,
44     "The # of getCTUDefinition called but the function is not in any other TU");
45 STATISTIC(NumGetCTUSuccess,
46           "The # of getCTUDefinition successfully returned the "
47           "requested function's body");
48 STATISTIC(NumUnsupportedNodeFound, "The # of imports when the ASTImporter "
49                                    "encountered an unsupported AST Node");
50 STATISTIC(NumNameConflicts, "The # of imports when the ASTImporter "
51                             "encountered an ODR error");
52 STATISTIC(NumTripleMismatch, "The # of triple mismatches");
53 STATISTIC(NumLangMismatch, "The # of language mismatches");
54 STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches");
55 STATISTIC(NumASTLoadThresholdReached,
56           "The # of ASTs not loaded because of threshold");
57 
58 // Same as Triple's equality operator, but we check a field only if that is
59 // known in both instances.
60 bool hasEqualKnownFields(const llvm::Triple &Lhs, const llvm::Triple &Rhs) {
61   using llvm::Triple;
62   if (Lhs.getArch() != Triple::UnknownArch &&
63       Rhs.getArch() != Triple::UnknownArch && Lhs.getArch() != Rhs.getArch())
64     return false;
65   if (Lhs.getSubArch() != Triple::NoSubArch &&
66       Rhs.getSubArch() != Triple::NoSubArch &&
67       Lhs.getSubArch() != Rhs.getSubArch())
68     return false;
69   if (Lhs.getVendor() != Triple::UnknownVendor &&
70       Rhs.getVendor() != Triple::UnknownVendor &&
71       Lhs.getVendor() != Rhs.getVendor())
72     return false;
73   if (!Lhs.isOSUnknown() && !Rhs.isOSUnknown() &&
74       Lhs.getOS() != Rhs.getOS())
75     return false;
76   if (Lhs.getEnvironment() != Triple::UnknownEnvironment &&
77       Rhs.getEnvironment() != Triple::UnknownEnvironment &&
78       Lhs.getEnvironment() != Rhs.getEnvironment())
79     return false;
80   if (Lhs.getObjectFormat() != Triple::UnknownObjectFormat &&
81       Rhs.getObjectFormat() != Triple::UnknownObjectFormat &&
82       Lhs.getObjectFormat() != Rhs.getObjectFormat())
83     return false;
84   return true;
85 }
86 
87 // FIXME: This class is will be removed after the transition to llvm::Error.
88 class IndexErrorCategory : public std::error_category {
89 public:
90   const char *name() const noexcept override { return "clang.index"; }
91 
92   std::string message(int Condition) const override {
93     switch (static_cast<index_error_code>(Condition)) {
94     case index_error_code::unspecified:
95       return "An unknown error has occurred.";
96     case index_error_code::missing_index_file:
97       return "The index file is missing.";
98     case index_error_code::invalid_index_format:
99       return "Invalid index file format.";
100     case index_error_code::multiple_definitions:
101       return "Multiple definitions in the index file.";
102     case index_error_code::missing_definition:
103       return "Missing definition from the index file.";
104     case index_error_code::failed_import:
105       return "Failed to import the definition.";
106     case index_error_code::failed_to_get_external_ast:
107       return "Failed to load external AST source.";
108     case index_error_code::failed_to_generate_usr:
109       return "Failed to generate USR.";
110     case index_error_code::triple_mismatch:
111       return "Triple mismatch";
112     case index_error_code::lang_mismatch:
113       return "Language mismatch";
114     case index_error_code::lang_dialect_mismatch:
115       return "Language dialect mismatch";
116     case index_error_code::load_threshold_reached:
117       return "Load threshold reached";
118     case index_error_code::invocation_list_ambiguous:
119       return "Invocation list file contains multiple references to the same "
120              "source file.";
121     case index_error_code::invocation_list_file_not_found:
122       return "Invocation list file is not found.";
123     case index_error_code::invocation_list_empty:
124       return "Invocation list file is empty.";
125     case index_error_code::invocation_list_wrong_format:
126       return "Invocation list file is in wrong format.";
127     case index_error_code::invocation_list_lookup_unsuccessful:
128       return "Invocation list file does not contain the requested source file.";
129     }
130     llvm_unreachable("Unrecognized index_error_code.");
131   }
132 };
133 
134 static llvm::ManagedStatic<IndexErrorCategory> Category;
135 } // end anonymous namespace
136 
137 char IndexError::ID;
138 
139 void IndexError::log(raw_ostream &OS) const {
140   OS << Category->message(static_cast<int>(Code)) << '\n';
141 }
142 
143 std::error_code IndexError::convertToErrorCode() const {
144   return std::error_code(static_cast<int>(Code), *Category);
145 }
146 
147 llvm::Expected<llvm::StringMap<std::string>>
148 parseCrossTUIndex(StringRef IndexPath) {
149   std::ifstream ExternalMapFile{std::string(IndexPath)};
150   if (!ExternalMapFile)
151     return llvm::make_error<IndexError>(index_error_code::missing_index_file,
152                                         IndexPath.str());
153 
154   llvm::StringMap<std::string> Result;
155   std::string Line;
156   unsigned LineNo = 1;
157   while (std::getline(ExternalMapFile, Line)) {
158     StringRef LineRef{Line};
159     const size_t Delimiter = LineRef.find(" ");
160     if (Delimiter > 0 && Delimiter != std::string::npos) {
161       StringRef LookupName = LineRef.substr(0, Delimiter);
162 
163       // Store paths with posix-style directory separator.
164       SmallVector<char, 32> FilePath;
165       llvm::Twine{LineRef.substr(Delimiter + 1)}.toVector(FilePath);
166       llvm::sys::path::native(FilePath, llvm::sys::path::Style::posix);
167 
168       bool InsertionOccured;
169       std::tie(std::ignore, InsertionOccured) =
170           Result.try_emplace(LookupName, FilePath.begin(), FilePath.end());
171       if (!InsertionOccured)
172         return llvm::make_error<IndexError>(
173             index_error_code::multiple_definitions, IndexPath.str(), LineNo);
174     } else
175       return llvm::make_error<IndexError>(
176           index_error_code::invalid_index_format, IndexPath.str(), LineNo);
177     ++LineNo;
178   }
179   return Result;
180 }
181 
182 std::string
183 createCrossTUIndexString(const llvm::StringMap<std::string> &Index) {
184   std::ostringstream Result;
185   for (const auto &E : Index)
186     Result << E.getKey().str() << " " << E.getValue() << '\n';
187   return Result.str();
188 }
189 
190 bool containsConst(const VarDecl *VD, const ASTContext &ACtx) {
191   CanQualType CT = ACtx.getCanonicalType(VD->getType());
192   if (!CT.isConstQualified()) {
193     const RecordType *RTy = CT->getAs<RecordType>();
194     if (!RTy || !RTy->hasConstFields())
195       return false;
196   }
197   return true;
198 }
199 
200 static bool hasBodyOrInit(const FunctionDecl *D, const FunctionDecl *&DefD) {
201   return D->hasBody(DefD);
202 }
203 static bool hasBodyOrInit(const VarDecl *D, const VarDecl *&DefD) {
204   return D->getAnyInitializer(DefD);
205 }
206 template <typename T> static bool hasBodyOrInit(const T *D) {
207   const T *Unused;
208   return hasBodyOrInit(D, Unused);
209 }
210 
211 CrossTranslationUnitContext::CrossTranslationUnitContext(CompilerInstance &CI)
212     : Context(CI.getASTContext()), ASTStorage(CI) {}
213 
214 CrossTranslationUnitContext::~CrossTranslationUnitContext() {}
215 
216 llvm::Optional<std::string>
217 CrossTranslationUnitContext::getLookupName(const NamedDecl *ND) {
218   SmallString<128> DeclUSR;
219   bool Ret = index::generateUSRForDecl(ND, DeclUSR);
220   if (Ret)
221     return {};
222   return std::string(DeclUSR.str());
223 }
224 
225 /// Recursively visits the decls of a DeclContext, and returns one with the
226 /// given USR.
227 template <typename T>
228 const T *
229 CrossTranslationUnitContext::findDefInDeclContext(const DeclContext *DC,
230                                                   StringRef LookupName) {
231   assert(DC && "Declaration Context must not be null");
232   for (const Decl *D : DC->decls()) {
233     const auto *SubDC = dyn_cast<DeclContext>(D);
234     if (SubDC)
235       if (const auto *ND = findDefInDeclContext<T>(SubDC, LookupName))
236         return ND;
237 
238     const auto *ND = dyn_cast<T>(D);
239     const T *ResultDecl;
240     if (!ND || !hasBodyOrInit(ND, ResultDecl))
241       continue;
242     llvm::Optional<std::string> ResultLookupName = getLookupName(ResultDecl);
243     if (!ResultLookupName || *ResultLookupName != LookupName)
244       continue;
245     return ResultDecl;
246   }
247   return nullptr;
248 }
249 
250 template <typename T>
251 llvm::Expected<const T *> CrossTranslationUnitContext::getCrossTUDefinitionImpl(
252     const T *D, StringRef CrossTUDir, StringRef IndexName,
253     bool DisplayCTUProgress) {
254   assert(D && "D is missing, bad call to this function!");
255   assert(!hasBodyOrInit(D) &&
256          "D has a body or init in current translation unit!");
257   ++NumGetCTUCalled;
258   const llvm::Optional<std::string> LookupName = getLookupName(D);
259   if (!LookupName)
260     return llvm::make_error<IndexError>(
261         index_error_code::failed_to_generate_usr);
262   llvm::Expected<ASTUnit *> ASTUnitOrError =
263       loadExternalAST(*LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
264   if (!ASTUnitOrError)
265     return ASTUnitOrError.takeError();
266   ASTUnit *Unit = *ASTUnitOrError;
267   assert(&Unit->getFileManager() ==
268          &Unit->getASTContext().getSourceManager().getFileManager());
269 
270   const llvm::Triple &TripleTo = Context.getTargetInfo().getTriple();
271   const llvm::Triple &TripleFrom =
272       Unit->getASTContext().getTargetInfo().getTriple();
273   // The imported AST had been generated for a different target.
274   // Some parts of the triple in the loaded ASTContext can be unknown while the
275   // very same parts in the target ASTContext are known. Thus we check for the
276   // known parts only.
277   if (!hasEqualKnownFields(TripleTo, TripleFrom)) {
278     // TODO: Pass the SourceLocation of the CallExpression for more precise
279     // diagnostics.
280     ++NumTripleMismatch;
281     return llvm::make_error<IndexError>(index_error_code::triple_mismatch,
282                                         std::string(Unit->getMainFileName()),
283                                         TripleTo.str(), TripleFrom.str());
284   }
285 
286   const auto &LangTo = Context.getLangOpts();
287   const auto &LangFrom = Unit->getASTContext().getLangOpts();
288 
289   // FIXME: Currenty we do not support CTU across C++ and C and across
290   // different dialects of C++.
291   if (LangTo.CPlusPlus != LangFrom.CPlusPlus) {
292     ++NumLangMismatch;
293     return llvm::make_error<IndexError>(index_error_code::lang_mismatch);
294   }
295 
296   // If CPP dialects are different then return with error.
297   //
298   // Consider this STL code:
299   //   template<typename _Alloc>
300   //     struct __alloc_traits
301   //   #if __cplusplus >= 201103L
302   //     : std::allocator_traits<_Alloc>
303   //   #endif
304   //     { // ...
305   //     };
306   // This class template would create ODR errors during merging the two units,
307   // since in one translation unit the class template has a base class, however
308   // in the other unit it has none.
309   if (LangTo.CPlusPlus11 != LangFrom.CPlusPlus11 ||
310       LangTo.CPlusPlus14 != LangFrom.CPlusPlus14 ||
311       LangTo.CPlusPlus17 != LangFrom.CPlusPlus17 ||
312       LangTo.CPlusPlus20 != LangFrom.CPlusPlus20) {
313     ++NumLangDialectMismatch;
314     return llvm::make_error<IndexError>(
315         index_error_code::lang_dialect_mismatch);
316   }
317 
318   TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl();
319   if (const T *ResultDecl = findDefInDeclContext<T>(TU, *LookupName))
320     return importDefinition(ResultDecl, Unit);
321   return llvm::make_error<IndexError>(index_error_code::failed_import);
322 }
323 
324 llvm::Expected<const FunctionDecl *>
325 CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD,
326                                                   StringRef CrossTUDir,
327                                                   StringRef IndexName,
328                                                   bool DisplayCTUProgress) {
329   return getCrossTUDefinitionImpl(FD, CrossTUDir, IndexName,
330                                   DisplayCTUProgress);
331 }
332 
333 llvm::Expected<const VarDecl *>
334 CrossTranslationUnitContext::getCrossTUDefinition(const VarDecl *VD,
335                                                   StringRef CrossTUDir,
336                                                   StringRef IndexName,
337                                                   bool DisplayCTUProgress) {
338   return getCrossTUDefinitionImpl(VD, CrossTUDir, IndexName,
339                                   DisplayCTUProgress);
340 }
341 
342 void CrossTranslationUnitContext::emitCrossTUDiagnostics(const IndexError &IE) {
343   switch (IE.getCode()) {
344   case index_error_code::missing_index_file:
345     Context.getDiagnostics().Report(diag::err_ctu_error_opening)
346         << IE.getFileName();
347     break;
348   case index_error_code::invalid_index_format:
349     Context.getDiagnostics().Report(diag::err_extdefmap_parsing)
350         << IE.getFileName() << IE.getLineNum();
351     break;
352   case index_error_code::multiple_definitions:
353     Context.getDiagnostics().Report(diag::err_multiple_def_index)
354         << IE.getLineNum();
355     break;
356   case index_error_code::triple_mismatch:
357     Context.getDiagnostics().Report(diag::warn_ctu_incompat_triple)
358         << IE.getFileName() << IE.getTripleToName() << IE.getTripleFromName();
359     break;
360   default:
361     break;
362   }
363 }
364 
365 CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
366     CompilerInstance &CI)
367     : Loader(CI, CI.getAnalyzerOpts()->CTUDir,
368              CI.getAnalyzerOpts()->CTUInvocationList),
369       LoadGuard(CI.getAnalyzerOpts()->CTUImportThreshold) {}
370 
371 llvm::Expected<ASTUnit *>
372 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
373     StringRef FileName, bool DisplayCTUProgress) {
374   // Try the cache first.
375   auto ASTCacheEntry = FileASTUnitMap.find(FileName);
376   if (ASTCacheEntry == FileASTUnitMap.end()) {
377 
378     // Do not load if the limit is reached.
379     if (!LoadGuard) {
380       ++NumASTLoadThresholdReached;
381       return llvm::make_error<IndexError>(
382           index_error_code::load_threshold_reached);
383     }
384 
385     auto LoadAttempt = Loader.load(FileName);
386 
387     if (!LoadAttempt)
388       return LoadAttempt.takeError();
389 
390     std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
391 
392     // Need the raw pointer and the unique_ptr as well.
393     ASTUnit *Unit = LoadedUnit.get();
394 
395     // Update the cache.
396     FileASTUnitMap[FileName] = std::move(LoadedUnit);
397 
398     LoadGuard.indicateLoadSuccess();
399 
400     if (DisplayCTUProgress)
401       llvm::errs() << "CTU loaded AST file: " << FileName << "\n";
402 
403     return Unit;
404 
405   } else {
406     // Found in the cache.
407     return ASTCacheEntry->second.get();
408   }
409 }
410 
411 llvm::Expected<ASTUnit *>
412 CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFunction(
413     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName,
414     bool DisplayCTUProgress) {
415   // Try the cache first.
416   auto ASTCacheEntry = NameASTUnitMap.find(FunctionName);
417   if (ASTCacheEntry == NameASTUnitMap.end()) {
418     // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
419 
420     // Ensure that the Index is loaded, as we need to search in it.
421     if (llvm::Error IndexLoadError =
422             ensureCTUIndexLoaded(CrossTUDir, IndexName))
423       return std::move(IndexLoadError);
424 
425     // Check if there is and entry in the index for the function.
426     if (!NameFileMap.count(FunctionName)) {
427       ++NumNotInOtherTU;
428       return llvm::make_error<IndexError>(index_error_code::missing_definition);
429     }
430 
431     // Search in the index for the filename where the definition of FuncitonName
432     // resides.
433     if (llvm::Expected<ASTUnit *> FoundForFile =
434             getASTUnitForFile(NameFileMap[FunctionName], DisplayCTUProgress)) {
435 
436       // Update the cache.
437       NameASTUnitMap[FunctionName] = *FoundForFile;
438       return *FoundForFile;
439 
440     } else {
441       return FoundForFile.takeError();
442     }
443   } else {
444     // Found in the cache.
445     return ASTCacheEntry->second;
446   }
447 }
448 
449 llvm::Expected<std::string>
450 CrossTranslationUnitContext::ASTUnitStorage::getFileForFunction(
451     StringRef FunctionName, StringRef CrossTUDir, StringRef IndexName) {
452   if (llvm::Error IndexLoadError = ensureCTUIndexLoaded(CrossTUDir, IndexName))
453     return std::move(IndexLoadError);
454   return NameFileMap[FunctionName];
455 }
456 
457 llvm::Error CrossTranslationUnitContext::ASTUnitStorage::ensureCTUIndexLoaded(
458     StringRef CrossTUDir, StringRef IndexName) {
459   // Dont initialize if the map is filled.
460   if (!NameFileMap.empty())
461     return llvm::Error::success();
462 
463   // Get the absolute path to the index file.
464   SmallString<256> IndexFile = CrossTUDir;
465   if (llvm::sys::path::is_absolute(IndexName))
466     IndexFile = IndexName;
467   else
468     llvm::sys::path::append(IndexFile, IndexName);
469 
470   if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
471     // Initialize member map.
472     NameFileMap = *IndexMapping;
473     return llvm::Error::success();
474   } else {
475     // Error while parsing CrossTU index file.
476     return IndexMapping.takeError();
477   };
478 }
479 
480 llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
481     StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
482     bool DisplayCTUProgress) {
483   // FIXME: The current implementation only supports loading decls with
484   //        a lookup name from a single translation unit. If multiple
485   //        translation units contains decls with the same lookup name an
486   //        error will be returned.
487 
488   // Try to get the value from the heavily cached storage.
489   llvm::Expected<ASTUnit *> Unit = ASTStorage.getASTUnitForFunction(
490       LookupName, CrossTUDir, IndexName, DisplayCTUProgress);
491 
492   if (!Unit)
493     return Unit.takeError();
494 
495   // Check whether the backing pointer of the Expected is a nullptr.
496   if (!*Unit)
497     return llvm::make_error<IndexError>(
498         index_error_code::failed_to_get_external_ast);
499 
500   return Unit;
501 }
502 
503 CrossTranslationUnitContext::ASTLoader::ASTLoader(
504     CompilerInstance &CI, StringRef CTUDir, StringRef InvocationListFilePath)
505     : CI(CI), CTUDir(CTUDir), InvocationListFilePath(InvocationListFilePath) {}
506 
507 CrossTranslationUnitContext::LoadResultTy
508 CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) {
509   llvm::SmallString<256> Path;
510   if (llvm::sys::path::is_absolute(Identifier, PathStyle)) {
511     Path = Identifier;
512   } else {
513     Path = CTUDir;
514     llvm::sys::path::append(Path, PathStyle, Identifier);
515   }
516 
517   // The path is stored in the InvocationList member in posix style. To
518   // successfully lookup an entry based on filepath, it must be converted.
519   llvm::sys::path::native(Path, PathStyle);
520 
521   // Normalize by removing relative path components.
522   llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle);
523 
524   if (Path.endswith(".ast"))
525     return loadFromDump(Path);
526   else
527     return loadFromSource(Path);
528 }
529 
530 CrossTranslationUnitContext::LoadResultTy
531 CrossTranslationUnitContext::ASTLoader::loadFromDump(StringRef ASTDumpPath) {
532   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
533   TextDiagnosticPrinter *DiagClient =
534       new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts);
535   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
536   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
537       new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
538   return ASTUnit::LoadFromASTFile(
539       std::string(ASTDumpPath.str()),
540       CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
541       Diags, CI.getFileSystemOpts());
542 }
543 
544 /// Load the AST from a source-file, which is supposed to be located inside the
545 /// YAML formatted invocation list file under the filesystem path specified by
546 /// \p InvocationList. The invocation list should contain absolute paths.
547 /// \p SourceFilePath is the absolute path of the source file that contains the
548 /// function definition the analysis is looking for. The Index is built by the
549 /// \p clang-extdef-mapping tool, which is also supposed to be generating
550 /// absolute paths.
551 ///
552 /// Proper diagnostic emission requires absolute paths, so even if a future
553 /// change introduces the handling of relative paths, this must be taken into
554 /// consideration.
555 CrossTranslationUnitContext::LoadResultTy
556 CrossTranslationUnitContext::ASTLoader::loadFromSource(
557     StringRef SourceFilePath) {
558 
559   if (llvm::Error InitError = lazyInitInvocationList())
560     return std::move(InitError);
561   assert(InvocationList);
562 
563   auto Invocation = InvocationList->find(SourceFilePath);
564   if (Invocation == InvocationList->end())
565     return llvm::make_error<IndexError>(
566         index_error_code::invocation_list_lookup_unsuccessful);
567 
568   const InvocationListTy::mapped_type &InvocationCommand = Invocation->second;
569 
570   SmallVector<const char *, 32> CommandLineArgs(InvocationCommand.size());
571   std::transform(InvocationCommand.begin(), InvocationCommand.end(),
572                  CommandLineArgs.begin(),
573                  [](auto &&CmdPart) { return CmdPart.c_str(); });
574 
575   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts{&CI.getDiagnosticOpts()};
576   auto *DiagClient = new ForwardingDiagnosticConsumer{CI.getDiagnosticClient()};
577   IntrusiveRefCntPtr<DiagnosticIDs> DiagID{
578       CI.getDiagnostics().getDiagnosticIDs()};
579   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
580       new DiagnosticsEngine{DiagID, &*DiagOpts, DiagClient});
581 
582   return std::unique_ptr<ASTUnit>(ASTUnit::LoadFromCommandLine(
583       CommandLineArgs.begin(), (CommandLineArgs.end()),
584       CI.getPCHContainerOperations(), Diags,
585       CI.getHeaderSearchOpts().ResourceDir));
586 }
587 
588 llvm::Expected<InvocationListTy>
589 parseInvocationList(StringRef FileContent, llvm::sys::path::Style PathStyle) {
590   InvocationListTy InvocationList;
591 
592   /// LLVM YAML parser is used to extract information from invocation list file.
593   llvm::SourceMgr SM;
594   llvm::yaml::Stream InvocationFile(FileContent, SM);
595 
596   /// Only the first document is processed.
597   llvm::yaml::document_iterator FirstInvocationFile = InvocationFile.begin();
598 
599   /// There has to be at least one document available.
600   if (FirstInvocationFile == InvocationFile.end())
601     return llvm::make_error<IndexError>(
602         index_error_code::invocation_list_empty);
603 
604   llvm::yaml::Node *DocumentRoot = FirstInvocationFile->getRoot();
605   if (!DocumentRoot)
606     return llvm::make_error<IndexError>(
607         index_error_code::invocation_list_wrong_format);
608 
609   /// According to the format specified the document must be a mapping, where
610   /// the keys are paths to source files, and values are sequences of invocation
611   /// parts.
612   auto *Mappings = dyn_cast<llvm::yaml::MappingNode>(DocumentRoot);
613   if (!Mappings)
614     return llvm::make_error<IndexError>(
615         index_error_code::invocation_list_wrong_format);
616 
617   for (auto &NextMapping : *Mappings) {
618     /// The keys should be strings, which represent a source-file path.
619     auto *Key = dyn_cast<llvm::yaml::ScalarNode>(NextMapping.getKey());
620     if (!Key)
621       return llvm::make_error<IndexError>(
622           index_error_code::invocation_list_wrong_format);
623 
624     SmallVector<char, 32> ValueStorage;
625     StringRef SourcePath = Key->getValue(ValueStorage);
626 
627     // Store paths with PathStyle directory separator.
628     SmallVector<char, 32> NativeSourcePath;
629     llvm::Twine{SourcePath}.toVector(NativeSourcePath);
630     llvm::sys::path::native(NativeSourcePath, PathStyle);
631 
632     StringRef InvocationKey{NativeSourcePath.begin(), NativeSourcePath.size()};
633 
634     if (InvocationList.find(InvocationKey) != InvocationList.end())
635       return llvm::make_error<IndexError>(
636           index_error_code::invocation_list_ambiguous);
637 
638     /// The values should be sequences of strings, each representing a part of
639     /// the invocation.
640     auto *Args = dyn_cast<llvm::yaml::SequenceNode>(NextMapping.getValue());
641     if (!Args)
642       return llvm::make_error<IndexError>(
643           index_error_code::invocation_list_wrong_format);
644 
645     for (auto &Arg : *Args) {
646       auto *CmdString = dyn_cast<llvm::yaml::ScalarNode>(&Arg);
647       if (!CmdString)
648         return llvm::make_error<IndexError>(
649             index_error_code::invocation_list_wrong_format);
650       /// Every conversion starts with an empty working storage, as it is not
651       /// clear if this is a requirement of the YAML parser.
652       ValueStorage.clear();
653       InvocationList[InvocationKey].emplace_back(
654           CmdString->getValue(ValueStorage));
655     }
656 
657     if (InvocationList[InvocationKey].empty())
658       return llvm::make_error<IndexError>(
659           index_error_code::invocation_list_wrong_format);
660   }
661 
662   return InvocationList;
663 }
664 
665 llvm::Error CrossTranslationUnitContext::ASTLoader::lazyInitInvocationList() {
666   /// Lazily initialize the invocation list member used for on-demand parsing.
667   if (InvocationList)
668     return llvm::Error::success();
669 
670   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileContent =
671       llvm::MemoryBuffer::getFile(InvocationListFilePath);
672   if (!FileContent)
673     return llvm::make_error<IndexError>(
674         index_error_code::invocation_list_file_not_found);
675   std::unique_ptr<llvm::MemoryBuffer> ContentBuffer = std::move(*FileContent);
676   assert(ContentBuffer && "If no error was produced after loading, the pointer "
677                           "should not be nullptr.");
678 
679   llvm::Expected<InvocationListTy> ExpectedInvocationList =
680       parseInvocationList(ContentBuffer->getBuffer(), PathStyle);
681 
682   if (!ExpectedInvocationList)
683     return ExpectedInvocationList.takeError();
684 
685   InvocationList = *ExpectedInvocationList;
686 
687   return llvm::Error::success();
688 }
689 
690 template <typename T>
691 llvm::Expected<const T *>
692 CrossTranslationUnitContext::importDefinitionImpl(const T *D, ASTUnit *Unit) {
693   assert(hasBodyOrInit(D) && "Decls to be imported should have body or init.");
694 
695   assert(&D->getASTContext() == &Unit->getASTContext() &&
696          "ASTContext of Decl and the unit should match.");
697   ASTImporter &Importer = getOrCreateASTImporter(Unit);
698 
699   auto ToDeclOrError = Importer.Import(D);
700   if (!ToDeclOrError) {
701     handleAllErrors(ToDeclOrError.takeError(),
702                     [&](const ImportError &IE) {
703                       switch (IE.Error) {
704                       case ImportError::NameConflict:
705                         ++NumNameConflicts;
706                          break;
707                       case ImportError::UnsupportedConstruct:
708                         ++NumUnsupportedNodeFound;
709                         break;
710                       case ImportError::Unknown:
711                         llvm_unreachable("Unknown import error happened.");
712                         break;
713                       }
714                     });
715     return llvm::make_error<IndexError>(index_error_code::failed_import);
716   }
717   auto *ToDecl = cast<T>(*ToDeclOrError);
718   assert(hasBodyOrInit(ToDecl) && "Imported Decl should have body or init.");
719   ++NumGetCTUSuccess;
720 
721   return ToDecl;
722 }
723 
724 llvm::Expected<const FunctionDecl *>
725 CrossTranslationUnitContext::importDefinition(const FunctionDecl *FD,
726                                               ASTUnit *Unit) {
727   return importDefinitionImpl(FD, Unit);
728 }
729 
730 llvm::Expected<const VarDecl *>
731 CrossTranslationUnitContext::importDefinition(const VarDecl *VD,
732                                               ASTUnit *Unit) {
733   return importDefinitionImpl(VD, Unit);
734 }
735 
736 void CrossTranslationUnitContext::lazyInitImporterSharedSt(
737     TranslationUnitDecl *ToTU) {
738   if (!ImporterSharedSt)
739     ImporterSharedSt = std::make_shared<ASTImporterSharedState>(*ToTU);
740 }
741 
742 ASTImporter &
743 CrossTranslationUnitContext::getOrCreateASTImporter(ASTUnit *Unit) {
744   ASTContext &From = Unit->getASTContext();
745 
746   auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl());
747   if (I != ASTUnitImporterMap.end())
748     return *I->second;
749   lazyInitImporterSharedSt(Context.getTranslationUnitDecl());
750   ASTImporter *NewImporter = new ASTImporter(
751       Context, Context.getSourceManager().getFileManager(), From,
752       From.getSourceManager().getFileManager(), false, ImporterSharedSt);
753   NewImporter->setFileIDImportHandler([this, Unit](FileID ToID, FileID FromID) {
754     assert(ImportedFileIDs.find(ToID) == ImportedFileIDs.end() &&
755            "FileID already imported, should not happen.");
756     ImportedFileIDs[ToID] = std::make_pair(FromID, Unit);
757   });
758   ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter);
759   return *NewImporter;
760 }
761 
762 llvm::Optional<std::pair<SourceLocation, ASTUnit *>>
763 CrossTranslationUnitContext::getImportedFromSourceLocation(
764     const clang::SourceLocation &ToLoc) const {
765   const SourceManager &SM = Context.getSourceManager();
766   auto DecToLoc = SM.getDecomposedLoc(ToLoc);
767 
768   auto I = ImportedFileIDs.find(DecToLoc.first);
769   if (I == ImportedFileIDs.end())
770     return {};
771 
772   FileID FromID = I->second.first;
773   clang::ASTUnit *Unit = I->second.second;
774   SourceLocation FromLoc =
775       Unit->getSourceManager().getComposedLoc(FromID, DecToLoc.second);
776 
777   return std::make_pair(FromLoc, Unit);
778 }
779 
780 } // namespace cross_tu
781 } // namespace clang
782