142418abaSMehdi Amini //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
242418abaSMehdi Amini //
342418abaSMehdi Amini //                     The LLVM Compiler Infrastructure
442418abaSMehdi Amini //
542418abaSMehdi Amini // This file is distributed under the University of Illinois Open Source
642418abaSMehdi Amini // License. See LICENSE.TXT for details.
742418abaSMehdi Amini //
842418abaSMehdi Amini //===----------------------------------------------------------------------===//
942418abaSMehdi Amini //
1042418abaSMehdi Amini // This file implements Function import based on summaries.
1142418abaSMehdi Amini //
1242418abaSMehdi Amini //===----------------------------------------------------------------------===//
1342418abaSMehdi Amini 
1442418abaSMehdi Amini #include "llvm/Transforms/IPO/FunctionImport.h"
15e9ea08a0SEugene Zelenko #include "llvm/ADT/ArrayRef.h"
16e9ea08a0SEugene Zelenko #include "llvm/ADT/STLExtras.h"
17e9ea08a0SEugene Zelenko #include "llvm/ADT/SetVector.h"
1801e32130SMehdi Amini #include "llvm/ADT/SmallVector.h"
19d29478f7STeresa Johnson #include "llvm/ADT/Statistic.h"
20e9ea08a0SEugene Zelenko #include "llvm/ADT/StringMap.h"
210efed325SSimon Pilgrim #include "llvm/ADT/StringRef.h"
22b040fcc6SCharles Saternos #include "llvm/ADT/StringSet.h"
23c15d60b7SPeter Collingbourne #include "llvm/Bitcode/BitcodeReader.h"
2442418abaSMehdi Amini #include "llvm/IR/AutoUpgrade.h"
2581bbf742STeresa Johnson #include "llvm/IR/Constants.h"
26e9ea08a0SEugene Zelenko #include "llvm/IR/Function.h"
27e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalAlias.h"
28e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalObject.h"
29e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalValue.h"
30e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalVariable.h"
31e9ea08a0SEugene Zelenko #include "llvm/IR/Metadata.h"
3242418abaSMehdi Amini #include "llvm/IR/Module.h"
33e9ea08a0SEugene Zelenko #include "llvm/IR/ModuleSummaryIndex.h"
3442418abaSMehdi Amini #include "llvm/IRReader/IRReader.h"
35e9ea08a0SEugene Zelenko #include "llvm/Linker/IRMover.h"
36e9ea08a0SEugene Zelenko #include "llvm/Object/ModuleSymbolTable.h"
37e9ea08a0SEugene Zelenko #include "llvm/Object/SymbolicFile.h"
38e9ea08a0SEugene Zelenko #include "llvm/Pass.h"
39e9ea08a0SEugene Zelenko #include "llvm/Support/Casting.h"
4042418abaSMehdi Amini #include "llvm/Support/CommandLine.h"
4142418abaSMehdi Amini #include "llvm/Support/Debug.h"
42e9ea08a0SEugene Zelenko #include "llvm/Support/Error.h"
43e9ea08a0SEugene Zelenko #include "llvm/Support/ErrorHandling.h"
44e9ea08a0SEugene Zelenko #include "llvm/Support/FileSystem.h"
4542418abaSMehdi Amini #include "llvm/Support/SourceMgr.h"
46e9ea08a0SEugene Zelenko #include "llvm/Support/raw_ostream.h"
4704c9a2d6STeresa Johnson #include "llvm/Transforms/IPO/Internalize.h"
4881bbf742STeresa Johnson #include "llvm/Transforms/Utils/Cloning.h"
49488a800aSTeresa Johnson #include "llvm/Transforms/Utils/FunctionImportUtils.h"
5081bbf742STeresa Johnson #include "llvm/Transforms/Utils/ValueMapper.h"
51e9ea08a0SEugene Zelenko #include <cassert>
52e9ea08a0SEugene Zelenko #include <memory>
53e9ea08a0SEugene Zelenko #include <set>
54e9ea08a0SEugene Zelenko #include <string>
55e9ea08a0SEugene Zelenko #include <system_error>
56e9ea08a0SEugene Zelenko #include <tuple>
57e9ea08a0SEugene Zelenko #include <utility>
587e88d0daSMehdi Amini 
5942418abaSMehdi Amini using namespace llvm;
6042418abaSMehdi Amini 
61e9ea08a0SEugene Zelenko #define DEBUG_TYPE "function-import"
62e9ea08a0SEugene Zelenko 
63d2c234a4STeresa Johnson STATISTIC(NumImportedFunctionsThinLink,
64d2c234a4STeresa Johnson           "Number of functions thin link decided to import");
65d2c234a4STeresa Johnson STATISTIC(NumImportedHotFunctionsThinLink,
66d2c234a4STeresa Johnson           "Number of hot functions thin link decided to import");
67d2c234a4STeresa Johnson STATISTIC(NumImportedCriticalFunctionsThinLink,
68d2c234a4STeresa Johnson           "Number of critical functions thin link decided to import");
69d2c234a4STeresa Johnson STATISTIC(NumImportedGlobalVarsThinLink,
70d2c234a4STeresa Johnson           "Number of global variables thin link decided to import");
71d2c234a4STeresa Johnson STATISTIC(NumImportedFunctions, "Number of functions imported in backend");
72d2c234a4STeresa Johnson STATISTIC(NumImportedGlobalVars,
73d2c234a4STeresa Johnson           "Number of global variables imported in backend");
746c475a75STeresa Johnson STATISTIC(NumImportedModules, "Number of modules imported from");
756c475a75STeresa Johnson STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");
766c475a75STeresa Johnson STATISTIC(NumLiveSymbols, "Number of live symbols in index");
77d29478f7STeresa Johnson 
7839303619STeresa Johnson /// Limit on instruction count of imported functions.
7939303619STeresa Johnson static cl::opt<unsigned> ImportInstrLimit(
8039303619STeresa Johnson     "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"),
8139303619STeresa Johnson     cl::desc("Only import functions with less than N instructions"));
8239303619STeresa Johnson 
83974706ebSTeresa Johnson static cl::opt<int> ImportCutoff(
84974706ebSTeresa Johnson     "import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"),
85974706ebSTeresa Johnson     cl::desc("Only import first N functions if N>=0 (default -1)"));
86974706ebSTeresa Johnson 
8740641748SMehdi Amini static cl::opt<float>
8840641748SMehdi Amini     ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7),
8940641748SMehdi Amini                       cl::Hidden, cl::value_desc("x"),
9040641748SMehdi Amini                       cl::desc("As we import functions, multiply the "
9140641748SMehdi Amini                                "`import-instr-limit` threshold by this factor "
9240641748SMehdi Amini                                "before processing newly imported functions"));
93ba72b95fSPiotr Padlewski 
94d2869473SPiotr Padlewski static cl::opt<float> ImportHotInstrFactor(
95d2869473SPiotr Padlewski     "import-hot-evolution-factor", cl::init(1.0), cl::Hidden,
96d2869473SPiotr Padlewski     cl::value_desc("x"),
97d2869473SPiotr Padlewski     cl::desc("As we import functions called from hot callsite, multiply the "
98d2869473SPiotr Padlewski              "`import-instr-limit` threshold by this factor "
99d2869473SPiotr Padlewski              "before processing newly imported functions"));
100d2869473SPiotr Padlewski 
101d9830eb7SPiotr Padlewski static cl::opt<float> ImportHotMultiplier(
1028260d665SDehao Chen     "import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"),
103ba72b95fSPiotr Padlewski     cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));
104ba72b95fSPiotr Padlewski 
10564c46574SDehao Chen static cl::opt<float> ImportCriticalMultiplier(
10664c46574SDehao Chen     "import-critical-multiplier", cl::init(100.0), cl::Hidden,
10764c46574SDehao Chen     cl::value_desc("x"),
10864c46574SDehao Chen     cl::desc(
10964c46574SDehao Chen         "Multiply the `import-instr-limit` threshold for critical callsites"));
11064c46574SDehao Chen 
111ba72b95fSPiotr Padlewski // FIXME: This multiplier was not really tuned up.
112ba72b95fSPiotr Padlewski static cl::opt<float> ImportColdMultiplier(
113ba72b95fSPiotr Padlewski     "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"),
114ba72b95fSPiotr Padlewski     cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));
11540641748SMehdi Amini 
116d29478f7STeresa Johnson static cl::opt<bool> PrintImports("print-imports", cl::init(false), cl::Hidden,
117d29478f7STeresa Johnson                                   cl::desc("Print imported functions"));
118d29478f7STeresa Johnson 
119cb9a82fcSTeresa Johnson static cl::opt<bool> PrintImportFailures(
120cb9a82fcSTeresa Johnson     "print-import-failures", cl::init(false), cl::Hidden,
121cb9a82fcSTeresa Johnson     cl::desc("Print information for functions rejected for importing"));
122cb9a82fcSTeresa Johnson 
1236c475a75STeresa Johnson static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden,
1246c475a75STeresa Johnson                                  cl::desc("Compute dead symbols"));
1256c475a75STeresa Johnson 
1263b776128SPiotr Padlewski static cl::opt<bool> EnableImportMetadata(
1273b776128SPiotr Padlewski     "enable-import-metadata", cl::init(
1283b776128SPiotr Padlewski #if !defined(NDEBUG)
1293b776128SPiotr Padlewski                                   true /*Enabled with asserts.*/
1303b776128SPiotr Padlewski #else
1313b776128SPiotr Padlewski                                   false
1323b776128SPiotr Padlewski #endif
1333b776128SPiotr Padlewski                                   ),
1343b776128SPiotr Padlewski     cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module'"));
1353b776128SPiotr Padlewski 
136e9ea08a0SEugene Zelenko /// Summary file to use for function importing when using -function-import from
137e9ea08a0SEugene Zelenko /// the command line.
138e9ea08a0SEugene Zelenko static cl::opt<std::string>
139e9ea08a0SEugene Zelenko     SummaryFile("summary-file",
140e9ea08a0SEugene Zelenko                 cl::desc("The summary file to use for function importing."));
141e9ea08a0SEugene Zelenko 
14281bbf742STeresa Johnson /// Used when testing importing from distributed indexes via opt
14381bbf742STeresa Johnson // -function-import.
14481bbf742STeresa Johnson static cl::opt<bool>
14581bbf742STeresa Johnson     ImportAllIndex("import-all-index",
14681bbf742STeresa Johnson                    cl::desc("Import all external functions in index."));
14781bbf742STeresa Johnson 
14842418abaSMehdi Amini // Load lazily a module from \p FileName in \p Context.
14942418abaSMehdi Amini static std::unique_ptr<Module> loadFile(const std::string &FileName,
15042418abaSMehdi Amini                                         LLVMContext &Context) {
15142418abaSMehdi Amini   SMDiagnostic Err;
152d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");
1536cba37ceSTeresa Johnson   // Metadata isn't loaded until functions are imported, to minimize
1546cba37ceSTeresa Johnson   // the memory overhead.
155a1080ee6STeresa Johnson   std::unique_ptr<Module> Result =
156a1080ee6STeresa Johnson       getLazyIRFileModule(FileName, Err, Context,
157a1080ee6STeresa Johnson                           /* ShouldLazyLoadMetadata = */ true);
15842418abaSMehdi Amini   if (!Result) {
15942418abaSMehdi Amini     Err.print("function-import", errs());
160d7ad221cSMehdi Amini     report_fatal_error("Abort");
16142418abaSMehdi Amini   }
16242418abaSMehdi Amini 
16342418abaSMehdi Amini   return Result;
16442418abaSMehdi Amini }
16542418abaSMehdi Amini 
16601e32130SMehdi Amini /// Given a list of possible callee implementation for a call site, select one
16701e32130SMehdi Amini /// that fits the \p Threshold.
16801e32130SMehdi Amini ///
16901e32130SMehdi Amini /// FIXME: select "best" instead of first that fits. But what is "best"?
17001e32130SMehdi Amini /// - The smallest: more likely to be inlined.
17101e32130SMehdi Amini /// - The one with the least outgoing edges (already well optimized).
17201e32130SMehdi Amini /// - One from a module already being imported from in order to reduce the
17301e32130SMehdi Amini ///   number of source modules parsed/linked.
17401e32130SMehdi Amini /// - One that has PGO data attached.
17501e32130SMehdi Amini /// - [insert you fancy metric here]
1762d28f7aaSMehdi Amini static const GlobalValueSummary *
177b4e1e829SMehdi Amini selectCallee(const ModuleSummaryIndex &Index,
1789667b91bSPeter Collingbourne              ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
179cb9a82fcSTeresa Johnson              unsigned Threshold, StringRef CallerModulePath,
180cb9a82fcSTeresa Johnson              FunctionImporter::ImportFailureReason &Reason,
181cb9a82fcSTeresa Johnson              GlobalValue::GUID GUID) {
182cb9a82fcSTeresa Johnson   Reason = FunctionImporter::ImportFailureReason::None;
18301e32130SMehdi Amini   auto It = llvm::find_if(
18428e457bcSTeresa Johnson       CalleeSummaryList,
18528e457bcSTeresa Johnson       [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
18628e457bcSTeresa Johnson         auto *GVSummary = SummaryPtr.get();
187cb9a82fcSTeresa Johnson         if (!Index.isGlobalValueLive(GVSummary)) {
188cb9a82fcSTeresa Johnson           Reason = FunctionImporter::ImportFailureReason::NotLive;
189eaf5172cSGeorge Rimar           return false;
190cb9a82fcSTeresa Johnson         }
191eaf5172cSGeorge Rimar 
19273305f82STeresa Johnson         // For SamplePGO, in computeImportForFunction the OriginalId
19373305f82STeresa Johnson         // may have been used to locate the callee summary list (See
19473305f82STeresa Johnson         // comment there).
19573305f82STeresa Johnson         // The mapping from OriginalId to GUID may return a GUID
19673305f82STeresa Johnson         // that corresponds to a static variable. Filter it out here.
19773305f82STeresa Johnson         // This can happen when
19873305f82STeresa Johnson         // 1) There is a call to a library function which is not defined
19973305f82STeresa Johnson         // in the index.
20073305f82STeresa Johnson         // 2) There is a static variable with the  OriginalGUID identical
20173305f82STeresa Johnson         // to the GUID of the library function in 1);
20273305f82STeresa Johnson         // When this happens, the logic for SamplePGO kicks in and
20373305f82STeresa Johnson         // the static variable in 2) will be found, which needs to be
20473305f82STeresa Johnson         // filtered out.
205cb9a82fcSTeresa Johnson         if (GVSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind) {
206cb9a82fcSTeresa Johnson           Reason = FunctionImporter::ImportFailureReason::GlobalVar;
20773305f82STeresa Johnson           return false;
208cb9a82fcSTeresa Johnson         }
209cb9a82fcSTeresa Johnson         if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) {
210cb9a82fcSTeresa Johnson           Reason = FunctionImporter::ImportFailureReason::InterposableLinkage;
2115b85d8d6SMehdi Amini           // There is no point in importing these, we can't inline them
21201e32130SMehdi Amini           return false;
213cb9a82fcSTeresa Johnson         }
2142c719cc1SMehdi Amini 
21581bbf742STeresa Johnson         auto *Summary = cast<FunctionSummary>(GVSummary->getBaseObject());
2167e88d0daSMehdi Amini 
21783aaf358STeresa Johnson         // If this is a local function, make sure we import the copy
21883aaf358STeresa Johnson         // in the caller's module. The only time a local function can
21983aaf358STeresa Johnson         // share an entry in the index is if there is a local with the same name
22083aaf358STeresa Johnson         // in another module that had the same source file name (in a different
22183aaf358STeresa Johnson         // directory), where each was compiled in their own directory so there
22283aaf358STeresa Johnson         // was not distinguishing path.
22383aaf358STeresa Johnson         // However, do the import from another module if there is only one
22483aaf358STeresa Johnson         // entry in the list - in that case this must be a reference due
22583aaf358STeresa Johnson         // to indirect call profile data, since a function pointer can point to
22683aaf358STeresa Johnson         // a local in another module.
22783aaf358STeresa Johnson         if (GlobalValue::isLocalLinkage(Summary->linkage()) &&
22883aaf358STeresa Johnson             CalleeSummaryList.size() > 1 &&
229cb9a82fcSTeresa Johnson             Summary->modulePath() != CallerModulePath) {
230cb9a82fcSTeresa Johnson           Reason =
231cb9a82fcSTeresa Johnson               FunctionImporter::ImportFailureReason::LocalLinkageNotInModule;
23283aaf358STeresa Johnson           return false;
233cb9a82fcSTeresa Johnson         }
23483aaf358STeresa Johnson 
235cb9a82fcSTeresa Johnson         if (Summary->instCount() > Threshold) {
236cb9a82fcSTeresa Johnson           Reason = FunctionImporter::ImportFailureReason::TooLarge;
237f9dc3deaSTeresa Johnson           return false;
238cb9a82fcSTeresa Johnson         }
239f9dc3deaSTeresa Johnson 
240*cb397461STeresa Johnson         // Skip if it isn't legal to import (e.g. may reference unpromotable
241*cb397461STeresa Johnson         // locals).
242cb9a82fcSTeresa Johnson         if (Summary->notEligibleToImport()) {
243cb9a82fcSTeresa Johnson           Reason = FunctionImporter::ImportFailureReason::NotEligible;
244b4e1e829SMehdi Amini           return false;
245cb9a82fcSTeresa Johnson         }
246b4e1e829SMehdi Amini 
247*cb397461STeresa Johnson         // Don't bother importing if we can't inline it anyway.
248*cb397461STeresa Johnson         if (Summary->fflags().NoInline) {
249*cb397461STeresa Johnson           Reason = FunctionImporter::ImportFailureReason::NoInline;
250*cb397461STeresa Johnson           return false;
251*cb397461STeresa Johnson         }
252*cb397461STeresa Johnson 
25301e32130SMehdi Amini         return true;
25401e32130SMehdi Amini       });
25528e457bcSTeresa Johnson   if (It == CalleeSummaryList.end())
25601e32130SMehdi Amini     return nullptr;
2577e88d0daSMehdi Amini 
258f9dc3deaSTeresa Johnson   return cast<GlobalValueSummary>(It->get());
259434e9561SRafael Espindola }
2607e88d0daSMehdi Amini 
261e9ea08a0SEugene Zelenko namespace {
262e9ea08a0SEugene Zelenko 
263475b51a7STeresa Johnson using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */,
264475b51a7STeresa Johnson                             GlobalValue::GUID>;
26501e32130SMehdi Amini 
266e9ea08a0SEugene Zelenko } // anonymous namespace
267e9ea08a0SEugene Zelenko 
2681958083dSTeresa Johnson static ValueInfo
2691958083dSTeresa Johnson updateValueInfoForIndirectCalls(const ModuleSummaryIndex &Index, ValueInfo VI) {
2701958083dSTeresa Johnson   if (!VI.getSummaryList().empty())
2711958083dSTeresa Johnson     return VI;
2721958083dSTeresa Johnson   // For SamplePGO, the indirect call targets for local functions will
2731958083dSTeresa Johnson   // have its original name annotated in profile. We try to find the
2741958083dSTeresa Johnson   // corresponding PGOFuncName as the GUID.
2751958083dSTeresa Johnson   // FIXME: Consider updating the edges in the graph after building
2761958083dSTeresa Johnson   // it, rather than needing to perform this mapping on each walk.
2771958083dSTeresa Johnson   auto GUID = Index.getGUIDFromOriginalID(VI.getGUID());
2781958083dSTeresa Johnson   if (GUID == 0)
27928d8a49fSEugene Leviant     return ValueInfo();
2801958083dSTeresa Johnson   return Index.getValueInfo(GUID);
2811958083dSTeresa Johnson }
2821958083dSTeresa Johnson 
28319e23874SEugene Leviant static void computeImportForReferencedGlobals(
28419e23874SEugene Leviant     const FunctionSummary &Summary, const GVSummaryMapTy &DefinedGVSummaries,
28519e23874SEugene Leviant     FunctionImporter::ImportMapTy &ImportList,
28619e23874SEugene Leviant     StringMap<FunctionImporter::ExportSetTy> *ExportLists) {
28719e23874SEugene Leviant   for (auto &VI : Summary.refs()) {
28819e23874SEugene Leviant     if (DefinedGVSummaries.count(VI.getGUID())) {
289d34e60caSNicola Zaghen       LLVM_DEBUG(
290d34e60caSNicola Zaghen           dbgs() << "Ref ignored! Target already in destination module.\n");
29119e23874SEugene Leviant       continue;
29219e23874SEugene Leviant     }
29319e23874SEugene Leviant 
2947e7b13d0STeresa Johnson     LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n");
29519e23874SEugene Leviant 
29619e23874SEugene Leviant     for (auto &RefSummary : VI.getSummaryList())
29719e23874SEugene Leviant       if (RefSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind &&
298eddf6b5dSEugene Leviant           !RefSummary->notEligibleToImport() &&
29919e23874SEugene Leviant           !GlobalValue::isInterposableLinkage(RefSummary->linkage()) &&
30019e23874SEugene Leviant           RefSummary->refs().empty()) {
301d2c234a4STeresa Johnson         auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID());
302d2c234a4STeresa Johnson         // Only update stat if we haven't already imported this variable.
303d2c234a4STeresa Johnson         if (ILI.second)
304d2c234a4STeresa Johnson           NumImportedGlobalVarsThinLink++;
30519e23874SEugene Leviant         if (ExportLists)
30619e23874SEugene Leviant           (*ExportLists)[RefSummary->modulePath()].insert(VI.getGUID());
30719e23874SEugene Leviant         break;
30819e23874SEugene Leviant       }
30919e23874SEugene Leviant   }
31019e23874SEugene Leviant }
31119e23874SEugene Leviant 
312cb9a82fcSTeresa Johnson static const char *
313cb9a82fcSTeresa Johnson getFailureName(FunctionImporter::ImportFailureReason Reason) {
314cb9a82fcSTeresa Johnson   switch (Reason) {
315cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::None:
316cb9a82fcSTeresa Johnson     return "None";
317cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::GlobalVar:
318cb9a82fcSTeresa Johnson     return "GlobalVar";
319cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::NotLive:
320cb9a82fcSTeresa Johnson     return "NotLive";
321cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::TooLarge:
322cb9a82fcSTeresa Johnson     return "TooLarge";
323cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::InterposableLinkage:
324cb9a82fcSTeresa Johnson     return "InterposableLinkage";
325cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
326cb9a82fcSTeresa Johnson     return "LocalLinkageNotInModule";
327cb9a82fcSTeresa Johnson   case FunctionImporter::ImportFailureReason::NotEligible:
328cb9a82fcSTeresa Johnson     return "NotEligible";
329*cb397461STeresa Johnson   case FunctionImporter::ImportFailureReason::NoInline:
330*cb397461STeresa Johnson     return "NoInline";
331cb9a82fcSTeresa Johnson   }
332cb9a82fcSTeresa Johnson   llvm_unreachable("invalid reason");
333cb9a82fcSTeresa Johnson }
334cb9a82fcSTeresa Johnson 
33501e32130SMehdi Amini /// Compute the list of functions to import for a given caller. Mark these
33601e32130SMehdi Amini /// imported functions and the symbols they reference in their source module as
33701e32130SMehdi Amini /// exported from their source module.
33801e32130SMehdi Amini static void computeImportForFunction(
3393255eec1STeresa Johnson     const FunctionSummary &Summary, const ModuleSummaryIndex &Index,
340d9830eb7SPiotr Padlewski     const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries,
34101e32130SMehdi Amini     SmallVectorImpl<EdgeInfo> &Worklist,
3429b490f10SMehdi Amini     FunctionImporter::ImportMapTy &ImportList,
343d68935c5STeresa Johnson     StringMap<FunctionImporter::ExportSetTy> *ExportLists,
344d68935c5STeresa Johnson     FunctionImporter::ImportThresholdsTy &ImportThresholds) {
34519e23874SEugene Leviant   computeImportForReferencedGlobals(Summary, DefinedGVSummaries, ImportList,
34619e23874SEugene Leviant                                     ExportLists);
347974706ebSTeresa Johnson   static int ImportCount = 0;
34801e32130SMehdi Amini   for (auto &Edge : Summary.calls()) {
3499667b91bSPeter Collingbourne     ValueInfo VI = Edge.first;
3507e7b13d0STeresa Johnson     LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold
3517e7b13d0STeresa Johnson                       << "\n");
35201e32130SMehdi Amini 
353974706ebSTeresa Johnson     if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) {
354d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff
355974706ebSTeresa Johnson                         << " reached.\n");
356974706ebSTeresa Johnson       continue;
357974706ebSTeresa Johnson     }
358974706ebSTeresa Johnson 
3591958083dSTeresa Johnson     VI = updateValueInfoForIndirectCalls(Index, VI);
3609667b91bSPeter Collingbourne     if (!VI)
3619667b91bSPeter Collingbourne       continue;
3624a435e08SDehao Chen 
3639667b91bSPeter Collingbourne     if (DefinedGVSummaries.count(VI.getGUID())) {
364d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");
3657e88d0daSMehdi Amini       continue;
366d450da32STeresa Johnson     }
36740641748SMehdi Amini 
368ba72b95fSPiotr Padlewski     auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float {
369ba72b95fSPiotr Padlewski       if (Hotness == CalleeInfo::HotnessType::Hot)
370ba72b95fSPiotr Padlewski         return ImportHotMultiplier;
371ba72b95fSPiotr Padlewski       if (Hotness == CalleeInfo::HotnessType::Cold)
372ba72b95fSPiotr Padlewski         return ImportColdMultiplier;
37364c46574SDehao Chen       if (Hotness == CalleeInfo::HotnessType::Critical)
37464c46574SDehao Chen         return ImportCriticalMultiplier;
375ba72b95fSPiotr Padlewski       return 1.0;
376ba72b95fSPiotr Padlewski     };
377ba72b95fSPiotr Padlewski 
378d9830eb7SPiotr Padlewski     const auto NewThreshold =
379c73cec84SEaswaran Raman         Threshold * GetBonusMultiplier(Edge.second.getHotness());
380d2869473SPiotr Padlewski 
381cb9a82fcSTeresa Johnson     auto IT = ImportThresholds.insert(std::make_pair(
382cb9a82fcSTeresa Johnson         VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr)));
383d68935c5STeresa Johnson     bool PreviouslyVisited = !IT.second;
384cb9a82fcSTeresa Johnson     auto &ProcessedThreshold = std::get<0>(IT.first->second);
385cb9a82fcSTeresa Johnson     auto &CalleeSummary = std::get<1>(IT.first->second);
386cb9a82fcSTeresa Johnson     auto &FailureInfo = std::get<2>(IT.first->second);
387d68935c5STeresa Johnson 
388d2c234a4STeresa Johnson     bool IsHotCallsite =
389d2c234a4STeresa Johnson         Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
390d2c234a4STeresa Johnson     bool IsCriticalCallsite =
391d2c234a4STeresa Johnson         Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
392d2c234a4STeresa Johnson 
393d68935c5STeresa Johnson     const FunctionSummary *ResolvedCalleeSummary = nullptr;
394d68935c5STeresa Johnson     if (CalleeSummary) {
395d68935c5STeresa Johnson       assert(PreviouslyVisited);
396d68935c5STeresa Johnson       // Since the traversal of the call graph is DFS, we can revisit a function
397d68935c5STeresa Johnson       // a second time with a higher threshold. In this case, it is added back
398d68935c5STeresa Johnson       // to the worklist with the new threshold (so that its own callee chains
399d68935c5STeresa Johnson       // can be considered with the higher threshold).
400d68935c5STeresa Johnson       if (NewThreshold <= ProcessedThreshold) {
401d68935c5STeresa Johnson         LLVM_DEBUG(
402d68935c5STeresa Johnson             dbgs() << "ignored! Target was already imported with Threshold "
403d68935c5STeresa Johnson                    << ProcessedThreshold << "\n");
404d68935c5STeresa Johnson         continue;
405d68935c5STeresa Johnson       }
406d68935c5STeresa Johnson       // Update with new larger threshold.
407d68935c5STeresa Johnson       ProcessedThreshold = NewThreshold;
408d68935c5STeresa Johnson       ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
409d68935c5STeresa Johnson     } else {
410d68935c5STeresa Johnson       // If we already rejected importing a callee at the same or higher
411d68935c5STeresa Johnson       // threshold, don't waste time calling selectCallee.
412d68935c5STeresa Johnson       if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
413d68935c5STeresa Johnson         LLVM_DEBUG(
414d68935c5STeresa Johnson             dbgs() << "ignored! Target was already rejected with Threshold "
415d68935c5STeresa Johnson             << ProcessedThreshold << "\n");
416cb9a82fcSTeresa Johnson         if (PrintImportFailures) {
417cb9a82fcSTeresa Johnson           assert(FailureInfo &&
418cb9a82fcSTeresa Johnson                  "Expected FailureInfo for previously rejected candidate");
419cb9a82fcSTeresa Johnson           FailureInfo->Attempts++;
420cb9a82fcSTeresa Johnson         }
421d68935c5STeresa Johnson         continue;
422d68935c5STeresa Johnson       }
423d68935c5STeresa Johnson 
424cb9a82fcSTeresa Johnson       FunctionImporter::ImportFailureReason Reason;
425d68935c5STeresa Johnson       CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold,
426cb9a82fcSTeresa Johnson                                    Summary.modulePath(), Reason, VI.getGUID());
42701e32130SMehdi Amini       if (!CalleeSummary) {
428d68935c5STeresa Johnson         // Update with new larger threshold if this was a retry (otherwise
429cb9a82fcSTeresa Johnson         // we would have already inserted with NewThreshold above). Also
430cb9a82fcSTeresa Johnson         // update failure info if requested.
431cb9a82fcSTeresa Johnson         if (PreviouslyVisited) {
432d68935c5STeresa Johnson           ProcessedThreshold = NewThreshold;
433cb9a82fcSTeresa Johnson           if (PrintImportFailures) {
434cb9a82fcSTeresa Johnson             assert(FailureInfo &&
435cb9a82fcSTeresa Johnson                    "Expected FailureInfo for previously rejected candidate");
436cb9a82fcSTeresa Johnson             FailureInfo->Reason = Reason;
437cb9a82fcSTeresa Johnson             FailureInfo->Attempts++;
438cb9a82fcSTeresa Johnson             FailureInfo->MaxHotness =
439cb9a82fcSTeresa Johnson                 std::max(FailureInfo->MaxHotness, Edge.second.getHotness());
440cb9a82fcSTeresa Johnson           }
441cb9a82fcSTeresa Johnson         } else if (PrintImportFailures) {
442cb9a82fcSTeresa Johnson           assert(!FailureInfo &&
443cb9a82fcSTeresa Johnson                  "Expected no FailureInfo for newly rejected candidate");
444cb9a82fcSTeresa Johnson           FailureInfo = llvm::make_unique<FunctionImporter::ImportFailureInfo>(
445cb9a82fcSTeresa Johnson               VI, Edge.second.getHotness(), Reason, 1);
446cb9a82fcSTeresa Johnson         }
447d34e60caSNicola Zaghen         LLVM_DEBUG(
448d34e60caSNicola Zaghen             dbgs() << "ignored! No qualifying callee with summary found.\n");
4497e88d0daSMehdi Amini         continue;
4507e88d0daSMehdi Amini       }
4512f0cc477SDavid Blaikie 
4522f0cc477SDavid Blaikie       // "Resolve" the summary
453d68935c5STeresa Johnson       CalleeSummary = CalleeSummary->getBaseObject();
454d68935c5STeresa Johnson       ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
4552d28f7aaSMehdi Amini 
456d9830eb7SPiotr Padlewski       assert(ResolvedCalleeSummary->instCount() <= NewThreshold &&
45701e32130SMehdi Amini              "selectCallee() didn't honor the threshold");
45801e32130SMehdi Amini 
4591b859a23STeresa Johnson       auto ExportModulePath = ResolvedCalleeSummary->modulePath();
460d68935c5STeresa Johnson       auto ILI = ImportList[ExportModulePath].insert(VI.getGUID());
461d68935c5STeresa Johnson       // We previously decided to import this GUID definition if it was already
462d68935c5STeresa Johnson       // inserted in the set of imports from the exporting module.
463d68935c5STeresa Johnson       bool PreviouslyImported = !ILI.second;
464d2c234a4STeresa Johnson       if (!PreviouslyImported) {
465d2c234a4STeresa Johnson         NumImportedFunctionsThinLink++;
466d2c234a4STeresa Johnson         if (IsHotCallsite)
467d2c234a4STeresa Johnson           NumImportedHotFunctionsThinLink++;
468d2c234a4STeresa Johnson         if (IsCriticalCallsite)
469d2c234a4STeresa Johnson           NumImportedCriticalFunctionsThinLink++;
470d2c234a4STeresa Johnson       }
471974706ebSTeresa Johnson 
4721b859a23STeresa Johnson       // Make exports in the source module.
4731b859a23STeresa Johnson       if (ExportLists) {
4741b859a23STeresa Johnson         auto &ExportList = (*ExportLists)[ExportModulePath];
4759667b91bSPeter Collingbourne         ExportList.insert(VI.getGUID());
47619f2aa78STeresa Johnson         if (!PreviouslyImported) {
47719f2aa78STeresa Johnson           // This is the first time this function was exported from its source
47819f2aa78STeresa Johnson           // module, so mark all functions and globals it references as exported
4791b859a23STeresa Johnson           // to the outside if they are defined in the same source module.
480edddca22STeresa Johnson           // For efficiency, we unconditionally add all the referenced GUIDs
481edddca22STeresa Johnson           // to the ExportList for this module, and will prune out any not
482edddca22STeresa Johnson           // defined in the module later in a single pass.
4831b859a23STeresa Johnson           for (auto &Edge : ResolvedCalleeSummary->calls()) {
4841b859a23STeresa Johnson             auto CalleeGUID = Edge.first.getGUID();
485edddca22STeresa Johnson             ExportList.insert(CalleeGUID);
4861b859a23STeresa Johnson           }
4871b859a23STeresa Johnson           for (auto &Ref : ResolvedCalleeSummary->refs()) {
4881b859a23STeresa Johnson             auto GUID = Ref.getGUID();
489edddca22STeresa Johnson             ExportList.insert(GUID);
4901b859a23STeresa Johnson           }
4911b859a23STeresa Johnson         }
49219f2aa78STeresa Johnson       }
493d68935c5STeresa Johnson     }
494d68935c5STeresa Johnson 
495d68935c5STeresa Johnson     auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {
496d68935c5STeresa Johnson       // Adjust the threshold for next level of imported functions.
497d68935c5STeresa Johnson       // The threshold is different for hot callsites because we can then
498d68935c5STeresa Johnson       // inline chains of hot calls.
499d68935c5STeresa Johnson       if (IsHotCallsite)
500d68935c5STeresa Johnson         return Threshold * ImportHotInstrFactor;
501d68935c5STeresa Johnson       return Threshold * ImportInstrFactor;
502d68935c5STeresa Johnson     };
503d68935c5STeresa Johnson 
504d68935c5STeresa Johnson     const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
505d68935c5STeresa Johnson 
506d68935c5STeresa Johnson     ImportCount++;
507d2869473SPiotr Padlewski 
50801e32130SMehdi Amini     // Insert the newly imported function to the worklist.
5099667b91bSPeter Collingbourne     Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID());
510d450da32STeresa Johnson   }
511d450da32STeresa Johnson }
512d450da32STeresa Johnson 
51301e32130SMehdi Amini /// Given the list of globals defined in a module, compute the list of imports
51401e32130SMehdi Amini /// as well as the list of "exports", i.e. the list of symbols referenced from
51501e32130SMehdi Amini /// another module (that may require promotion).
51601e32130SMehdi Amini static void ComputeImportForModule(
517c851d216STeresa Johnson     const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index,
518cb9a82fcSTeresa Johnson     StringRef ModName, FunctionImporter::ImportMapTy &ImportList,
51956584bbfSEvgeniy Stepanov     StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) {
52001e32130SMehdi Amini   // Worklist contains the list of function imported in this module, for which
52101e32130SMehdi Amini   // we will analyse the callees and may import further down the callgraph.
52201e32130SMehdi Amini   SmallVector<EdgeInfo, 128> Worklist;
523d68935c5STeresa Johnson   FunctionImporter::ImportThresholdsTy ImportThresholds;
52401e32130SMehdi Amini 
52501e32130SMehdi Amini   // Populate the worklist with the import for the functions in the current
52601e32130SMehdi Amini   // module
52728e457bcSTeresa Johnson   for (auto &GVSummary : DefinedGVSummaries) {
5287e7b13d0STeresa Johnson #ifndef NDEBUG
5297e7b13d0STeresa Johnson     // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID
5307e7b13d0STeresa Johnson     // so this map look up (and possibly others) can be avoided.
5317e7b13d0STeresa Johnson     auto VI = Index.getValueInfo(GVSummary.first);
5327e7b13d0STeresa Johnson #endif
53356584bbfSEvgeniy Stepanov     if (!Index.isGlobalValueLive(GVSummary.second)) {
5347e7b13d0STeresa Johnson       LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");
5356c475a75STeresa Johnson       continue;
5366c475a75STeresa Johnson     }
537cfbd0892SPeter Collingbourne     auto *FuncSummary =
538cfbd0892SPeter Collingbourne         dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject());
5391aafabf7SMehdi Amini     if (!FuncSummary)
5401aafabf7SMehdi Amini       // Skip import for global variables
5411aafabf7SMehdi Amini       continue;
5427e7b13d0STeresa Johnson     LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");
5432d28f7aaSMehdi Amini     computeImportForFunction(*FuncSummary, Index, ImportInstrLimit,
5449b490f10SMehdi Amini                              DefinedGVSummaries, Worklist, ImportList,
545d68935c5STeresa Johnson                              ExportLists, ImportThresholds);
54601e32130SMehdi Amini   }
54701e32130SMehdi Amini 
548d2869473SPiotr Padlewski   // Process the newly imported functions and add callees to the worklist.
54942418abaSMehdi Amini   while (!Worklist.empty()) {
55001e32130SMehdi Amini     auto FuncInfo = Worklist.pop_back_val();
551475b51a7STeresa Johnson     auto *Summary = std::get<0>(FuncInfo);
552475b51a7STeresa Johnson     auto Threshold = std::get<1>(FuncInfo);
55342418abaSMehdi Amini 
5541aafabf7SMehdi Amini     computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries,
555d68935c5STeresa Johnson                              Worklist, ImportList, ExportLists,
556d68935c5STeresa Johnson                              ImportThresholds);
557c8c55170SMehdi Amini   }
558cb9a82fcSTeresa Johnson 
559cb9a82fcSTeresa Johnson   // Print stats about functions considered but rejected for importing
560cb9a82fcSTeresa Johnson   // when requested.
561cb9a82fcSTeresa Johnson   if (PrintImportFailures) {
562cb9a82fcSTeresa Johnson     dbgs() << "Missed imports into module " << ModName << "\n";
563cb9a82fcSTeresa Johnson     for (auto &I : ImportThresholds) {
564cb9a82fcSTeresa Johnson       auto &ProcessedThreshold = std::get<0>(I.second);
565cb9a82fcSTeresa Johnson       auto &CalleeSummary = std::get<1>(I.second);
566cb9a82fcSTeresa Johnson       auto &FailureInfo = std::get<2>(I.second);
567cb9a82fcSTeresa Johnson       if (CalleeSummary)
568cb9a82fcSTeresa Johnson         continue; // We are going to import.
569cb9a82fcSTeresa Johnson       assert(FailureInfo);
570cb9a82fcSTeresa Johnson       FunctionSummary *FS = nullptr;
571cb9a82fcSTeresa Johnson       if (!FailureInfo->VI.getSummaryList().empty())
572cb9a82fcSTeresa Johnson         FS = dyn_cast<FunctionSummary>(
573cb9a82fcSTeresa Johnson             FailureInfo->VI.getSummaryList()[0]->getBaseObject());
574cb9a82fcSTeresa Johnson       dbgs() << FailureInfo->VI
575cb9a82fcSTeresa Johnson              << ": Reason = " << getFailureName(FailureInfo->Reason)
576cb9a82fcSTeresa Johnson              << ", Threshold = " << ProcessedThreshold
577cb9a82fcSTeresa Johnson              << ", Size = " << (FS ? (int)FS->instCount() : -1)
578cb9a82fcSTeresa Johnson              << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness)
579cb9a82fcSTeresa Johnson              << ", Attempts = " << FailureInfo->Attempts << "\n";
580cb9a82fcSTeresa Johnson     }
581cb9a82fcSTeresa Johnson   }
58242418abaSMehdi Amini }
583ffe2e4aaSMehdi Amini 
58419e23874SEugene Leviant #ifndef NDEBUG
58519e23874SEugene Leviant static bool isGlobalVarSummary(const ModuleSummaryIndex &Index,
58619e23874SEugene Leviant                                GlobalValue::GUID G) {
58719e23874SEugene Leviant   if (const auto &VI = Index.getValueInfo(G)) {
58819e23874SEugene Leviant     auto SL = VI.getSummaryList();
58919e23874SEugene Leviant     if (!SL.empty())
59019e23874SEugene Leviant       return SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind;
59119e23874SEugene Leviant   }
59219e23874SEugene Leviant   return false;
59319e23874SEugene Leviant }
59419e23874SEugene Leviant 
59519e23874SEugene Leviant static GlobalValue::GUID getGUID(GlobalValue::GUID G) { return G; }
59619e23874SEugene Leviant 
59719e23874SEugene Leviant template <class T>
5981fc0da48SBenjamin Kramer static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index,
5991fc0da48SBenjamin Kramer                                       T &Cont) {
60019e23874SEugene Leviant   unsigned NumGVS = 0;
60119e23874SEugene Leviant   for (auto &V : Cont)
60219e23874SEugene Leviant     if (isGlobalVarSummary(Index, getGUID(V)))
60319e23874SEugene Leviant       ++NumGVS;
60419e23874SEugene Leviant   return NumGVS;
60519e23874SEugene Leviant }
60619e23874SEugene Leviant #endif
60719e23874SEugene Leviant 
608c86af334STeresa Johnson /// Compute all the import and export for every module using the Index.
60901e32130SMehdi Amini void llvm::ComputeCrossModuleImport(
61001e32130SMehdi Amini     const ModuleSummaryIndex &Index,
611c851d216STeresa Johnson     const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
61201e32130SMehdi Amini     StringMap<FunctionImporter::ImportMapTy> &ImportLists,
61356584bbfSEvgeniy Stepanov     StringMap<FunctionImporter::ExportSetTy> &ExportLists) {
61401e32130SMehdi Amini   // For each module that has function defined, compute the import/export lists.
6151aafabf7SMehdi Amini   for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
6169b490f10SMehdi Amini     auto &ImportList = ImportLists[DefinedGVSummaries.first()];
617d34e60caSNicola Zaghen     LLVM_DEBUG(dbgs() << "Computing import for Module '"
6181aafabf7SMehdi Amini                       << DefinedGVSummaries.first() << "'\n");
619cb9a82fcSTeresa Johnson     ComputeImportForModule(DefinedGVSummaries.second, Index,
620cb9a82fcSTeresa Johnson                            DefinedGVSummaries.first(), ImportList,
62156584bbfSEvgeniy Stepanov                            &ExportLists);
62201e32130SMehdi Amini   }
62301e32130SMehdi Amini 
624edddca22STeresa Johnson   // When computing imports we added all GUIDs referenced by anything
625edddca22STeresa Johnson   // imported from the module to its ExportList. Now we prune each ExportList
626edddca22STeresa Johnson   // of any not defined in that module. This is more efficient than checking
627edddca22STeresa Johnson   // while computing imports because some of the summary lists may be long
628edddca22STeresa Johnson   // due to linkonce (comdat) copies.
629edddca22STeresa Johnson   for (auto &ELI : ExportLists) {
630edddca22STeresa Johnson     const auto &DefinedGVSummaries =
631edddca22STeresa Johnson         ModuleToDefinedGVSummaries.lookup(ELI.first());
632edddca22STeresa Johnson     for (auto EI = ELI.second.begin(); EI != ELI.second.end();) {
633edddca22STeresa Johnson       if (!DefinedGVSummaries.count(*EI))
634edddca22STeresa Johnson         EI = ELI.second.erase(EI);
635edddca22STeresa Johnson       else
636edddca22STeresa Johnson         ++EI;
637edddca22STeresa Johnson     }
638edddca22STeresa Johnson   }
639edddca22STeresa Johnson 
64001e32130SMehdi Amini #ifndef NDEBUG
641d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()
64201e32130SMehdi Amini                     << " modules:\n");
64301e32130SMehdi Amini   for (auto &ModuleImports : ImportLists) {
64401e32130SMehdi Amini     auto ModName = ModuleImports.first();
64501e32130SMehdi Amini     auto &Exports = ExportLists[ModName];
64619e23874SEugene Leviant     unsigned NumGVS = numGlobalVarSummaries(Index, Exports);
647d34e60caSNicola Zaghen     LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "
64819e23874SEugene Leviant                       << Exports.size() - NumGVS << " functions and " << NumGVS
649d34e60caSNicola Zaghen                       << " vars. Imports from " << ModuleImports.second.size()
650d34e60caSNicola Zaghen                       << " modules.\n");
65101e32130SMehdi Amini     for (auto &Src : ModuleImports.second) {
65201e32130SMehdi Amini       auto SrcModName = Src.first();
65319e23874SEugene Leviant       unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second);
654d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod
65519e23874SEugene Leviant                         << " functions imported from " << SrcModName << "\n");
656d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod
657d34e60caSNicola Zaghen                         << " global vars imported from " << SrcModName << "\n");
65801e32130SMehdi Amini     }
65901e32130SMehdi Amini   }
66001e32130SMehdi Amini #endif
66101e32130SMehdi Amini }
66201e32130SMehdi Amini 
66381bbf742STeresa Johnson #ifndef NDEBUG
66419e23874SEugene Leviant static void dumpImportListForModule(const ModuleSummaryIndex &Index,
66519e23874SEugene Leviant                                     StringRef ModulePath,
66681bbf742STeresa Johnson                                     FunctionImporter::ImportMapTy &ImportList) {
667d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
66881bbf742STeresa Johnson                     << ImportList.size() << " modules.\n");
66981bbf742STeresa Johnson   for (auto &Src : ImportList) {
67081bbf742STeresa Johnson     auto SrcModName = Src.first();
67119e23874SEugene Leviant     unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second);
672d34e60caSNicola Zaghen     LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod
67319e23874SEugene Leviant                       << " functions imported from " << SrcModName << "\n");
674d34e60caSNicola Zaghen     LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod << " vars imported from "
67581bbf742STeresa Johnson                       << SrcModName << "\n");
67681bbf742STeresa Johnson   }
67781bbf742STeresa Johnson }
67869b2de84STeresa Johnson #endif
67981bbf742STeresa Johnson 
680c86af334STeresa Johnson /// Compute all the imports for the given module in the Index.
681c86af334STeresa Johnson void llvm::ComputeCrossModuleImportForModule(
682c86af334STeresa Johnson     StringRef ModulePath, const ModuleSummaryIndex &Index,
683c86af334STeresa Johnson     FunctionImporter::ImportMapTy &ImportList) {
684c86af334STeresa Johnson   // Collect the list of functions this module defines.
685c86af334STeresa Johnson   // GUID -> Summary
686c851d216STeresa Johnson   GVSummaryMapTy FunctionSummaryMap;
68728e457bcSTeresa Johnson   Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);
688c86af334STeresa Johnson 
689c86af334STeresa Johnson   // Compute the import list for this module.
690d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
691cb9a82fcSTeresa Johnson   ComputeImportForModule(FunctionSummaryMap, Index, ModulePath, ImportList);
692c86af334STeresa Johnson 
693c86af334STeresa Johnson #ifndef NDEBUG
69419e23874SEugene Leviant   dumpImportListForModule(Index, ModulePath, ImportList);
69581bbf742STeresa Johnson #endif
696c86af334STeresa Johnson }
69781bbf742STeresa Johnson 
69881bbf742STeresa Johnson // Mark all external summaries in Index for import into the given module.
69981bbf742STeresa Johnson // Used for distributed builds using a distributed index.
70081bbf742STeresa Johnson void llvm::ComputeCrossModuleImportForModuleFromIndex(
70181bbf742STeresa Johnson     StringRef ModulePath, const ModuleSummaryIndex &Index,
70281bbf742STeresa Johnson     FunctionImporter::ImportMapTy &ImportList) {
70381bbf742STeresa Johnson   for (auto &GlobalList : Index) {
70481bbf742STeresa Johnson     // Ignore entries for undefined references.
70581bbf742STeresa Johnson     if (GlobalList.second.SummaryList.empty())
70681bbf742STeresa Johnson       continue;
70781bbf742STeresa Johnson 
70881bbf742STeresa Johnson     auto GUID = GlobalList.first;
70981bbf742STeresa Johnson     assert(GlobalList.second.SummaryList.size() == 1 &&
71081bbf742STeresa Johnson            "Expected individual combined index to have one summary per GUID");
71181bbf742STeresa Johnson     auto &Summary = GlobalList.second.SummaryList[0];
71281bbf742STeresa Johnson     // Skip the summaries for the importing module. These are included to
71381bbf742STeresa Johnson     // e.g. record required linkage changes.
71481bbf742STeresa Johnson     if (Summary->modulePath() == ModulePath)
71581bbf742STeresa Johnson       continue;
716d68935c5STeresa Johnson     // Add an entry to provoke importing by thinBackend.
717d68935c5STeresa Johnson     ImportList[Summary->modulePath()].insert(GUID);
71881bbf742STeresa Johnson   }
71981bbf742STeresa Johnson #ifndef NDEBUG
72019e23874SEugene Leviant   dumpImportListForModule(Index, ModulePath, ImportList);
721c86af334STeresa Johnson #endif
722c86af334STeresa Johnson }
723c86af334STeresa Johnson 
72456584bbfSEvgeniy Stepanov void llvm::computeDeadSymbols(
72556584bbfSEvgeniy Stepanov     ModuleSummaryIndex &Index,
726eaf5172cSGeorge Rimar     const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
727eaf5172cSGeorge Rimar     function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) {
72856584bbfSEvgeniy Stepanov   assert(!Index.withGlobalValueDeadStripping());
7296c475a75STeresa Johnson   if (!ComputeDead)
73056584bbfSEvgeniy Stepanov     return;
7316c475a75STeresa Johnson   if (GUIDPreservedSymbols.empty())
7326c475a75STeresa Johnson     // Don't do anything when nothing is live, this is friendly with tests.
73356584bbfSEvgeniy Stepanov     return;
73456584bbfSEvgeniy Stepanov   unsigned LiveSymbols = 0;
7359667b91bSPeter Collingbourne   SmallVector<ValueInfo, 128> Worklist;
7369667b91bSPeter Collingbourne   Worklist.reserve(GUIDPreservedSymbols.size() * 2);
7379667b91bSPeter Collingbourne   for (auto GUID : GUIDPreservedSymbols) {
7389667b91bSPeter Collingbourne     ValueInfo VI = Index.getValueInfo(GUID);
7399667b91bSPeter Collingbourne     if (!VI)
7409667b91bSPeter Collingbourne       continue;
74156584bbfSEvgeniy Stepanov     for (auto &S : VI.getSummaryList())
74256584bbfSEvgeniy Stepanov       S->setLive(true);
7436c475a75STeresa Johnson   }
74456584bbfSEvgeniy Stepanov 
7456c475a75STeresa Johnson   // Add values flagged in the index as live roots to the worklist.
7467e7b13d0STeresa Johnson   for (const auto &Entry : Index) {
7477e7b13d0STeresa Johnson     auto VI = Index.getValueInfo(Entry);
74856584bbfSEvgeniy Stepanov     for (auto &S : Entry.second.SummaryList)
74956584bbfSEvgeniy Stepanov       if (S->isLive()) {
7507e7b13d0STeresa Johnson         LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n");
7517e7b13d0STeresa Johnson         Worklist.push_back(VI);
75256584bbfSEvgeniy Stepanov         ++LiveSymbols;
75356584bbfSEvgeniy Stepanov         break;
7546c475a75STeresa Johnson       }
7557e7b13d0STeresa Johnson   }
7566c475a75STeresa Johnson 
75756584bbfSEvgeniy Stepanov   // Make value live and add it to the worklist if it was not live before.
75856584bbfSEvgeniy Stepanov   auto visit = [&](ValueInfo VI) {
7591958083dSTeresa Johnson     // FIXME: If we knew which edges were created for indirect call profiles,
7601958083dSTeresa Johnson     // we could skip them here. Any that are live should be reached via
7611958083dSTeresa Johnson     // other edges, e.g. reference edges. Otherwise, using a profile collected
7621958083dSTeresa Johnson     // on a slightly different binary might provoke preserving, importing
7631958083dSTeresa Johnson     // and ultimately promoting calls to functions not linked into this
7641958083dSTeresa Johnson     // binary, which increases the binary size unnecessarily. Note that
7651958083dSTeresa Johnson     // if this code changes, the importer needs to change so that edges
7661958083dSTeresa Johnson     // to functions marked dead are skipped.
7671958083dSTeresa Johnson     VI = updateValueInfoForIndirectCalls(Index, VI);
7681958083dSTeresa Johnson     if (!VI)
7691958083dSTeresa Johnson       return;
77056584bbfSEvgeniy Stepanov     for (auto &S : VI.getSummaryList())
771f625118eSTeresa Johnson       if (S->isLive())
772f625118eSTeresa Johnson         return;
773eaf5172cSGeorge Rimar 
774aab60006SVlad Tsyrklevich     // We only keep live symbols that are known to be non-prevailing if any are
775bfdad33bSXin Tong     // available_externally, linkonceodr, weakodr. Those symbols are discarded
776bfdad33bSXin Tong     // later in the EliminateAvailableExternally pass and setting them to
777bfdad33bSXin Tong     // not-live could break downstreams users of liveness information (PR36483)
778bfdad33bSXin Tong     // or limit optimization opportunities.
779aab60006SVlad Tsyrklevich     if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
780bfdad33bSXin Tong       bool KeepAliveLinkage = false;
781aab60006SVlad Tsyrklevich       bool Interposable = false;
782aab60006SVlad Tsyrklevich       for (auto &S : VI.getSummaryList()) {
783bfdad33bSXin Tong         if (S->linkage() == GlobalValue::AvailableExternallyLinkage ||
784bfdad33bSXin Tong             S->linkage() == GlobalValue::WeakODRLinkage ||
785bfdad33bSXin Tong             S->linkage() == GlobalValue::LinkOnceODRLinkage)
786bfdad33bSXin Tong           KeepAliveLinkage = true;
787aab60006SVlad Tsyrklevich         else if (GlobalValue::isInterposableLinkage(S->linkage()))
788aab60006SVlad Tsyrklevich           Interposable = true;
789aab60006SVlad Tsyrklevich       }
790aab60006SVlad Tsyrklevich 
791bfdad33bSXin Tong       if (!KeepAliveLinkage)
792eaf5172cSGeorge Rimar         return;
793eaf5172cSGeorge Rimar 
794aab60006SVlad Tsyrklevich       if (Interposable)
795bfdad33bSXin Tong         report_fatal_error(
796bfdad33bSXin Tong           "Interposable and available_externally/linkonce_odr/weak_odr symbol");
797aab60006SVlad Tsyrklevich     }
798aab60006SVlad Tsyrklevich 
799f625118eSTeresa Johnson     for (auto &S : VI.getSummaryList())
80056584bbfSEvgeniy Stepanov       S->setLive(true);
80156584bbfSEvgeniy Stepanov     ++LiveSymbols;
80256584bbfSEvgeniy Stepanov     Worklist.push_back(VI);
80356584bbfSEvgeniy Stepanov   };
80456584bbfSEvgeniy Stepanov 
8056c475a75STeresa Johnson   while (!Worklist.empty()) {
8069667b91bSPeter Collingbourne     auto VI = Worklist.pop_back_val();
8079667b91bSPeter Collingbourne     for (auto &Summary : VI.getSummaryList()) {
808cfbd0892SPeter Collingbourne       GlobalValueSummary *Base = Summary->getBaseObject();
809eaf5172cSGeorge Rimar       // Set base value live in case it is an alias.
810eaf5172cSGeorge Rimar       Base->setLive(true);
811cfbd0892SPeter Collingbourne       for (auto Ref : Base->refs())
81256584bbfSEvgeniy Stepanov         visit(Ref);
813cfbd0892SPeter Collingbourne       if (auto *FS = dyn_cast<FunctionSummary>(Base))
81456584bbfSEvgeniy Stepanov         for (auto Call : FS->calls())
81556584bbfSEvgeniy Stepanov           visit(Call.first);
8166c475a75STeresa Johnson     }
8176c475a75STeresa Johnson   }
81856584bbfSEvgeniy Stepanov   Index.setWithGlobalValueDeadStripping();
81956584bbfSEvgeniy Stepanov 
82056584bbfSEvgeniy Stepanov   unsigned DeadSymbols = Index.size() - LiveSymbols;
821d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols
82256584bbfSEvgeniy Stepanov                     << " symbols Dead \n");
82356584bbfSEvgeniy Stepanov   NumDeadSymbols += DeadSymbols;
82456584bbfSEvgeniy Stepanov   NumLiveSymbols += LiveSymbols;
8256c475a75STeresa Johnson }
8266c475a75STeresa Johnson 
82784174c37STeresa Johnson /// Compute the set of summaries needed for a ThinLTO backend compilation of
82884174c37STeresa Johnson /// \p ModulePath.
82984174c37STeresa Johnson void llvm::gatherImportedSummariesForModule(
83084174c37STeresa Johnson     StringRef ModulePath,
83184174c37STeresa Johnson     const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
832cdbcbf74SMehdi Amini     const FunctionImporter::ImportMapTy &ImportList,
83384174c37STeresa Johnson     std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
83484174c37STeresa Johnson   // Include all summaries from the importing module.
83584174c37STeresa Johnson   ModuleToSummariesForIndex[ModulePath] =
83684174c37STeresa Johnson       ModuleToDefinedGVSummaries.lookup(ModulePath);
83784174c37STeresa Johnson   // Include summaries for imports.
83888c491ddSMehdi Amini   for (auto &ILI : ImportList) {
83984174c37STeresa Johnson     auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()];
84084174c37STeresa Johnson     const auto &DefinedGVSummaries =
84184174c37STeresa Johnson         ModuleToDefinedGVSummaries.lookup(ILI.first());
84284174c37STeresa Johnson     for (auto &GI : ILI.second) {
843d68935c5STeresa Johnson       const auto &DS = DefinedGVSummaries.find(GI);
84484174c37STeresa Johnson       assert(DS != DefinedGVSummaries.end() &&
84584174c37STeresa Johnson              "Expected a defined summary for imported global value");
846d68935c5STeresa Johnson       SummariesForIndex[GI] = DS->second;
84784174c37STeresa Johnson     }
84884174c37STeresa Johnson   }
84984174c37STeresa Johnson }
85084174c37STeresa Johnson 
8518570fe47STeresa Johnson /// Emit the files \p ModulePath will import from into \p OutputFilename.
852c0320ef4STeresa Johnson std::error_code llvm::EmitImportsFiles(
853c0320ef4STeresa Johnson     StringRef ModulePath, StringRef OutputFilename,
854c0320ef4STeresa Johnson     const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
8558570fe47STeresa Johnson   std::error_code EC;
8568570fe47STeresa Johnson   raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None);
8578570fe47STeresa Johnson   if (EC)
8588570fe47STeresa Johnson     return EC;
859c0320ef4STeresa Johnson   for (auto &ILI : ModuleToSummariesForIndex)
860c0320ef4STeresa Johnson     // The ModuleToSummariesForIndex map includes an entry for the current
861c0320ef4STeresa Johnson     // Module (needed for writing out the index files). We don't want to
862c0320ef4STeresa Johnson     // include it in the imports file, however, so filter it out.
863c0320ef4STeresa Johnson     if (ILI.first != ModulePath)
864c0320ef4STeresa Johnson       ImportsOS << ILI.first << "\n";
8658570fe47STeresa Johnson   return std::error_code();
8668570fe47STeresa Johnson }
8678570fe47STeresa Johnson 
8685a95c477STeresa Johnson bool llvm::convertToDeclaration(GlobalValue &GV) {
869d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName()
870d34e60caSNicola Zaghen                     << "\n");
8714566c6dbSTeresa Johnson   if (Function *F = dyn_cast<Function>(&GV)) {
8724566c6dbSTeresa Johnson     F->deleteBody();
8734566c6dbSTeresa Johnson     F->clearMetadata();
8747873669bSPeter Collingbourne     F->setComdat(nullptr);
8754566c6dbSTeresa Johnson   } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(&GV)) {
8764566c6dbSTeresa Johnson     V->setInitializer(nullptr);
8774566c6dbSTeresa Johnson     V->setLinkage(GlobalValue::ExternalLinkage);
8784566c6dbSTeresa Johnson     V->clearMetadata();
8797873669bSPeter Collingbourne     V->setComdat(nullptr);
8805a95c477STeresa Johnson   } else {
8815a95c477STeresa Johnson     GlobalValue *NewGV;
8825a95c477STeresa Johnson     if (GV.getValueType()->isFunctionTy())
8835a95c477STeresa Johnson       NewGV =
8845a95c477STeresa Johnson           Function::Create(cast<FunctionType>(GV.getValueType()),
8855a95c477STeresa Johnson                            GlobalValue::ExternalLinkage, "", GV.getParent());
8865a95c477STeresa Johnson     else
8875a95c477STeresa Johnson       NewGV =
8885a95c477STeresa Johnson           new GlobalVariable(*GV.getParent(), GV.getValueType(),
8895a95c477STeresa Johnson                              /*isConstant*/ false, GlobalValue::ExternalLinkage,
8905a95c477STeresa Johnson                              /*init*/ nullptr, "",
8915a95c477STeresa Johnson                              /*insertbefore*/ nullptr, GV.getThreadLocalMode(),
8925a95c477STeresa Johnson                              GV.getType()->getAddressSpace());
8935a95c477STeresa Johnson     NewGV->takeName(&GV);
8945a95c477STeresa Johnson     GV.replaceAllUsesWith(NewGV);
8955a95c477STeresa Johnson     return false;
8965a95c477STeresa Johnson   }
8975a95c477STeresa Johnson   return true;
898eaf5172cSGeorge Rimar }
8994566c6dbSTeresa Johnson 
900eaf5172cSGeorge Rimar /// Fixup WeakForLinker linkages in \p TheModule based on summary analysis.
901eaf5172cSGeorge Rimar void llvm::thinLTOResolveWeakForLinkerModule(
902eaf5172cSGeorge Rimar     Module &TheModule, const GVSummaryMapTy &DefinedGlobals) {
90304c9a2d6STeresa Johnson   auto updateLinkage = [&](GlobalValue &GV) {
90404c9a2d6STeresa Johnson     // See if the global summary analysis computed a new resolved linkage.
90504c9a2d6STeresa Johnson     const auto &GS = DefinedGlobals.find(GV.getGUID());
90604c9a2d6STeresa Johnson     if (GS == DefinedGlobals.end())
90704c9a2d6STeresa Johnson       return;
90804c9a2d6STeresa Johnson     auto NewLinkage = GS->second->linkage();
90904c9a2d6STeresa Johnson     if (NewLinkage == GV.getLinkage())
91004c9a2d6STeresa Johnson       return;
9116a5fbe52SDavide Italiano 
9126a5fbe52SDavide Italiano     // Switch the linkage to weakany if asked for, e.g. we do this for
9136a5fbe52SDavide Italiano     // linker redefined symbols (via --wrap or --defsym).
914f4891d29SDavide Italiano     // We record that the visibility should be changed here in `addThinLTO`
915f4891d29SDavide Italiano     // as we need access to the resolution vectors for each input file in
916f4891d29SDavide Italiano     // order to find which symbols have been redefined.
917f4891d29SDavide Italiano     // We may consider reorganizing this code and moving the linkage recording
918f4891d29SDavide Italiano     // somewhere else, e.g. in thinLTOResolveWeakForLinkerInIndex.
9196a5fbe52SDavide Italiano     if (NewLinkage == GlobalValue::WeakAnyLinkage) {
9206a5fbe52SDavide Italiano       GV.setLinkage(NewLinkage);
9216a5fbe52SDavide Italiano       return;
9226a5fbe52SDavide Italiano     }
9236a5fbe52SDavide Italiano 
9246a5fbe52SDavide Italiano     if (!GlobalValue::isWeakForLinker(GV.getLinkage()))
9256a5fbe52SDavide Italiano       return;
9264566c6dbSTeresa Johnson     // Check for a non-prevailing def that has interposable linkage
9274566c6dbSTeresa Johnson     // (e.g. non-odr weak or linkonce). In that case we can't simply
9284566c6dbSTeresa Johnson     // convert to available_externally, since it would lose the
9294566c6dbSTeresa Johnson     // interposable property and possibly get inlined. Simply drop
9304566c6dbSTeresa Johnson     // the definition in that case.
9314566c6dbSTeresa Johnson     if (GlobalValue::isAvailableExternallyLinkage(NewLinkage) &&
9325a95c477STeresa Johnson         GlobalValue::isInterposableLinkage(GV.getLinkage())) {
9335a95c477STeresa Johnson       if (!convertToDeclaration(GV))
9345a95c477STeresa Johnson         // FIXME: Change this to collect replaced GVs and later erase
9355a95c477STeresa Johnson         // them from the parent module once thinLTOResolveWeakForLinkerGUID is
9365a95c477STeresa Johnson         // changed to enable this for aliases.
9375a95c477STeresa Johnson         llvm_unreachable("Expected GV to be converted");
9385a95c477STeresa Johnson     } else {
93933ba93c2SSteven Wu       // If the original symbols has global unnamed addr and linkonce_odr linkage,
94033ba93c2SSteven Wu       // it should be an auto hide symbol. Add hidden visibility to the symbol to
94133ba93c2SSteven Wu       // preserve the property.
94233ba93c2SSteven Wu       if (GV.hasLinkOnceODRLinkage() && GV.hasGlobalUnnamedAddr() &&
94333ba93c2SSteven Wu           NewLinkage == GlobalValue::WeakODRLinkage)
94433ba93c2SSteven Wu         GV.setVisibility(GlobalValue::HiddenVisibility);
94533ba93c2SSteven Wu 
946d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName()
947d34e60caSNicola Zaghen                         << "` from " << GV.getLinkage() << " to " << NewLinkage
948d34e60caSNicola Zaghen                         << "\n");
94904c9a2d6STeresa Johnson       GV.setLinkage(NewLinkage);
9504566c6dbSTeresa Johnson     }
9514566c6dbSTeresa Johnson     // Remove declarations from comdats, including available_externally
9526107a419STeresa Johnson     // as this is a declaration for the linker, and will be dropped eventually.
9536107a419STeresa Johnson     // It is illegal for comdats to contain declarations.
9546107a419STeresa Johnson     auto *GO = dyn_cast_or_null<GlobalObject>(&GV);
9554566c6dbSTeresa Johnson     if (GO && GO->isDeclarationForLinker() && GO->hasComdat())
9566107a419STeresa Johnson       GO->setComdat(nullptr);
95704c9a2d6STeresa Johnson   };
95804c9a2d6STeresa Johnson 
95904c9a2d6STeresa Johnson   // Process functions and global now
96004c9a2d6STeresa Johnson   for (auto &GV : TheModule)
96104c9a2d6STeresa Johnson     updateLinkage(GV);
96204c9a2d6STeresa Johnson   for (auto &GV : TheModule.globals())
96304c9a2d6STeresa Johnson     updateLinkage(GV);
96404c9a2d6STeresa Johnson   for (auto &GV : TheModule.aliases())
96504c9a2d6STeresa Johnson     updateLinkage(GV);
96604c9a2d6STeresa Johnson }
96704c9a2d6STeresa Johnson 
96804c9a2d6STeresa Johnson /// Run internalization on \p TheModule based on symmary analysis.
96904c9a2d6STeresa Johnson void llvm::thinLTOInternalizeModule(Module &TheModule,
97004c9a2d6STeresa Johnson                                     const GVSummaryMapTy &DefinedGlobals) {
97104c9a2d6STeresa Johnson   // Declare a callback for the internalize pass that will ask for every
97204c9a2d6STeresa Johnson   // candidate GlobalValue if it can be internalized or not.
97304c9a2d6STeresa Johnson   auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {
97404c9a2d6STeresa Johnson     // Lookup the linkage recorded in the summaries during global analysis.
975c3d677f9SPeter Collingbourne     auto GS = DefinedGlobals.find(GV.getGUID());
97604c9a2d6STeresa Johnson     if (GS == DefinedGlobals.end()) {
97704c9a2d6STeresa Johnson       // Must have been promoted (possibly conservatively). Find original
97804c9a2d6STeresa Johnson       // name so that we can access the correct summary and see if it can
97904c9a2d6STeresa Johnson       // be internalized again.
98004c9a2d6STeresa Johnson       // FIXME: Eventually we should control promotion instead of promoting
98104c9a2d6STeresa Johnson       // and internalizing again.
98204c9a2d6STeresa Johnson       StringRef OrigName =
98304c9a2d6STeresa Johnson           ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName());
98404c9a2d6STeresa Johnson       std::string OrigId = GlobalValue::getGlobalIdentifier(
98504c9a2d6STeresa Johnson           OrigName, GlobalValue::InternalLinkage,
98604c9a2d6STeresa Johnson           TheModule.getSourceFileName());
987c3d677f9SPeter Collingbourne       GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId));
9887ab1f692STeresa Johnson       if (GS == DefinedGlobals.end()) {
9897ab1f692STeresa Johnson         // Also check the original non-promoted non-globalized name. In some
9907ab1f692STeresa Johnson         // cases a preempted weak value is linked in as a local copy because
9917ab1f692STeresa Johnson         // it is referenced by an alias (IRLinker::linkGlobalValueProto).
9927ab1f692STeresa Johnson         // In that case, since it was originally not a local value, it was
9937ab1f692STeresa Johnson         // recorded in the index using the original name.
9947ab1f692STeresa Johnson         // FIXME: This may not be needed once PR27866 is fixed.
995c3d677f9SPeter Collingbourne         GS = DefinedGlobals.find(GlobalValue::getGUID(OrigName));
99604c9a2d6STeresa Johnson         assert(GS != DefinedGlobals.end());
9977ab1f692STeresa Johnson       }
998c3d677f9SPeter Collingbourne     }
999c3d677f9SPeter Collingbourne     return !GlobalValue::isLocalLinkage(GS->second->linkage());
100004c9a2d6STeresa Johnson   };
100104c9a2d6STeresa Johnson 
100204c9a2d6STeresa Johnson   // FIXME: See if we can just internalize directly here via linkage changes
100304c9a2d6STeresa Johnson   // based on the index, rather than invoking internalizeModule.
1004e9ea08a0SEugene Zelenko   internalizeModule(TheModule, MustPreserveGV);
100504c9a2d6STeresa Johnson }
100604c9a2d6STeresa Johnson 
100781bbf742STeresa Johnson /// Make alias a clone of its aliasee.
100881bbf742STeresa Johnson static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) {
100981bbf742STeresa Johnson   Function *Fn = cast<Function>(GA->getBaseObject());
101081bbf742STeresa Johnson 
101181bbf742STeresa Johnson   ValueToValueMapTy VMap;
101281bbf742STeresa Johnson   Function *NewFn = CloneFunction(Fn, VMap);
101381bbf742STeresa Johnson   // Clone should use the original alias's linkage and name, and we ensure
101481bbf742STeresa Johnson   // all uses of alias instead use the new clone (casted if necessary).
101581bbf742STeresa Johnson   NewFn->setLinkage(GA->getLinkage());
101681bbf742STeresa Johnson   GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewFn, GA->getType()));
101781bbf742STeresa Johnson   NewFn->takeName(GA);
101881bbf742STeresa Johnson   return NewFn;
101981bbf742STeresa Johnson }
102081bbf742STeresa Johnson 
1021c8c55170SMehdi Amini // Automatically import functions in Module \p DestModule based on the summaries
1022c8c55170SMehdi Amini // index.
10237f00d0a1SPeter Collingbourne Expected<bool> FunctionImporter::importFunctions(
102466043797SAdrian Prantl     Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) {
1025d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Starting import for Module "
1026311fef6eSMehdi Amini                     << DestModule.getModuleIdentifier() << "\n");
102719e23874SEugene Leviant   unsigned ImportedCount = 0, ImportedGVCount = 0;
1028c8c55170SMehdi Amini 
10296d8f817fSPeter Collingbourne   IRMover Mover(DestModule);
10307e88d0daSMehdi Amini   // Do the actual import of functions now, one Module at a time
103101e32130SMehdi Amini   std::set<StringRef> ModuleNameOrderedList;
103201e32130SMehdi Amini   for (auto &FunctionsToImportPerModule : ImportList) {
103301e32130SMehdi Amini     ModuleNameOrderedList.insert(FunctionsToImportPerModule.first());
103401e32130SMehdi Amini   }
103501e32130SMehdi Amini   for (auto &Name : ModuleNameOrderedList) {
10367e88d0daSMehdi Amini     // Get the module for the import
103701e32130SMehdi Amini     const auto &FunctionsToImportPerModule = ImportList.find(Name);
103801e32130SMehdi Amini     assert(FunctionsToImportPerModule != ImportList.end());
1039d9445c49SPeter Collingbourne     Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(Name);
1040d9445c49SPeter Collingbourne     if (!SrcModuleOrErr)
1041d9445c49SPeter Collingbourne       return SrcModuleOrErr.takeError();
1042d9445c49SPeter Collingbourne     std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
10437e88d0daSMehdi Amini     assert(&DestModule.getContext() == &SrcModule->getContext() &&
10447e88d0daSMehdi Amini            "Context mismatch");
10457e88d0daSMehdi Amini 
10466cba37ceSTeresa Johnson     // If modules were created with lazy metadata loading, materialize it
10476cba37ceSTeresa Johnson     // now, before linking it (otherwise this will be a noop).
10487f00d0a1SPeter Collingbourne     if (Error Err = SrcModule->materializeMetadata())
10497f00d0a1SPeter Collingbourne       return std::move(Err);
1050e5a61917STeresa Johnson 
105101e32130SMehdi Amini     auto &ImportGUIDs = FunctionsToImportPerModule->second;
105201e32130SMehdi Amini     // Find the globals to import
10536d8f817fSPeter Collingbourne     SetVector<GlobalValue *> GlobalsToImport;
10541f685e01SPiotr Padlewski     for (Function &F : *SrcModule) {
10551f685e01SPiotr Padlewski       if (!F.hasName())
10560beb858eSTeresa Johnson         continue;
10571f685e01SPiotr Padlewski       auto GUID = F.getGUID();
10580beb858eSTeresa Johnson       auto Import = ImportGUIDs.count(GUID);
1059d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing function "
1060d34e60caSNicola Zaghen                         << GUID << " " << F.getName() << " from "
1061aeb1e59bSMehdi Amini                         << SrcModule->getSourceFileName() << "\n");
10620beb858eSTeresa Johnson       if (Import) {
10637f00d0a1SPeter Collingbourne         if (Error Err = F.materialize())
10647f00d0a1SPeter Collingbourne           return std::move(Err);
10653b776128SPiotr Padlewski         if (EnableImportMetadata) {
10666deaa6afSPiotr Padlewski           // Add 'thinlto_src_module' metadata for statistics and debugging.
10673b776128SPiotr Padlewski           F.setMetadata(
10683b776128SPiotr Padlewski               "thinlto_src_module",
1069e9ea08a0SEugene Zelenko               MDNode::get(DestModule.getContext(),
1070e9ea08a0SEugene Zelenko                           {MDString::get(DestModule.getContext(),
10716deaa6afSPiotr Padlewski                                          SrcModule->getSourceFileName())}));
10723b776128SPiotr Padlewski         }
10731f685e01SPiotr Padlewski         GlobalsToImport.insert(&F);
107401e32130SMehdi Amini       }
107501e32130SMehdi Amini     }
10761f685e01SPiotr Padlewski     for (GlobalVariable &GV : SrcModule->globals()) {
10772d28f7aaSMehdi Amini       if (!GV.hasName())
10782d28f7aaSMehdi Amini         continue;
10792d28f7aaSMehdi Amini       auto GUID = GV.getGUID();
10802d28f7aaSMehdi Amini       auto Import = ImportGUIDs.count(GUID);
1081d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing global "
1082d34e60caSNicola Zaghen                         << GUID << " " << GV.getName() << " from "
1083aeb1e59bSMehdi Amini                         << SrcModule->getSourceFileName() << "\n");
10842d28f7aaSMehdi Amini       if (Import) {
10857f00d0a1SPeter Collingbourne         if (Error Err = GV.materialize())
10867f00d0a1SPeter Collingbourne           return std::move(Err);
108719e23874SEugene Leviant         ImportedGVCount += GlobalsToImport.insert(&GV);
10882d28f7aaSMehdi Amini       }
10892d28f7aaSMehdi Amini     }
10901f685e01SPiotr Padlewski     for (GlobalAlias &GA : SrcModule->aliases()) {
10911f685e01SPiotr Padlewski       if (!GA.hasName())
109201e32130SMehdi Amini         continue;
10931f685e01SPiotr Padlewski       auto GUID = GA.getGUID();
109481bbf742STeresa Johnson       auto Import = ImportGUIDs.count(GUID);
1095d34e60caSNicola Zaghen       LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing alias "
1096d34e60caSNicola Zaghen                         << GUID << " " << GA.getName() << " from "
1097aeb1e59bSMehdi Amini                         << SrcModule->getSourceFileName() << "\n");
109881bbf742STeresa Johnson       if (Import) {
109981bbf742STeresa Johnson         if (Error Err = GA.materialize())
110081bbf742STeresa Johnson           return std::move(Err);
110181bbf742STeresa Johnson         // Import alias as a copy of its aliasee.
110281bbf742STeresa Johnson         GlobalObject *Base = GA.getBaseObject();
110381bbf742STeresa Johnson         if (Error Err = Base->materialize())
110481bbf742STeresa Johnson           return std::move(Err);
110581bbf742STeresa Johnson         auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA);
1106d34e60caSNicola Zaghen         LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << Base->getGUID()
110781bbf742STeresa Johnson                           << " " << Base->getName() << " from "
110881bbf742STeresa Johnson                           << SrcModule->getSourceFileName() << "\n");
110981bbf742STeresa Johnson         if (EnableImportMetadata) {
111081bbf742STeresa Johnson           // Add 'thinlto_src_module' metadata for statistics and debugging.
111181bbf742STeresa Johnson           Fn->setMetadata(
111281bbf742STeresa Johnson               "thinlto_src_module",
111381bbf742STeresa Johnson               MDNode::get(DestModule.getContext(),
111481bbf742STeresa Johnson                           {MDString::get(DestModule.getContext(),
111581bbf742STeresa Johnson                                          SrcModule->getSourceFileName())}));
111601e32130SMehdi Amini         }
111781bbf742STeresa Johnson         GlobalsToImport.insert(Fn);
111881bbf742STeresa Johnson       }
111981bbf742STeresa Johnson     }
112001e32130SMehdi Amini 
112119ef4fadSMehdi Amini     // Upgrade debug info after we're done materializing all the globals and we
112219ef4fadSMehdi Amini     // have loaded all the required metadata!
112319ef4fadSMehdi Amini     UpgradeDebugInfo(*SrcModule);
112419ef4fadSMehdi Amini 
11257e88d0daSMehdi Amini     // Link in the specified functions.
112601e32130SMehdi Amini     if (renameModuleForThinLTO(*SrcModule, Index, &GlobalsToImport))
11278d05185aSMehdi Amini       return true;
11288d05185aSMehdi Amini 
1129d29478f7STeresa Johnson     if (PrintImports) {
1130d29478f7STeresa Johnson       for (const auto *GV : GlobalsToImport)
1131d29478f7STeresa Johnson         dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName()
1132d29478f7STeresa Johnson                << " from " << SrcModule->getSourceFileName() << "\n";
1133d29478f7STeresa Johnson     }
1134d29478f7STeresa Johnson 
11356d8f817fSPeter Collingbourne     if (Mover.move(std::move(SrcModule), GlobalsToImport.getArrayRef(),
11366d8f817fSPeter Collingbourne                    [](GlobalValue &, IRMover::ValueAdder) {},
1137e6fd9ff9SPeter Collingbourne                    /*IsPerformingImport=*/true))
11387e88d0daSMehdi Amini       report_fatal_error("Function Import: link error");
11397e88d0daSMehdi Amini 
114001e32130SMehdi Amini     ImportedCount += GlobalsToImport.size();
11416c475a75STeresa Johnson     NumImportedModules++;
11427e88d0daSMehdi Amini   }
1143e5a61917STeresa Johnson 
114419e23874SEugene Leviant   NumImportedFunctions += (ImportedCount - ImportedGVCount);
114519e23874SEugene Leviant   NumImportedGlobalVars += ImportedGVCount;
1146d29478f7STeresa Johnson 
1147d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount
1148d34e60caSNicola Zaghen                     << " functions for Module "
1149d34e60caSNicola Zaghen                     << DestModule.getModuleIdentifier() << "\n");
1150d34e60caSNicola Zaghen   LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount
115119e23874SEugene Leviant                     << " global variables for Module "
1152c8c55170SMehdi Amini                     << DestModule.getModuleIdentifier() << "\n");
1153c8c55170SMehdi Amini   return ImportedCount;
115442418abaSMehdi Amini }
115542418abaSMehdi Amini 
1156598bd2a2SPeter Collingbourne static bool doImportingForModule(Module &M) {
1157598bd2a2SPeter Collingbourne   if (SummaryFile.empty())
1158598bd2a2SPeter Collingbourne     report_fatal_error("error: -function-import requires -summary-file\n");
11596de481a3SPeter Collingbourne   Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr =
11606de481a3SPeter Collingbourne       getModuleSummaryIndexForFile(SummaryFile);
11616de481a3SPeter Collingbourne   if (!IndexPtrOrErr) {
11626de481a3SPeter Collingbourne     logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(),
11636de481a3SPeter Collingbourne                           "Error loading file '" + SummaryFile + "': ");
116442418abaSMehdi Amini     return false;
116542418abaSMehdi Amini   }
1166598bd2a2SPeter Collingbourne   std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr);
116742418abaSMehdi Amini 
1168c86af334STeresa Johnson   // First step is collecting the import list.
1169c86af334STeresa Johnson   FunctionImporter::ImportMapTy ImportList;
117081bbf742STeresa Johnson   // If requested, simply import all functions in the index. This is used
117181bbf742STeresa Johnson   // when testing distributed backend handling via the opt tool, when
117281bbf742STeresa Johnson   // we have distributed indexes containing exactly the summaries to import.
117381bbf742STeresa Johnson   if (ImportAllIndex)
117481bbf742STeresa Johnson     ComputeCrossModuleImportForModuleFromIndex(M.getModuleIdentifier(), *Index,
117581bbf742STeresa Johnson                                                ImportList);
117681bbf742STeresa Johnson   else
1177c86af334STeresa Johnson     ComputeCrossModuleImportForModule(M.getModuleIdentifier(), *Index,
1178c86af334STeresa Johnson                                       ImportList);
117901e32130SMehdi Amini 
11804fef68cbSTeresa Johnson   // Conservatively mark all internal values as promoted. This interface is
11814fef68cbSTeresa Johnson   // only used when doing importing via the function importing pass. The pass
11824fef68cbSTeresa Johnson   // is only enabled when testing importing via the 'opt' tool, which does
11834fef68cbSTeresa Johnson   // not do the ThinLink that would normally determine what values to promote.
11844fef68cbSTeresa Johnson   for (auto &I : *Index) {
11859667b91bSPeter Collingbourne     for (auto &S : I.second.SummaryList) {
11864fef68cbSTeresa Johnson       if (GlobalValue::isLocalLinkage(S->linkage()))
11874fef68cbSTeresa Johnson         S->setLinkage(GlobalValue::ExternalLinkage);
11884fef68cbSTeresa Johnson     }
11894fef68cbSTeresa Johnson   }
11904fef68cbSTeresa Johnson 
119101e32130SMehdi Amini   // Next we need to promote to global scope and rename any local values that
11921b00f2d9STeresa Johnson   // are potentially exported to other modules.
119301e32130SMehdi Amini   if (renameModuleForThinLTO(M, *Index, nullptr)) {
11941b00f2d9STeresa Johnson     errs() << "Error renaming module\n";
11951b00f2d9STeresa Johnson     return false;
11961b00f2d9STeresa Johnson   }
11971b00f2d9STeresa Johnson 
119842418abaSMehdi Amini   // Perform the import now.
1199d16c8065SMehdi Amini   auto ModuleLoader = [&M](StringRef Identifier) {
1200d16c8065SMehdi Amini     return loadFile(Identifier, M.getContext());
1201d16c8065SMehdi Amini   };
12029d2bfc48SRafael Espindola   FunctionImporter Importer(*Index, ModuleLoader);
120337e24591SPeter Collingbourne   Expected<bool> Result = Importer.importFunctions(M, ImportList);
12047f00d0a1SPeter Collingbourne 
12057f00d0a1SPeter Collingbourne   // FIXME: Probably need to propagate Errors through the pass manager.
12067f00d0a1SPeter Collingbourne   if (!Result) {
12077f00d0a1SPeter Collingbourne     logAllUnhandledErrors(Result.takeError(), errs(),
12087f00d0a1SPeter Collingbourne                           "Error importing module: ");
12097f00d0a1SPeter Collingbourne     return false;
12107f00d0a1SPeter Collingbourne   }
12117f00d0a1SPeter Collingbourne 
12127f00d0a1SPeter Collingbourne   return *Result;
121321241571STeresa Johnson }
121421241571STeresa Johnson 
121521241571STeresa Johnson namespace {
1216e9ea08a0SEugene Zelenko 
121721241571STeresa Johnson /// Pass that performs cross-module function import provided a summary file.
121821241571STeresa Johnson class FunctionImportLegacyPass : public ModulePass {
121921241571STeresa Johnson public:
122021241571STeresa Johnson   /// Pass identification, replacement for typeid
122121241571STeresa Johnson   static char ID;
122221241571STeresa Johnson 
1223e9ea08a0SEugene Zelenko   explicit FunctionImportLegacyPass() : ModulePass(ID) {}
1224e9ea08a0SEugene Zelenko 
122521241571STeresa Johnson   /// Specify pass name for debug output
1226117296c0SMehdi Amini   StringRef getPassName() const override { return "Function Importing"; }
122721241571STeresa Johnson 
122821241571STeresa Johnson   bool runOnModule(Module &M) override {
122921241571STeresa Johnson     if (skipModule(M))
123021241571STeresa Johnson       return false;
123121241571STeresa Johnson 
1232598bd2a2SPeter Collingbourne     return doImportingForModule(M);
123342418abaSMehdi Amini   }
123442418abaSMehdi Amini };
1235e9ea08a0SEugene Zelenko 
1236e9ea08a0SEugene Zelenko } // end anonymous namespace
123742418abaSMehdi Amini 
123821241571STeresa Johnson PreservedAnalyses FunctionImportPass::run(Module &M,
1239fd03ac6aSSean Silva                                           ModuleAnalysisManager &AM) {
1240598bd2a2SPeter Collingbourne   if (!doImportingForModule(M))
124121241571STeresa Johnson     return PreservedAnalyses::all();
124221241571STeresa Johnson 
124321241571STeresa Johnson   return PreservedAnalyses::none();
124421241571STeresa Johnson }
124521241571STeresa Johnson 
124621241571STeresa Johnson char FunctionImportLegacyPass::ID = 0;
124721241571STeresa Johnson INITIALIZE_PASS(FunctionImportLegacyPass, "function-import",
124842418abaSMehdi Amini                 "Summary Based Function Import", false, false)
124942418abaSMehdi Amini 
125042418abaSMehdi Amini namespace llvm {
1251e9ea08a0SEugene Zelenko 
1252598bd2a2SPeter Collingbourne Pass *createFunctionImportPass() {
1253598bd2a2SPeter Collingbourne   return new FunctionImportLegacyPass();
12545fcbdb71STeresa Johnson }
1255e9ea08a0SEugene Zelenko 
1256e9ea08a0SEugene Zelenko } // end namespace llvm
1257