142418abaSMehdi Amini //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===// 242418abaSMehdi Amini // 342418abaSMehdi Amini // The LLVM Compiler Infrastructure 442418abaSMehdi Amini // 542418abaSMehdi Amini // This file is distributed under the University of Illinois Open Source 642418abaSMehdi Amini // License. See LICENSE.TXT for details. 742418abaSMehdi Amini // 842418abaSMehdi Amini //===----------------------------------------------------------------------===// 942418abaSMehdi Amini // 1042418abaSMehdi Amini // This file implements Function import based on summaries. 1142418abaSMehdi Amini // 1242418abaSMehdi Amini //===----------------------------------------------------------------------===// 1342418abaSMehdi Amini 1442418abaSMehdi Amini #include "llvm/Transforms/IPO/FunctionImport.h" 15e9ea08a0SEugene Zelenko #include "llvm/ADT/ArrayRef.h" 16e9ea08a0SEugene Zelenko #include "llvm/ADT/STLExtras.h" 17e9ea08a0SEugene Zelenko #include "llvm/ADT/SetVector.h" 1801e32130SMehdi Amini #include "llvm/ADT/SmallVector.h" 19d29478f7STeresa Johnson #include "llvm/ADT/Statistic.h" 20e9ea08a0SEugene Zelenko #include "llvm/ADT/StringMap.h" 210efed325SSimon Pilgrim #include "llvm/ADT/StringRef.h" 22b040fcc6SCharles Saternos #include "llvm/ADT/StringSet.h" 23c15d60b7SPeter Collingbourne #include "llvm/Bitcode/BitcodeReader.h" 2442418abaSMehdi Amini #include "llvm/IR/AutoUpgrade.h" 2581bbf742STeresa Johnson #include "llvm/IR/Constants.h" 26e9ea08a0SEugene Zelenko #include "llvm/IR/Function.h" 27e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalAlias.h" 28e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalObject.h" 29e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalValue.h" 30e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalVariable.h" 31e9ea08a0SEugene Zelenko #include "llvm/IR/Metadata.h" 3242418abaSMehdi Amini #include "llvm/IR/Module.h" 33e9ea08a0SEugene Zelenko #include "llvm/IR/ModuleSummaryIndex.h" 3442418abaSMehdi Amini #include "llvm/IRReader/IRReader.h" 35e9ea08a0SEugene Zelenko #include "llvm/Linker/IRMover.h" 36e9ea08a0SEugene Zelenko #include "llvm/Object/ModuleSymbolTable.h" 37e9ea08a0SEugene Zelenko #include "llvm/Object/SymbolicFile.h" 38e9ea08a0SEugene Zelenko #include "llvm/Pass.h" 39e9ea08a0SEugene Zelenko #include "llvm/Support/Casting.h" 4042418abaSMehdi Amini #include "llvm/Support/CommandLine.h" 4142418abaSMehdi Amini #include "llvm/Support/Debug.h" 42e9ea08a0SEugene Zelenko #include "llvm/Support/Error.h" 43e9ea08a0SEugene Zelenko #include "llvm/Support/ErrorHandling.h" 44e9ea08a0SEugene Zelenko #include "llvm/Support/FileSystem.h" 4542418abaSMehdi Amini #include "llvm/Support/SourceMgr.h" 46e9ea08a0SEugene Zelenko #include "llvm/Support/raw_ostream.h" 4704c9a2d6STeresa Johnson #include "llvm/Transforms/IPO/Internalize.h" 4881bbf742STeresa Johnson #include "llvm/Transforms/Utils/Cloning.h" 49488a800aSTeresa Johnson #include "llvm/Transforms/Utils/FunctionImportUtils.h" 5081bbf742STeresa Johnson #include "llvm/Transforms/Utils/ValueMapper.h" 51e9ea08a0SEugene Zelenko #include <cassert> 52e9ea08a0SEugene Zelenko #include <memory> 53e9ea08a0SEugene Zelenko #include <set> 54e9ea08a0SEugene Zelenko #include <string> 55e9ea08a0SEugene Zelenko #include <system_error> 56e9ea08a0SEugene Zelenko #include <tuple> 57e9ea08a0SEugene Zelenko #include <utility> 587e88d0daSMehdi Amini 5942418abaSMehdi Amini using namespace llvm; 6042418abaSMehdi Amini 61e9ea08a0SEugene Zelenko #define DEBUG_TYPE "function-import" 62e9ea08a0SEugene Zelenko 63d2c234a4STeresa Johnson STATISTIC(NumImportedFunctionsThinLink, 64d2c234a4STeresa Johnson "Number of functions thin link decided to import"); 65d2c234a4STeresa Johnson STATISTIC(NumImportedHotFunctionsThinLink, 66d2c234a4STeresa Johnson "Number of hot functions thin link decided to import"); 67d2c234a4STeresa Johnson STATISTIC(NumImportedCriticalFunctionsThinLink, 68d2c234a4STeresa Johnson "Number of critical functions thin link decided to import"); 69d2c234a4STeresa Johnson STATISTIC(NumImportedGlobalVarsThinLink, 70d2c234a4STeresa Johnson "Number of global variables thin link decided to import"); 71d2c234a4STeresa Johnson STATISTIC(NumImportedFunctions, "Number of functions imported in backend"); 72d2c234a4STeresa Johnson STATISTIC(NumImportedGlobalVars, 73d2c234a4STeresa Johnson "Number of global variables imported in backend"); 746c475a75STeresa Johnson STATISTIC(NumImportedModules, "Number of modules imported from"); 756c475a75STeresa Johnson STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index"); 766c475a75STeresa Johnson STATISTIC(NumLiveSymbols, "Number of live symbols in index"); 77d29478f7STeresa Johnson 7839303619STeresa Johnson /// Limit on instruction count of imported functions. 7939303619STeresa Johnson static cl::opt<unsigned> ImportInstrLimit( 8039303619STeresa Johnson "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), 8139303619STeresa Johnson cl::desc("Only import functions with less than N instructions")); 8239303619STeresa Johnson 83974706ebSTeresa Johnson static cl::opt<int> ImportCutoff( 84974706ebSTeresa Johnson "import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), 85974706ebSTeresa Johnson cl::desc("Only import first N functions if N>=0 (default -1)")); 86974706ebSTeresa Johnson 8740641748SMehdi Amini static cl::opt<float> 8840641748SMehdi Amini ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), 8940641748SMehdi Amini cl::Hidden, cl::value_desc("x"), 9040641748SMehdi Amini cl::desc("As we import functions, multiply the " 9140641748SMehdi Amini "`import-instr-limit` threshold by this factor " 9240641748SMehdi Amini "before processing newly imported functions")); 93ba72b95fSPiotr Padlewski 94d2869473SPiotr Padlewski static cl::opt<float> ImportHotInstrFactor( 95d2869473SPiotr Padlewski "import-hot-evolution-factor", cl::init(1.0), cl::Hidden, 96d2869473SPiotr Padlewski cl::value_desc("x"), 97d2869473SPiotr Padlewski cl::desc("As we import functions called from hot callsite, multiply the " 98d2869473SPiotr Padlewski "`import-instr-limit` threshold by this factor " 99d2869473SPiotr Padlewski "before processing newly imported functions")); 100d2869473SPiotr Padlewski 101d9830eb7SPiotr Padlewski static cl::opt<float> ImportHotMultiplier( 1028260d665SDehao Chen "import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), 103ba72b95fSPiotr Padlewski cl::desc("Multiply the `import-instr-limit` threshold for hot callsites")); 104ba72b95fSPiotr Padlewski 10564c46574SDehao Chen static cl::opt<float> ImportCriticalMultiplier( 10664c46574SDehao Chen "import-critical-multiplier", cl::init(100.0), cl::Hidden, 10764c46574SDehao Chen cl::value_desc("x"), 10864c46574SDehao Chen cl::desc( 10964c46574SDehao Chen "Multiply the `import-instr-limit` threshold for critical callsites")); 11064c46574SDehao Chen 111ba72b95fSPiotr Padlewski // FIXME: This multiplier was not really tuned up. 112ba72b95fSPiotr Padlewski static cl::opt<float> ImportColdMultiplier( 113ba72b95fSPiotr Padlewski "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), 114ba72b95fSPiotr Padlewski cl::desc("Multiply the `import-instr-limit` threshold for cold callsites")); 11540641748SMehdi Amini 116d29478f7STeresa Johnson static cl::opt<bool> PrintImports("print-imports", cl::init(false), cl::Hidden, 117d29478f7STeresa Johnson cl::desc("Print imported functions")); 118d29478f7STeresa Johnson 119cb9a82fcSTeresa Johnson static cl::opt<bool> PrintImportFailures( 120cb9a82fcSTeresa Johnson "print-import-failures", cl::init(false), cl::Hidden, 121cb9a82fcSTeresa Johnson cl::desc("Print information for functions rejected for importing")); 122cb9a82fcSTeresa Johnson 1236c475a75STeresa Johnson static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden, 1246c475a75STeresa Johnson cl::desc("Compute dead symbols")); 1256c475a75STeresa Johnson 1263b776128SPiotr Padlewski static cl::opt<bool> EnableImportMetadata( 1273b776128SPiotr Padlewski "enable-import-metadata", cl::init( 1283b776128SPiotr Padlewski #if !defined(NDEBUG) 1293b776128SPiotr Padlewski true /*Enabled with asserts.*/ 1303b776128SPiotr Padlewski #else 1313b776128SPiotr Padlewski false 1323b776128SPiotr Padlewski #endif 1333b776128SPiotr Padlewski ), 1343b776128SPiotr Padlewski cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module'")); 1353b776128SPiotr Padlewski 136e9ea08a0SEugene Zelenko /// Summary file to use for function importing when using -function-import from 137e9ea08a0SEugene Zelenko /// the command line. 138e9ea08a0SEugene Zelenko static cl::opt<std::string> 139e9ea08a0SEugene Zelenko SummaryFile("summary-file", 140e9ea08a0SEugene Zelenko cl::desc("The summary file to use for function importing.")); 141e9ea08a0SEugene Zelenko 14281bbf742STeresa Johnson /// Used when testing importing from distributed indexes via opt 14381bbf742STeresa Johnson // -function-import. 14481bbf742STeresa Johnson static cl::opt<bool> 14581bbf742STeresa Johnson ImportAllIndex("import-all-index", 14681bbf742STeresa Johnson cl::desc("Import all external functions in index.")); 14781bbf742STeresa Johnson 14842418abaSMehdi Amini // Load lazily a module from \p FileName in \p Context. 14942418abaSMehdi Amini static std::unique_ptr<Module> loadFile(const std::string &FileName, 15042418abaSMehdi Amini LLVMContext &Context) { 15142418abaSMehdi Amini SMDiagnostic Err; 152d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n"); 1536cba37ceSTeresa Johnson // Metadata isn't loaded until functions are imported, to minimize 1546cba37ceSTeresa Johnson // the memory overhead. 155a1080ee6STeresa Johnson std::unique_ptr<Module> Result = 156a1080ee6STeresa Johnson getLazyIRFileModule(FileName, Err, Context, 157a1080ee6STeresa Johnson /* ShouldLazyLoadMetadata = */ true); 15842418abaSMehdi Amini if (!Result) { 15942418abaSMehdi Amini Err.print("function-import", errs()); 160d7ad221cSMehdi Amini report_fatal_error("Abort"); 16142418abaSMehdi Amini } 16242418abaSMehdi Amini 16342418abaSMehdi Amini return Result; 16442418abaSMehdi Amini } 16542418abaSMehdi Amini 16601e32130SMehdi Amini /// Given a list of possible callee implementation for a call site, select one 16701e32130SMehdi Amini /// that fits the \p Threshold. 16801e32130SMehdi Amini /// 16901e32130SMehdi Amini /// FIXME: select "best" instead of first that fits. But what is "best"? 17001e32130SMehdi Amini /// - The smallest: more likely to be inlined. 17101e32130SMehdi Amini /// - The one with the least outgoing edges (already well optimized). 17201e32130SMehdi Amini /// - One from a module already being imported from in order to reduce the 17301e32130SMehdi Amini /// number of source modules parsed/linked. 17401e32130SMehdi Amini /// - One that has PGO data attached. 17501e32130SMehdi Amini /// - [insert you fancy metric here] 1762d28f7aaSMehdi Amini static const GlobalValueSummary * 177b4e1e829SMehdi Amini selectCallee(const ModuleSummaryIndex &Index, 1789667b91bSPeter Collingbourne ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList, 179cb9a82fcSTeresa Johnson unsigned Threshold, StringRef CallerModulePath, 180cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason &Reason, 181cb9a82fcSTeresa Johnson GlobalValue::GUID GUID) { 182cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::None; 18301e32130SMehdi Amini auto It = llvm::find_if( 18428e457bcSTeresa Johnson CalleeSummaryList, 18528e457bcSTeresa Johnson [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) { 18628e457bcSTeresa Johnson auto *GVSummary = SummaryPtr.get(); 187cb9a82fcSTeresa Johnson if (!Index.isGlobalValueLive(GVSummary)) { 188cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::NotLive; 189eaf5172cSGeorge Rimar return false; 190cb9a82fcSTeresa Johnson } 191eaf5172cSGeorge Rimar 19273305f82STeresa Johnson // For SamplePGO, in computeImportForFunction the OriginalId 19373305f82STeresa Johnson // may have been used to locate the callee summary list (See 19473305f82STeresa Johnson // comment there). 19573305f82STeresa Johnson // The mapping from OriginalId to GUID may return a GUID 19673305f82STeresa Johnson // that corresponds to a static variable. Filter it out here. 19773305f82STeresa Johnson // This can happen when 19873305f82STeresa Johnson // 1) There is a call to a library function which is not defined 19973305f82STeresa Johnson // in the index. 20073305f82STeresa Johnson // 2) There is a static variable with the OriginalGUID identical 20173305f82STeresa Johnson // to the GUID of the library function in 1); 20273305f82STeresa Johnson // When this happens, the logic for SamplePGO kicks in and 20373305f82STeresa Johnson // the static variable in 2) will be found, which needs to be 20473305f82STeresa Johnson // filtered out. 205cb9a82fcSTeresa Johnson if (GVSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind) { 206cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::GlobalVar; 20773305f82STeresa Johnson return false; 208cb9a82fcSTeresa Johnson } 209cb9a82fcSTeresa Johnson if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) { 210cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::InterposableLinkage; 2115b85d8d6SMehdi Amini // There is no point in importing these, we can't inline them 21201e32130SMehdi Amini return false; 213cb9a82fcSTeresa Johnson } 2142c719cc1SMehdi Amini 21581bbf742STeresa Johnson auto *Summary = cast<FunctionSummary>(GVSummary->getBaseObject()); 2167e88d0daSMehdi Amini 21783aaf358STeresa Johnson // If this is a local function, make sure we import the copy 21883aaf358STeresa Johnson // in the caller's module. The only time a local function can 21983aaf358STeresa Johnson // share an entry in the index is if there is a local with the same name 22083aaf358STeresa Johnson // in another module that had the same source file name (in a different 22183aaf358STeresa Johnson // directory), where each was compiled in their own directory so there 22283aaf358STeresa Johnson // was not distinguishing path. 22383aaf358STeresa Johnson // However, do the import from another module if there is only one 22483aaf358STeresa Johnson // entry in the list - in that case this must be a reference due 22583aaf358STeresa Johnson // to indirect call profile data, since a function pointer can point to 22683aaf358STeresa Johnson // a local in another module. 22783aaf358STeresa Johnson if (GlobalValue::isLocalLinkage(Summary->linkage()) && 22883aaf358STeresa Johnson CalleeSummaryList.size() > 1 && 229cb9a82fcSTeresa Johnson Summary->modulePath() != CallerModulePath) { 230cb9a82fcSTeresa Johnson Reason = 231cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason::LocalLinkageNotInModule; 23283aaf358STeresa Johnson return false; 233cb9a82fcSTeresa Johnson } 23483aaf358STeresa Johnson 235cb9a82fcSTeresa Johnson if (Summary->instCount() > Threshold) { 236cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::TooLarge; 237f9dc3deaSTeresa Johnson return false; 238cb9a82fcSTeresa Johnson } 239f9dc3deaSTeresa Johnson 240*cb397461STeresa Johnson // Skip if it isn't legal to import (e.g. may reference unpromotable 241*cb397461STeresa Johnson // locals). 242cb9a82fcSTeresa Johnson if (Summary->notEligibleToImport()) { 243cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::NotEligible; 244b4e1e829SMehdi Amini return false; 245cb9a82fcSTeresa Johnson } 246b4e1e829SMehdi Amini 247*cb397461STeresa Johnson // Don't bother importing if we can't inline it anyway. 248*cb397461STeresa Johnson if (Summary->fflags().NoInline) { 249*cb397461STeresa Johnson Reason = FunctionImporter::ImportFailureReason::NoInline; 250*cb397461STeresa Johnson return false; 251*cb397461STeresa Johnson } 252*cb397461STeresa Johnson 25301e32130SMehdi Amini return true; 25401e32130SMehdi Amini }); 25528e457bcSTeresa Johnson if (It == CalleeSummaryList.end()) 25601e32130SMehdi Amini return nullptr; 2577e88d0daSMehdi Amini 258f9dc3deaSTeresa Johnson return cast<GlobalValueSummary>(It->get()); 259434e9561SRafael Espindola } 2607e88d0daSMehdi Amini 261e9ea08a0SEugene Zelenko namespace { 262e9ea08a0SEugene Zelenko 263475b51a7STeresa Johnson using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */, 264475b51a7STeresa Johnson GlobalValue::GUID>; 26501e32130SMehdi Amini 266e9ea08a0SEugene Zelenko } // anonymous namespace 267e9ea08a0SEugene Zelenko 2681958083dSTeresa Johnson static ValueInfo 2691958083dSTeresa Johnson updateValueInfoForIndirectCalls(const ModuleSummaryIndex &Index, ValueInfo VI) { 2701958083dSTeresa Johnson if (!VI.getSummaryList().empty()) 2711958083dSTeresa Johnson return VI; 2721958083dSTeresa Johnson // For SamplePGO, the indirect call targets for local functions will 2731958083dSTeresa Johnson // have its original name annotated in profile. We try to find the 2741958083dSTeresa Johnson // corresponding PGOFuncName as the GUID. 2751958083dSTeresa Johnson // FIXME: Consider updating the edges in the graph after building 2761958083dSTeresa Johnson // it, rather than needing to perform this mapping on each walk. 2771958083dSTeresa Johnson auto GUID = Index.getGUIDFromOriginalID(VI.getGUID()); 2781958083dSTeresa Johnson if (GUID == 0) 27928d8a49fSEugene Leviant return ValueInfo(); 2801958083dSTeresa Johnson return Index.getValueInfo(GUID); 2811958083dSTeresa Johnson } 2821958083dSTeresa Johnson 28319e23874SEugene Leviant static void computeImportForReferencedGlobals( 28419e23874SEugene Leviant const FunctionSummary &Summary, const GVSummaryMapTy &DefinedGVSummaries, 28519e23874SEugene Leviant FunctionImporter::ImportMapTy &ImportList, 28619e23874SEugene Leviant StringMap<FunctionImporter::ExportSetTy> *ExportLists) { 28719e23874SEugene Leviant for (auto &VI : Summary.refs()) { 28819e23874SEugene Leviant if (DefinedGVSummaries.count(VI.getGUID())) { 289d34e60caSNicola Zaghen LLVM_DEBUG( 290d34e60caSNicola Zaghen dbgs() << "Ref ignored! Target already in destination module.\n"); 29119e23874SEugene Leviant continue; 29219e23874SEugene Leviant } 29319e23874SEugene Leviant 2947e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n"); 29519e23874SEugene Leviant 29619e23874SEugene Leviant for (auto &RefSummary : VI.getSummaryList()) 29719e23874SEugene Leviant if (RefSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind && 298eddf6b5dSEugene Leviant !RefSummary->notEligibleToImport() && 29919e23874SEugene Leviant !GlobalValue::isInterposableLinkage(RefSummary->linkage()) && 30019e23874SEugene Leviant RefSummary->refs().empty()) { 301d2c234a4STeresa Johnson auto ILI = ImportList[RefSummary->modulePath()].insert(VI.getGUID()); 302d2c234a4STeresa Johnson // Only update stat if we haven't already imported this variable. 303d2c234a4STeresa Johnson if (ILI.second) 304d2c234a4STeresa Johnson NumImportedGlobalVarsThinLink++; 30519e23874SEugene Leviant if (ExportLists) 30619e23874SEugene Leviant (*ExportLists)[RefSummary->modulePath()].insert(VI.getGUID()); 30719e23874SEugene Leviant break; 30819e23874SEugene Leviant } 30919e23874SEugene Leviant } 31019e23874SEugene Leviant } 31119e23874SEugene Leviant 312cb9a82fcSTeresa Johnson static const char * 313cb9a82fcSTeresa Johnson getFailureName(FunctionImporter::ImportFailureReason Reason) { 314cb9a82fcSTeresa Johnson switch (Reason) { 315cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::None: 316cb9a82fcSTeresa Johnson return "None"; 317cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::GlobalVar: 318cb9a82fcSTeresa Johnson return "GlobalVar"; 319cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::NotLive: 320cb9a82fcSTeresa Johnson return "NotLive"; 321cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::TooLarge: 322cb9a82fcSTeresa Johnson return "TooLarge"; 323cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::InterposableLinkage: 324cb9a82fcSTeresa Johnson return "InterposableLinkage"; 325cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule: 326cb9a82fcSTeresa Johnson return "LocalLinkageNotInModule"; 327cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::NotEligible: 328cb9a82fcSTeresa Johnson return "NotEligible"; 329*cb397461STeresa Johnson case FunctionImporter::ImportFailureReason::NoInline: 330*cb397461STeresa Johnson return "NoInline"; 331cb9a82fcSTeresa Johnson } 332cb9a82fcSTeresa Johnson llvm_unreachable("invalid reason"); 333cb9a82fcSTeresa Johnson } 334cb9a82fcSTeresa Johnson 33501e32130SMehdi Amini /// Compute the list of functions to import for a given caller. Mark these 33601e32130SMehdi Amini /// imported functions and the symbols they reference in their source module as 33701e32130SMehdi Amini /// exported from their source module. 33801e32130SMehdi Amini static void computeImportForFunction( 3393255eec1STeresa Johnson const FunctionSummary &Summary, const ModuleSummaryIndex &Index, 340d9830eb7SPiotr Padlewski const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, 34101e32130SMehdi Amini SmallVectorImpl<EdgeInfo> &Worklist, 3429b490f10SMehdi Amini FunctionImporter::ImportMapTy &ImportList, 343d68935c5STeresa Johnson StringMap<FunctionImporter::ExportSetTy> *ExportLists, 344d68935c5STeresa Johnson FunctionImporter::ImportThresholdsTy &ImportThresholds) { 34519e23874SEugene Leviant computeImportForReferencedGlobals(Summary, DefinedGVSummaries, ImportList, 34619e23874SEugene Leviant ExportLists); 347974706ebSTeresa Johnson static int ImportCount = 0; 34801e32130SMehdi Amini for (auto &Edge : Summary.calls()) { 3499667b91bSPeter Collingbourne ValueInfo VI = Edge.first; 3507e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold 3517e7b13d0STeresa Johnson << "\n"); 35201e32130SMehdi Amini 353974706ebSTeresa Johnson if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) { 354d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff 355974706ebSTeresa Johnson << " reached.\n"); 356974706ebSTeresa Johnson continue; 357974706ebSTeresa Johnson } 358974706ebSTeresa Johnson 3591958083dSTeresa Johnson VI = updateValueInfoForIndirectCalls(Index, VI); 3609667b91bSPeter Collingbourne if (!VI) 3619667b91bSPeter Collingbourne continue; 3624a435e08SDehao Chen 3639667b91bSPeter Collingbourne if (DefinedGVSummaries.count(VI.getGUID())) { 364d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n"); 3657e88d0daSMehdi Amini continue; 366d450da32STeresa Johnson } 36740641748SMehdi Amini 368ba72b95fSPiotr Padlewski auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float { 369ba72b95fSPiotr Padlewski if (Hotness == CalleeInfo::HotnessType::Hot) 370ba72b95fSPiotr Padlewski return ImportHotMultiplier; 371ba72b95fSPiotr Padlewski if (Hotness == CalleeInfo::HotnessType::Cold) 372ba72b95fSPiotr Padlewski return ImportColdMultiplier; 37364c46574SDehao Chen if (Hotness == CalleeInfo::HotnessType::Critical) 37464c46574SDehao Chen return ImportCriticalMultiplier; 375ba72b95fSPiotr Padlewski return 1.0; 376ba72b95fSPiotr Padlewski }; 377ba72b95fSPiotr Padlewski 378d9830eb7SPiotr Padlewski const auto NewThreshold = 379c73cec84SEaswaran Raman Threshold * GetBonusMultiplier(Edge.second.getHotness()); 380d2869473SPiotr Padlewski 381cb9a82fcSTeresa Johnson auto IT = ImportThresholds.insert(std::make_pair( 382cb9a82fcSTeresa Johnson VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr))); 383d68935c5STeresa Johnson bool PreviouslyVisited = !IT.second; 384cb9a82fcSTeresa Johnson auto &ProcessedThreshold = std::get<0>(IT.first->second); 385cb9a82fcSTeresa Johnson auto &CalleeSummary = std::get<1>(IT.first->second); 386cb9a82fcSTeresa Johnson auto &FailureInfo = std::get<2>(IT.first->second); 387d68935c5STeresa Johnson 388d2c234a4STeresa Johnson bool IsHotCallsite = 389d2c234a4STeresa Johnson Edge.second.getHotness() == CalleeInfo::HotnessType::Hot; 390d2c234a4STeresa Johnson bool IsCriticalCallsite = 391d2c234a4STeresa Johnson Edge.second.getHotness() == CalleeInfo::HotnessType::Critical; 392d2c234a4STeresa Johnson 393d68935c5STeresa Johnson const FunctionSummary *ResolvedCalleeSummary = nullptr; 394d68935c5STeresa Johnson if (CalleeSummary) { 395d68935c5STeresa Johnson assert(PreviouslyVisited); 396d68935c5STeresa Johnson // Since the traversal of the call graph is DFS, we can revisit a function 397d68935c5STeresa Johnson // a second time with a higher threshold. In this case, it is added back 398d68935c5STeresa Johnson // to the worklist with the new threshold (so that its own callee chains 399d68935c5STeresa Johnson // can be considered with the higher threshold). 400d68935c5STeresa Johnson if (NewThreshold <= ProcessedThreshold) { 401d68935c5STeresa Johnson LLVM_DEBUG( 402d68935c5STeresa Johnson dbgs() << "ignored! Target was already imported with Threshold " 403d68935c5STeresa Johnson << ProcessedThreshold << "\n"); 404d68935c5STeresa Johnson continue; 405d68935c5STeresa Johnson } 406d68935c5STeresa Johnson // Update with new larger threshold. 407d68935c5STeresa Johnson ProcessedThreshold = NewThreshold; 408d68935c5STeresa Johnson ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 409d68935c5STeresa Johnson } else { 410d68935c5STeresa Johnson // If we already rejected importing a callee at the same or higher 411d68935c5STeresa Johnson // threshold, don't waste time calling selectCallee. 412d68935c5STeresa Johnson if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) { 413d68935c5STeresa Johnson LLVM_DEBUG( 414d68935c5STeresa Johnson dbgs() << "ignored! Target was already rejected with Threshold " 415d68935c5STeresa Johnson << ProcessedThreshold << "\n"); 416cb9a82fcSTeresa Johnson if (PrintImportFailures) { 417cb9a82fcSTeresa Johnson assert(FailureInfo && 418cb9a82fcSTeresa Johnson "Expected FailureInfo for previously rejected candidate"); 419cb9a82fcSTeresa Johnson FailureInfo->Attempts++; 420cb9a82fcSTeresa Johnson } 421d68935c5STeresa Johnson continue; 422d68935c5STeresa Johnson } 423d68935c5STeresa Johnson 424cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason Reason; 425d68935c5STeresa Johnson CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold, 426cb9a82fcSTeresa Johnson Summary.modulePath(), Reason, VI.getGUID()); 42701e32130SMehdi Amini if (!CalleeSummary) { 428d68935c5STeresa Johnson // Update with new larger threshold if this was a retry (otherwise 429cb9a82fcSTeresa Johnson // we would have already inserted with NewThreshold above). Also 430cb9a82fcSTeresa Johnson // update failure info if requested. 431cb9a82fcSTeresa Johnson if (PreviouslyVisited) { 432d68935c5STeresa Johnson ProcessedThreshold = NewThreshold; 433cb9a82fcSTeresa Johnson if (PrintImportFailures) { 434cb9a82fcSTeresa Johnson assert(FailureInfo && 435cb9a82fcSTeresa Johnson "Expected FailureInfo for previously rejected candidate"); 436cb9a82fcSTeresa Johnson FailureInfo->Reason = Reason; 437cb9a82fcSTeresa Johnson FailureInfo->Attempts++; 438cb9a82fcSTeresa Johnson FailureInfo->MaxHotness = 439cb9a82fcSTeresa Johnson std::max(FailureInfo->MaxHotness, Edge.second.getHotness()); 440cb9a82fcSTeresa Johnson } 441cb9a82fcSTeresa Johnson } else if (PrintImportFailures) { 442cb9a82fcSTeresa Johnson assert(!FailureInfo && 443cb9a82fcSTeresa Johnson "Expected no FailureInfo for newly rejected candidate"); 444cb9a82fcSTeresa Johnson FailureInfo = llvm::make_unique<FunctionImporter::ImportFailureInfo>( 445cb9a82fcSTeresa Johnson VI, Edge.second.getHotness(), Reason, 1); 446cb9a82fcSTeresa Johnson } 447d34e60caSNicola Zaghen LLVM_DEBUG( 448d34e60caSNicola Zaghen dbgs() << "ignored! No qualifying callee with summary found.\n"); 4497e88d0daSMehdi Amini continue; 4507e88d0daSMehdi Amini } 4512f0cc477SDavid Blaikie 4522f0cc477SDavid Blaikie // "Resolve" the summary 453d68935c5STeresa Johnson CalleeSummary = CalleeSummary->getBaseObject(); 454d68935c5STeresa Johnson ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 4552d28f7aaSMehdi Amini 456d9830eb7SPiotr Padlewski assert(ResolvedCalleeSummary->instCount() <= NewThreshold && 45701e32130SMehdi Amini "selectCallee() didn't honor the threshold"); 45801e32130SMehdi Amini 4591b859a23STeresa Johnson auto ExportModulePath = ResolvedCalleeSummary->modulePath(); 460d68935c5STeresa Johnson auto ILI = ImportList[ExportModulePath].insert(VI.getGUID()); 461d68935c5STeresa Johnson // We previously decided to import this GUID definition if it was already 462d68935c5STeresa Johnson // inserted in the set of imports from the exporting module. 463d68935c5STeresa Johnson bool PreviouslyImported = !ILI.second; 464d2c234a4STeresa Johnson if (!PreviouslyImported) { 465d2c234a4STeresa Johnson NumImportedFunctionsThinLink++; 466d2c234a4STeresa Johnson if (IsHotCallsite) 467d2c234a4STeresa Johnson NumImportedHotFunctionsThinLink++; 468d2c234a4STeresa Johnson if (IsCriticalCallsite) 469d2c234a4STeresa Johnson NumImportedCriticalFunctionsThinLink++; 470d2c234a4STeresa Johnson } 471974706ebSTeresa Johnson 4721b859a23STeresa Johnson // Make exports in the source module. 4731b859a23STeresa Johnson if (ExportLists) { 4741b859a23STeresa Johnson auto &ExportList = (*ExportLists)[ExportModulePath]; 4759667b91bSPeter Collingbourne ExportList.insert(VI.getGUID()); 47619f2aa78STeresa Johnson if (!PreviouslyImported) { 47719f2aa78STeresa Johnson // This is the first time this function was exported from its source 47819f2aa78STeresa Johnson // module, so mark all functions and globals it references as exported 4791b859a23STeresa Johnson // to the outside if they are defined in the same source module. 480edddca22STeresa Johnson // For efficiency, we unconditionally add all the referenced GUIDs 481edddca22STeresa Johnson // to the ExportList for this module, and will prune out any not 482edddca22STeresa Johnson // defined in the module later in a single pass. 4831b859a23STeresa Johnson for (auto &Edge : ResolvedCalleeSummary->calls()) { 4841b859a23STeresa Johnson auto CalleeGUID = Edge.first.getGUID(); 485edddca22STeresa Johnson ExportList.insert(CalleeGUID); 4861b859a23STeresa Johnson } 4871b859a23STeresa Johnson for (auto &Ref : ResolvedCalleeSummary->refs()) { 4881b859a23STeresa Johnson auto GUID = Ref.getGUID(); 489edddca22STeresa Johnson ExportList.insert(GUID); 4901b859a23STeresa Johnson } 4911b859a23STeresa Johnson } 49219f2aa78STeresa Johnson } 493d68935c5STeresa Johnson } 494d68935c5STeresa Johnson 495d68935c5STeresa Johnson auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) { 496d68935c5STeresa Johnson // Adjust the threshold for next level of imported functions. 497d68935c5STeresa Johnson // The threshold is different for hot callsites because we can then 498d68935c5STeresa Johnson // inline chains of hot calls. 499d68935c5STeresa Johnson if (IsHotCallsite) 500d68935c5STeresa Johnson return Threshold * ImportHotInstrFactor; 501d68935c5STeresa Johnson return Threshold * ImportInstrFactor; 502d68935c5STeresa Johnson }; 503d68935c5STeresa Johnson 504d68935c5STeresa Johnson const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite); 505d68935c5STeresa Johnson 506d68935c5STeresa Johnson ImportCount++; 507d2869473SPiotr Padlewski 50801e32130SMehdi Amini // Insert the newly imported function to the worklist. 5099667b91bSPeter Collingbourne Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID()); 510d450da32STeresa Johnson } 511d450da32STeresa Johnson } 512d450da32STeresa Johnson 51301e32130SMehdi Amini /// Given the list of globals defined in a module, compute the list of imports 51401e32130SMehdi Amini /// as well as the list of "exports", i.e. the list of symbols referenced from 51501e32130SMehdi Amini /// another module (that may require promotion). 51601e32130SMehdi Amini static void ComputeImportForModule( 517c851d216STeresa Johnson const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index, 518cb9a82fcSTeresa Johnson StringRef ModName, FunctionImporter::ImportMapTy &ImportList, 51956584bbfSEvgeniy Stepanov StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) { 52001e32130SMehdi Amini // Worklist contains the list of function imported in this module, for which 52101e32130SMehdi Amini // we will analyse the callees and may import further down the callgraph. 52201e32130SMehdi Amini SmallVector<EdgeInfo, 128> Worklist; 523d68935c5STeresa Johnson FunctionImporter::ImportThresholdsTy ImportThresholds; 52401e32130SMehdi Amini 52501e32130SMehdi Amini // Populate the worklist with the import for the functions in the current 52601e32130SMehdi Amini // module 52728e457bcSTeresa Johnson for (auto &GVSummary : DefinedGVSummaries) { 5287e7b13d0STeresa Johnson #ifndef NDEBUG 5297e7b13d0STeresa Johnson // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID 5307e7b13d0STeresa Johnson // so this map look up (and possibly others) can be avoided. 5317e7b13d0STeresa Johnson auto VI = Index.getValueInfo(GVSummary.first); 5327e7b13d0STeresa Johnson #endif 53356584bbfSEvgeniy Stepanov if (!Index.isGlobalValueLive(GVSummary.second)) { 5347e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n"); 5356c475a75STeresa Johnson continue; 5366c475a75STeresa Johnson } 537cfbd0892SPeter Collingbourne auto *FuncSummary = 538cfbd0892SPeter Collingbourne dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject()); 5391aafabf7SMehdi Amini if (!FuncSummary) 5401aafabf7SMehdi Amini // Skip import for global variables 5411aafabf7SMehdi Amini continue; 5427e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n"); 5432d28f7aaSMehdi Amini computeImportForFunction(*FuncSummary, Index, ImportInstrLimit, 5449b490f10SMehdi Amini DefinedGVSummaries, Worklist, ImportList, 545d68935c5STeresa Johnson ExportLists, ImportThresholds); 54601e32130SMehdi Amini } 54701e32130SMehdi Amini 548d2869473SPiotr Padlewski // Process the newly imported functions and add callees to the worklist. 54942418abaSMehdi Amini while (!Worklist.empty()) { 55001e32130SMehdi Amini auto FuncInfo = Worklist.pop_back_val(); 551475b51a7STeresa Johnson auto *Summary = std::get<0>(FuncInfo); 552475b51a7STeresa Johnson auto Threshold = std::get<1>(FuncInfo); 55342418abaSMehdi Amini 5541aafabf7SMehdi Amini computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries, 555d68935c5STeresa Johnson Worklist, ImportList, ExportLists, 556d68935c5STeresa Johnson ImportThresholds); 557c8c55170SMehdi Amini } 558cb9a82fcSTeresa Johnson 559cb9a82fcSTeresa Johnson // Print stats about functions considered but rejected for importing 560cb9a82fcSTeresa Johnson // when requested. 561cb9a82fcSTeresa Johnson if (PrintImportFailures) { 562cb9a82fcSTeresa Johnson dbgs() << "Missed imports into module " << ModName << "\n"; 563cb9a82fcSTeresa Johnson for (auto &I : ImportThresholds) { 564cb9a82fcSTeresa Johnson auto &ProcessedThreshold = std::get<0>(I.second); 565cb9a82fcSTeresa Johnson auto &CalleeSummary = std::get<1>(I.second); 566cb9a82fcSTeresa Johnson auto &FailureInfo = std::get<2>(I.second); 567cb9a82fcSTeresa Johnson if (CalleeSummary) 568cb9a82fcSTeresa Johnson continue; // We are going to import. 569cb9a82fcSTeresa Johnson assert(FailureInfo); 570cb9a82fcSTeresa Johnson FunctionSummary *FS = nullptr; 571cb9a82fcSTeresa Johnson if (!FailureInfo->VI.getSummaryList().empty()) 572cb9a82fcSTeresa Johnson FS = dyn_cast<FunctionSummary>( 573cb9a82fcSTeresa Johnson FailureInfo->VI.getSummaryList()[0]->getBaseObject()); 574cb9a82fcSTeresa Johnson dbgs() << FailureInfo->VI 575cb9a82fcSTeresa Johnson << ": Reason = " << getFailureName(FailureInfo->Reason) 576cb9a82fcSTeresa Johnson << ", Threshold = " << ProcessedThreshold 577cb9a82fcSTeresa Johnson << ", Size = " << (FS ? (int)FS->instCount() : -1) 578cb9a82fcSTeresa Johnson << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness) 579cb9a82fcSTeresa Johnson << ", Attempts = " << FailureInfo->Attempts << "\n"; 580cb9a82fcSTeresa Johnson } 581cb9a82fcSTeresa Johnson } 58242418abaSMehdi Amini } 583ffe2e4aaSMehdi Amini 58419e23874SEugene Leviant #ifndef NDEBUG 58519e23874SEugene Leviant static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, 58619e23874SEugene Leviant GlobalValue::GUID G) { 58719e23874SEugene Leviant if (const auto &VI = Index.getValueInfo(G)) { 58819e23874SEugene Leviant auto SL = VI.getSummaryList(); 58919e23874SEugene Leviant if (!SL.empty()) 59019e23874SEugene Leviant return SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind; 59119e23874SEugene Leviant } 59219e23874SEugene Leviant return false; 59319e23874SEugene Leviant } 59419e23874SEugene Leviant 59519e23874SEugene Leviant static GlobalValue::GUID getGUID(GlobalValue::GUID G) { return G; } 59619e23874SEugene Leviant 59719e23874SEugene Leviant template <class T> 5981fc0da48SBenjamin Kramer static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, 5991fc0da48SBenjamin Kramer T &Cont) { 60019e23874SEugene Leviant unsigned NumGVS = 0; 60119e23874SEugene Leviant for (auto &V : Cont) 60219e23874SEugene Leviant if (isGlobalVarSummary(Index, getGUID(V))) 60319e23874SEugene Leviant ++NumGVS; 60419e23874SEugene Leviant return NumGVS; 60519e23874SEugene Leviant } 60619e23874SEugene Leviant #endif 60719e23874SEugene Leviant 608c86af334STeresa Johnson /// Compute all the import and export for every module using the Index. 60901e32130SMehdi Amini void llvm::ComputeCrossModuleImport( 61001e32130SMehdi Amini const ModuleSummaryIndex &Index, 611c851d216STeresa Johnson const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 61201e32130SMehdi Amini StringMap<FunctionImporter::ImportMapTy> &ImportLists, 61356584bbfSEvgeniy Stepanov StringMap<FunctionImporter::ExportSetTy> &ExportLists) { 61401e32130SMehdi Amini // For each module that has function defined, compute the import/export lists. 6151aafabf7SMehdi Amini for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) { 6169b490f10SMehdi Amini auto &ImportList = ImportLists[DefinedGVSummaries.first()]; 617d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Computing import for Module '" 6181aafabf7SMehdi Amini << DefinedGVSummaries.first() << "'\n"); 619cb9a82fcSTeresa Johnson ComputeImportForModule(DefinedGVSummaries.second, Index, 620cb9a82fcSTeresa Johnson DefinedGVSummaries.first(), ImportList, 62156584bbfSEvgeniy Stepanov &ExportLists); 62201e32130SMehdi Amini } 62301e32130SMehdi Amini 624edddca22STeresa Johnson // When computing imports we added all GUIDs referenced by anything 625edddca22STeresa Johnson // imported from the module to its ExportList. Now we prune each ExportList 626edddca22STeresa Johnson // of any not defined in that module. This is more efficient than checking 627edddca22STeresa Johnson // while computing imports because some of the summary lists may be long 628edddca22STeresa Johnson // due to linkonce (comdat) copies. 629edddca22STeresa Johnson for (auto &ELI : ExportLists) { 630edddca22STeresa Johnson const auto &DefinedGVSummaries = 631edddca22STeresa Johnson ModuleToDefinedGVSummaries.lookup(ELI.first()); 632edddca22STeresa Johnson for (auto EI = ELI.second.begin(); EI != ELI.second.end();) { 633edddca22STeresa Johnson if (!DefinedGVSummaries.count(*EI)) 634edddca22STeresa Johnson EI = ELI.second.erase(EI); 635edddca22STeresa Johnson else 636edddca22STeresa Johnson ++EI; 637edddca22STeresa Johnson } 638edddca22STeresa Johnson } 639edddca22STeresa Johnson 64001e32130SMehdi Amini #ifndef NDEBUG 641d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size() 64201e32130SMehdi Amini << " modules:\n"); 64301e32130SMehdi Amini for (auto &ModuleImports : ImportLists) { 64401e32130SMehdi Amini auto ModName = ModuleImports.first(); 64501e32130SMehdi Amini auto &Exports = ExportLists[ModName]; 64619e23874SEugene Leviant unsigned NumGVS = numGlobalVarSummaries(Index, Exports); 647d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports " 64819e23874SEugene Leviant << Exports.size() - NumGVS << " functions and " << NumGVS 649d34e60caSNicola Zaghen << " vars. Imports from " << ModuleImports.second.size() 650d34e60caSNicola Zaghen << " modules.\n"); 65101e32130SMehdi Amini for (auto &Src : ModuleImports.second) { 65201e32130SMehdi Amini auto SrcModName = Src.first(); 65319e23874SEugene Leviant unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 654d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 65519e23874SEugene Leviant << " functions imported from " << SrcModName << "\n"); 656d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod 657d34e60caSNicola Zaghen << " global vars imported from " << SrcModName << "\n"); 65801e32130SMehdi Amini } 65901e32130SMehdi Amini } 66001e32130SMehdi Amini #endif 66101e32130SMehdi Amini } 66201e32130SMehdi Amini 66381bbf742STeresa Johnson #ifndef NDEBUG 66419e23874SEugene Leviant static void dumpImportListForModule(const ModuleSummaryIndex &Index, 66519e23874SEugene Leviant StringRef ModulePath, 66681bbf742STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 667d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from " 66881bbf742STeresa Johnson << ImportList.size() << " modules.\n"); 66981bbf742STeresa Johnson for (auto &Src : ImportList) { 67081bbf742STeresa Johnson auto SrcModName = Src.first(); 67119e23874SEugene Leviant unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 672d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 67319e23874SEugene Leviant << " functions imported from " << SrcModName << "\n"); 674d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod << " vars imported from " 67581bbf742STeresa Johnson << SrcModName << "\n"); 67681bbf742STeresa Johnson } 67781bbf742STeresa Johnson } 67869b2de84STeresa Johnson #endif 67981bbf742STeresa Johnson 680c86af334STeresa Johnson /// Compute all the imports for the given module in the Index. 681c86af334STeresa Johnson void llvm::ComputeCrossModuleImportForModule( 682c86af334STeresa Johnson StringRef ModulePath, const ModuleSummaryIndex &Index, 683c86af334STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 684c86af334STeresa Johnson // Collect the list of functions this module defines. 685c86af334STeresa Johnson // GUID -> Summary 686c851d216STeresa Johnson GVSummaryMapTy FunctionSummaryMap; 68728e457bcSTeresa Johnson Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap); 688c86af334STeresa Johnson 689c86af334STeresa Johnson // Compute the import list for this module. 690d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n"); 691cb9a82fcSTeresa Johnson ComputeImportForModule(FunctionSummaryMap, Index, ModulePath, ImportList); 692c86af334STeresa Johnson 693c86af334STeresa Johnson #ifndef NDEBUG 69419e23874SEugene Leviant dumpImportListForModule(Index, ModulePath, ImportList); 69581bbf742STeresa Johnson #endif 696c86af334STeresa Johnson } 69781bbf742STeresa Johnson 69881bbf742STeresa Johnson // Mark all external summaries in Index for import into the given module. 69981bbf742STeresa Johnson // Used for distributed builds using a distributed index. 70081bbf742STeresa Johnson void llvm::ComputeCrossModuleImportForModuleFromIndex( 70181bbf742STeresa Johnson StringRef ModulePath, const ModuleSummaryIndex &Index, 70281bbf742STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 70381bbf742STeresa Johnson for (auto &GlobalList : Index) { 70481bbf742STeresa Johnson // Ignore entries for undefined references. 70581bbf742STeresa Johnson if (GlobalList.second.SummaryList.empty()) 70681bbf742STeresa Johnson continue; 70781bbf742STeresa Johnson 70881bbf742STeresa Johnson auto GUID = GlobalList.first; 70981bbf742STeresa Johnson assert(GlobalList.second.SummaryList.size() == 1 && 71081bbf742STeresa Johnson "Expected individual combined index to have one summary per GUID"); 71181bbf742STeresa Johnson auto &Summary = GlobalList.second.SummaryList[0]; 71281bbf742STeresa Johnson // Skip the summaries for the importing module. These are included to 71381bbf742STeresa Johnson // e.g. record required linkage changes. 71481bbf742STeresa Johnson if (Summary->modulePath() == ModulePath) 71581bbf742STeresa Johnson continue; 716d68935c5STeresa Johnson // Add an entry to provoke importing by thinBackend. 717d68935c5STeresa Johnson ImportList[Summary->modulePath()].insert(GUID); 71881bbf742STeresa Johnson } 71981bbf742STeresa Johnson #ifndef NDEBUG 72019e23874SEugene Leviant dumpImportListForModule(Index, ModulePath, ImportList); 721c86af334STeresa Johnson #endif 722c86af334STeresa Johnson } 723c86af334STeresa Johnson 72456584bbfSEvgeniy Stepanov void llvm::computeDeadSymbols( 72556584bbfSEvgeniy Stepanov ModuleSummaryIndex &Index, 726eaf5172cSGeorge Rimar const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, 727eaf5172cSGeorge Rimar function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) { 72856584bbfSEvgeniy Stepanov assert(!Index.withGlobalValueDeadStripping()); 7296c475a75STeresa Johnson if (!ComputeDead) 73056584bbfSEvgeniy Stepanov return; 7316c475a75STeresa Johnson if (GUIDPreservedSymbols.empty()) 7326c475a75STeresa Johnson // Don't do anything when nothing is live, this is friendly with tests. 73356584bbfSEvgeniy Stepanov return; 73456584bbfSEvgeniy Stepanov unsigned LiveSymbols = 0; 7359667b91bSPeter Collingbourne SmallVector<ValueInfo, 128> Worklist; 7369667b91bSPeter Collingbourne Worklist.reserve(GUIDPreservedSymbols.size() * 2); 7379667b91bSPeter Collingbourne for (auto GUID : GUIDPreservedSymbols) { 7389667b91bSPeter Collingbourne ValueInfo VI = Index.getValueInfo(GUID); 7399667b91bSPeter Collingbourne if (!VI) 7409667b91bSPeter Collingbourne continue; 74156584bbfSEvgeniy Stepanov for (auto &S : VI.getSummaryList()) 74256584bbfSEvgeniy Stepanov S->setLive(true); 7436c475a75STeresa Johnson } 74456584bbfSEvgeniy Stepanov 7456c475a75STeresa Johnson // Add values flagged in the index as live roots to the worklist. 7467e7b13d0STeresa Johnson for (const auto &Entry : Index) { 7477e7b13d0STeresa Johnson auto VI = Index.getValueInfo(Entry); 74856584bbfSEvgeniy Stepanov for (auto &S : Entry.second.SummaryList) 74956584bbfSEvgeniy Stepanov if (S->isLive()) { 7507e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n"); 7517e7b13d0STeresa Johnson Worklist.push_back(VI); 75256584bbfSEvgeniy Stepanov ++LiveSymbols; 75356584bbfSEvgeniy Stepanov break; 7546c475a75STeresa Johnson } 7557e7b13d0STeresa Johnson } 7566c475a75STeresa Johnson 75756584bbfSEvgeniy Stepanov // Make value live and add it to the worklist if it was not live before. 75856584bbfSEvgeniy Stepanov auto visit = [&](ValueInfo VI) { 7591958083dSTeresa Johnson // FIXME: If we knew which edges were created for indirect call profiles, 7601958083dSTeresa Johnson // we could skip them here. Any that are live should be reached via 7611958083dSTeresa Johnson // other edges, e.g. reference edges. Otherwise, using a profile collected 7621958083dSTeresa Johnson // on a slightly different binary might provoke preserving, importing 7631958083dSTeresa Johnson // and ultimately promoting calls to functions not linked into this 7641958083dSTeresa Johnson // binary, which increases the binary size unnecessarily. Note that 7651958083dSTeresa Johnson // if this code changes, the importer needs to change so that edges 7661958083dSTeresa Johnson // to functions marked dead are skipped. 7671958083dSTeresa Johnson VI = updateValueInfoForIndirectCalls(Index, VI); 7681958083dSTeresa Johnson if (!VI) 7691958083dSTeresa Johnson return; 77056584bbfSEvgeniy Stepanov for (auto &S : VI.getSummaryList()) 771f625118eSTeresa Johnson if (S->isLive()) 772f625118eSTeresa Johnson return; 773eaf5172cSGeorge Rimar 774aab60006SVlad Tsyrklevich // We only keep live symbols that are known to be non-prevailing if any are 775bfdad33bSXin Tong // available_externally, linkonceodr, weakodr. Those symbols are discarded 776bfdad33bSXin Tong // later in the EliminateAvailableExternally pass and setting them to 777bfdad33bSXin Tong // not-live could break downstreams users of liveness information (PR36483) 778bfdad33bSXin Tong // or limit optimization opportunities. 779aab60006SVlad Tsyrklevich if (isPrevailing(VI.getGUID()) == PrevailingType::No) { 780bfdad33bSXin Tong bool KeepAliveLinkage = false; 781aab60006SVlad Tsyrklevich bool Interposable = false; 782aab60006SVlad Tsyrklevich for (auto &S : VI.getSummaryList()) { 783bfdad33bSXin Tong if (S->linkage() == GlobalValue::AvailableExternallyLinkage || 784bfdad33bSXin Tong S->linkage() == GlobalValue::WeakODRLinkage || 785bfdad33bSXin Tong S->linkage() == GlobalValue::LinkOnceODRLinkage) 786bfdad33bSXin Tong KeepAliveLinkage = true; 787aab60006SVlad Tsyrklevich else if (GlobalValue::isInterposableLinkage(S->linkage())) 788aab60006SVlad Tsyrklevich Interposable = true; 789aab60006SVlad Tsyrklevich } 790aab60006SVlad Tsyrklevich 791bfdad33bSXin Tong if (!KeepAliveLinkage) 792eaf5172cSGeorge Rimar return; 793eaf5172cSGeorge Rimar 794aab60006SVlad Tsyrklevich if (Interposable) 795bfdad33bSXin Tong report_fatal_error( 796bfdad33bSXin Tong "Interposable and available_externally/linkonce_odr/weak_odr symbol"); 797aab60006SVlad Tsyrklevich } 798aab60006SVlad Tsyrklevich 799f625118eSTeresa Johnson for (auto &S : VI.getSummaryList()) 80056584bbfSEvgeniy Stepanov S->setLive(true); 80156584bbfSEvgeniy Stepanov ++LiveSymbols; 80256584bbfSEvgeniy Stepanov Worklist.push_back(VI); 80356584bbfSEvgeniy Stepanov }; 80456584bbfSEvgeniy Stepanov 8056c475a75STeresa Johnson while (!Worklist.empty()) { 8069667b91bSPeter Collingbourne auto VI = Worklist.pop_back_val(); 8079667b91bSPeter Collingbourne for (auto &Summary : VI.getSummaryList()) { 808cfbd0892SPeter Collingbourne GlobalValueSummary *Base = Summary->getBaseObject(); 809eaf5172cSGeorge Rimar // Set base value live in case it is an alias. 810eaf5172cSGeorge Rimar Base->setLive(true); 811cfbd0892SPeter Collingbourne for (auto Ref : Base->refs()) 81256584bbfSEvgeniy Stepanov visit(Ref); 813cfbd0892SPeter Collingbourne if (auto *FS = dyn_cast<FunctionSummary>(Base)) 81456584bbfSEvgeniy Stepanov for (auto Call : FS->calls()) 81556584bbfSEvgeniy Stepanov visit(Call.first); 8166c475a75STeresa Johnson } 8176c475a75STeresa Johnson } 81856584bbfSEvgeniy Stepanov Index.setWithGlobalValueDeadStripping(); 81956584bbfSEvgeniy Stepanov 82056584bbfSEvgeniy Stepanov unsigned DeadSymbols = Index.size() - LiveSymbols; 821d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols 82256584bbfSEvgeniy Stepanov << " symbols Dead \n"); 82356584bbfSEvgeniy Stepanov NumDeadSymbols += DeadSymbols; 82456584bbfSEvgeniy Stepanov NumLiveSymbols += LiveSymbols; 8256c475a75STeresa Johnson } 8266c475a75STeresa Johnson 82784174c37STeresa Johnson /// Compute the set of summaries needed for a ThinLTO backend compilation of 82884174c37STeresa Johnson /// \p ModulePath. 82984174c37STeresa Johnson void llvm::gatherImportedSummariesForModule( 83084174c37STeresa Johnson StringRef ModulePath, 83184174c37STeresa Johnson const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 832cdbcbf74SMehdi Amini const FunctionImporter::ImportMapTy &ImportList, 83384174c37STeresa Johnson std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 83484174c37STeresa Johnson // Include all summaries from the importing module. 83584174c37STeresa Johnson ModuleToSummariesForIndex[ModulePath] = 83684174c37STeresa Johnson ModuleToDefinedGVSummaries.lookup(ModulePath); 83784174c37STeresa Johnson // Include summaries for imports. 83888c491ddSMehdi Amini for (auto &ILI : ImportList) { 83984174c37STeresa Johnson auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()]; 84084174c37STeresa Johnson const auto &DefinedGVSummaries = 84184174c37STeresa Johnson ModuleToDefinedGVSummaries.lookup(ILI.first()); 84284174c37STeresa Johnson for (auto &GI : ILI.second) { 843d68935c5STeresa Johnson const auto &DS = DefinedGVSummaries.find(GI); 84484174c37STeresa Johnson assert(DS != DefinedGVSummaries.end() && 84584174c37STeresa Johnson "Expected a defined summary for imported global value"); 846d68935c5STeresa Johnson SummariesForIndex[GI] = DS->second; 84784174c37STeresa Johnson } 84884174c37STeresa Johnson } 84984174c37STeresa Johnson } 85084174c37STeresa Johnson 8518570fe47STeresa Johnson /// Emit the files \p ModulePath will import from into \p OutputFilename. 852c0320ef4STeresa Johnson std::error_code llvm::EmitImportsFiles( 853c0320ef4STeresa Johnson StringRef ModulePath, StringRef OutputFilename, 854c0320ef4STeresa Johnson const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 8558570fe47STeresa Johnson std::error_code EC; 8568570fe47STeresa Johnson raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None); 8578570fe47STeresa Johnson if (EC) 8588570fe47STeresa Johnson return EC; 859c0320ef4STeresa Johnson for (auto &ILI : ModuleToSummariesForIndex) 860c0320ef4STeresa Johnson // The ModuleToSummariesForIndex map includes an entry for the current 861c0320ef4STeresa Johnson // Module (needed for writing out the index files). We don't want to 862c0320ef4STeresa Johnson // include it in the imports file, however, so filter it out. 863c0320ef4STeresa Johnson if (ILI.first != ModulePath) 864c0320ef4STeresa Johnson ImportsOS << ILI.first << "\n"; 8658570fe47STeresa Johnson return std::error_code(); 8668570fe47STeresa Johnson } 8678570fe47STeresa Johnson 8685a95c477STeresa Johnson bool llvm::convertToDeclaration(GlobalValue &GV) { 869d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName() 870d34e60caSNicola Zaghen << "\n"); 8714566c6dbSTeresa Johnson if (Function *F = dyn_cast<Function>(&GV)) { 8724566c6dbSTeresa Johnson F->deleteBody(); 8734566c6dbSTeresa Johnson F->clearMetadata(); 8747873669bSPeter Collingbourne F->setComdat(nullptr); 8754566c6dbSTeresa Johnson } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(&GV)) { 8764566c6dbSTeresa Johnson V->setInitializer(nullptr); 8774566c6dbSTeresa Johnson V->setLinkage(GlobalValue::ExternalLinkage); 8784566c6dbSTeresa Johnson V->clearMetadata(); 8797873669bSPeter Collingbourne V->setComdat(nullptr); 8805a95c477STeresa Johnson } else { 8815a95c477STeresa Johnson GlobalValue *NewGV; 8825a95c477STeresa Johnson if (GV.getValueType()->isFunctionTy()) 8835a95c477STeresa Johnson NewGV = 8845a95c477STeresa Johnson Function::Create(cast<FunctionType>(GV.getValueType()), 8855a95c477STeresa Johnson GlobalValue::ExternalLinkage, "", GV.getParent()); 8865a95c477STeresa Johnson else 8875a95c477STeresa Johnson NewGV = 8885a95c477STeresa Johnson new GlobalVariable(*GV.getParent(), GV.getValueType(), 8895a95c477STeresa Johnson /*isConstant*/ false, GlobalValue::ExternalLinkage, 8905a95c477STeresa Johnson /*init*/ nullptr, "", 8915a95c477STeresa Johnson /*insertbefore*/ nullptr, GV.getThreadLocalMode(), 8925a95c477STeresa Johnson GV.getType()->getAddressSpace()); 8935a95c477STeresa Johnson NewGV->takeName(&GV); 8945a95c477STeresa Johnson GV.replaceAllUsesWith(NewGV); 8955a95c477STeresa Johnson return false; 8965a95c477STeresa Johnson } 8975a95c477STeresa Johnson return true; 898eaf5172cSGeorge Rimar } 8994566c6dbSTeresa Johnson 900eaf5172cSGeorge Rimar /// Fixup WeakForLinker linkages in \p TheModule based on summary analysis. 901eaf5172cSGeorge Rimar void llvm::thinLTOResolveWeakForLinkerModule( 902eaf5172cSGeorge Rimar Module &TheModule, const GVSummaryMapTy &DefinedGlobals) { 90304c9a2d6STeresa Johnson auto updateLinkage = [&](GlobalValue &GV) { 90404c9a2d6STeresa Johnson // See if the global summary analysis computed a new resolved linkage. 90504c9a2d6STeresa Johnson const auto &GS = DefinedGlobals.find(GV.getGUID()); 90604c9a2d6STeresa Johnson if (GS == DefinedGlobals.end()) 90704c9a2d6STeresa Johnson return; 90804c9a2d6STeresa Johnson auto NewLinkage = GS->second->linkage(); 90904c9a2d6STeresa Johnson if (NewLinkage == GV.getLinkage()) 91004c9a2d6STeresa Johnson return; 9116a5fbe52SDavide Italiano 9126a5fbe52SDavide Italiano // Switch the linkage to weakany if asked for, e.g. we do this for 9136a5fbe52SDavide Italiano // linker redefined symbols (via --wrap or --defsym). 914f4891d29SDavide Italiano // We record that the visibility should be changed here in `addThinLTO` 915f4891d29SDavide Italiano // as we need access to the resolution vectors for each input file in 916f4891d29SDavide Italiano // order to find which symbols have been redefined. 917f4891d29SDavide Italiano // We may consider reorganizing this code and moving the linkage recording 918f4891d29SDavide Italiano // somewhere else, e.g. in thinLTOResolveWeakForLinkerInIndex. 9196a5fbe52SDavide Italiano if (NewLinkage == GlobalValue::WeakAnyLinkage) { 9206a5fbe52SDavide Italiano GV.setLinkage(NewLinkage); 9216a5fbe52SDavide Italiano return; 9226a5fbe52SDavide Italiano } 9236a5fbe52SDavide Italiano 9246a5fbe52SDavide Italiano if (!GlobalValue::isWeakForLinker(GV.getLinkage())) 9256a5fbe52SDavide Italiano return; 9264566c6dbSTeresa Johnson // Check for a non-prevailing def that has interposable linkage 9274566c6dbSTeresa Johnson // (e.g. non-odr weak or linkonce). In that case we can't simply 9284566c6dbSTeresa Johnson // convert to available_externally, since it would lose the 9294566c6dbSTeresa Johnson // interposable property and possibly get inlined. Simply drop 9304566c6dbSTeresa Johnson // the definition in that case. 9314566c6dbSTeresa Johnson if (GlobalValue::isAvailableExternallyLinkage(NewLinkage) && 9325a95c477STeresa Johnson GlobalValue::isInterposableLinkage(GV.getLinkage())) { 9335a95c477STeresa Johnson if (!convertToDeclaration(GV)) 9345a95c477STeresa Johnson // FIXME: Change this to collect replaced GVs and later erase 9355a95c477STeresa Johnson // them from the parent module once thinLTOResolveWeakForLinkerGUID is 9365a95c477STeresa Johnson // changed to enable this for aliases. 9375a95c477STeresa Johnson llvm_unreachable("Expected GV to be converted"); 9385a95c477STeresa Johnson } else { 93933ba93c2SSteven Wu // If the original symbols has global unnamed addr and linkonce_odr linkage, 94033ba93c2SSteven Wu // it should be an auto hide symbol. Add hidden visibility to the symbol to 94133ba93c2SSteven Wu // preserve the property. 94233ba93c2SSteven Wu if (GV.hasLinkOnceODRLinkage() && GV.hasGlobalUnnamedAddr() && 94333ba93c2SSteven Wu NewLinkage == GlobalValue::WeakODRLinkage) 94433ba93c2SSteven Wu GV.setVisibility(GlobalValue::HiddenVisibility); 94533ba93c2SSteven Wu 946d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() 947d34e60caSNicola Zaghen << "` from " << GV.getLinkage() << " to " << NewLinkage 948d34e60caSNicola Zaghen << "\n"); 94904c9a2d6STeresa Johnson GV.setLinkage(NewLinkage); 9504566c6dbSTeresa Johnson } 9514566c6dbSTeresa Johnson // Remove declarations from comdats, including available_externally 9526107a419STeresa Johnson // as this is a declaration for the linker, and will be dropped eventually. 9536107a419STeresa Johnson // It is illegal for comdats to contain declarations. 9546107a419STeresa Johnson auto *GO = dyn_cast_or_null<GlobalObject>(&GV); 9554566c6dbSTeresa Johnson if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) 9566107a419STeresa Johnson GO->setComdat(nullptr); 95704c9a2d6STeresa Johnson }; 95804c9a2d6STeresa Johnson 95904c9a2d6STeresa Johnson // Process functions and global now 96004c9a2d6STeresa Johnson for (auto &GV : TheModule) 96104c9a2d6STeresa Johnson updateLinkage(GV); 96204c9a2d6STeresa Johnson for (auto &GV : TheModule.globals()) 96304c9a2d6STeresa Johnson updateLinkage(GV); 96404c9a2d6STeresa Johnson for (auto &GV : TheModule.aliases()) 96504c9a2d6STeresa Johnson updateLinkage(GV); 96604c9a2d6STeresa Johnson } 96704c9a2d6STeresa Johnson 96804c9a2d6STeresa Johnson /// Run internalization on \p TheModule based on symmary analysis. 96904c9a2d6STeresa Johnson void llvm::thinLTOInternalizeModule(Module &TheModule, 97004c9a2d6STeresa Johnson const GVSummaryMapTy &DefinedGlobals) { 97104c9a2d6STeresa Johnson // Declare a callback for the internalize pass that will ask for every 97204c9a2d6STeresa Johnson // candidate GlobalValue if it can be internalized or not. 97304c9a2d6STeresa Johnson auto MustPreserveGV = [&](const GlobalValue &GV) -> bool { 97404c9a2d6STeresa Johnson // Lookup the linkage recorded in the summaries during global analysis. 975c3d677f9SPeter Collingbourne auto GS = DefinedGlobals.find(GV.getGUID()); 97604c9a2d6STeresa Johnson if (GS == DefinedGlobals.end()) { 97704c9a2d6STeresa Johnson // Must have been promoted (possibly conservatively). Find original 97804c9a2d6STeresa Johnson // name so that we can access the correct summary and see if it can 97904c9a2d6STeresa Johnson // be internalized again. 98004c9a2d6STeresa Johnson // FIXME: Eventually we should control promotion instead of promoting 98104c9a2d6STeresa Johnson // and internalizing again. 98204c9a2d6STeresa Johnson StringRef OrigName = 98304c9a2d6STeresa Johnson ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName()); 98404c9a2d6STeresa Johnson std::string OrigId = GlobalValue::getGlobalIdentifier( 98504c9a2d6STeresa Johnson OrigName, GlobalValue::InternalLinkage, 98604c9a2d6STeresa Johnson TheModule.getSourceFileName()); 987c3d677f9SPeter Collingbourne GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId)); 9887ab1f692STeresa Johnson if (GS == DefinedGlobals.end()) { 9897ab1f692STeresa Johnson // Also check the original non-promoted non-globalized name. In some 9907ab1f692STeresa Johnson // cases a preempted weak value is linked in as a local copy because 9917ab1f692STeresa Johnson // it is referenced by an alias (IRLinker::linkGlobalValueProto). 9927ab1f692STeresa Johnson // In that case, since it was originally not a local value, it was 9937ab1f692STeresa Johnson // recorded in the index using the original name. 9947ab1f692STeresa Johnson // FIXME: This may not be needed once PR27866 is fixed. 995c3d677f9SPeter Collingbourne GS = DefinedGlobals.find(GlobalValue::getGUID(OrigName)); 99604c9a2d6STeresa Johnson assert(GS != DefinedGlobals.end()); 9977ab1f692STeresa Johnson } 998c3d677f9SPeter Collingbourne } 999c3d677f9SPeter Collingbourne return !GlobalValue::isLocalLinkage(GS->second->linkage()); 100004c9a2d6STeresa Johnson }; 100104c9a2d6STeresa Johnson 100204c9a2d6STeresa Johnson // FIXME: See if we can just internalize directly here via linkage changes 100304c9a2d6STeresa Johnson // based on the index, rather than invoking internalizeModule. 1004e9ea08a0SEugene Zelenko internalizeModule(TheModule, MustPreserveGV); 100504c9a2d6STeresa Johnson } 100604c9a2d6STeresa Johnson 100781bbf742STeresa Johnson /// Make alias a clone of its aliasee. 100881bbf742STeresa Johnson static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) { 100981bbf742STeresa Johnson Function *Fn = cast<Function>(GA->getBaseObject()); 101081bbf742STeresa Johnson 101181bbf742STeresa Johnson ValueToValueMapTy VMap; 101281bbf742STeresa Johnson Function *NewFn = CloneFunction(Fn, VMap); 101381bbf742STeresa Johnson // Clone should use the original alias's linkage and name, and we ensure 101481bbf742STeresa Johnson // all uses of alias instead use the new clone (casted if necessary). 101581bbf742STeresa Johnson NewFn->setLinkage(GA->getLinkage()); 101681bbf742STeresa Johnson GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewFn, GA->getType())); 101781bbf742STeresa Johnson NewFn->takeName(GA); 101881bbf742STeresa Johnson return NewFn; 101981bbf742STeresa Johnson } 102081bbf742STeresa Johnson 1021c8c55170SMehdi Amini // Automatically import functions in Module \p DestModule based on the summaries 1022c8c55170SMehdi Amini // index. 10237f00d0a1SPeter Collingbourne Expected<bool> FunctionImporter::importFunctions( 102466043797SAdrian Prantl Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) { 1025d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Starting import for Module " 1026311fef6eSMehdi Amini << DestModule.getModuleIdentifier() << "\n"); 102719e23874SEugene Leviant unsigned ImportedCount = 0, ImportedGVCount = 0; 1028c8c55170SMehdi Amini 10296d8f817fSPeter Collingbourne IRMover Mover(DestModule); 10307e88d0daSMehdi Amini // Do the actual import of functions now, one Module at a time 103101e32130SMehdi Amini std::set<StringRef> ModuleNameOrderedList; 103201e32130SMehdi Amini for (auto &FunctionsToImportPerModule : ImportList) { 103301e32130SMehdi Amini ModuleNameOrderedList.insert(FunctionsToImportPerModule.first()); 103401e32130SMehdi Amini } 103501e32130SMehdi Amini for (auto &Name : ModuleNameOrderedList) { 10367e88d0daSMehdi Amini // Get the module for the import 103701e32130SMehdi Amini const auto &FunctionsToImportPerModule = ImportList.find(Name); 103801e32130SMehdi Amini assert(FunctionsToImportPerModule != ImportList.end()); 1039d9445c49SPeter Collingbourne Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(Name); 1040d9445c49SPeter Collingbourne if (!SrcModuleOrErr) 1041d9445c49SPeter Collingbourne return SrcModuleOrErr.takeError(); 1042d9445c49SPeter Collingbourne std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr); 10437e88d0daSMehdi Amini assert(&DestModule.getContext() == &SrcModule->getContext() && 10447e88d0daSMehdi Amini "Context mismatch"); 10457e88d0daSMehdi Amini 10466cba37ceSTeresa Johnson // If modules were created with lazy metadata loading, materialize it 10476cba37ceSTeresa Johnson // now, before linking it (otherwise this will be a noop). 10487f00d0a1SPeter Collingbourne if (Error Err = SrcModule->materializeMetadata()) 10497f00d0a1SPeter Collingbourne return std::move(Err); 1050e5a61917STeresa Johnson 105101e32130SMehdi Amini auto &ImportGUIDs = FunctionsToImportPerModule->second; 105201e32130SMehdi Amini // Find the globals to import 10536d8f817fSPeter Collingbourne SetVector<GlobalValue *> GlobalsToImport; 10541f685e01SPiotr Padlewski for (Function &F : *SrcModule) { 10551f685e01SPiotr Padlewski if (!F.hasName()) 10560beb858eSTeresa Johnson continue; 10571f685e01SPiotr Padlewski auto GUID = F.getGUID(); 10580beb858eSTeresa Johnson auto Import = ImportGUIDs.count(GUID); 1059d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing function " 1060d34e60caSNicola Zaghen << GUID << " " << F.getName() << " from " 1061aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 10620beb858eSTeresa Johnson if (Import) { 10637f00d0a1SPeter Collingbourne if (Error Err = F.materialize()) 10647f00d0a1SPeter Collingbourne return std::move(Err); 10653b776128SPiotr Padlewski if (EnableImportMetadata) { 10666deaa6afSPiotr Padlewski // Add 'thinlto_src_module' metadata for statistics and debugging. 10673b776128SPiotr Padlewski F.setMetadata( 10683b776128SPiotr Padlewski "thinlto_src_module", 1069e9ea08a0SEugene Zelenko MDNode::get(DestModule.getContext(), 1070e9ea08a0SEugene Zelenko {MDString::get(DestModule.getContext(), 10716deaa6afSPiotr Padlewski SrcModule->getSourceFileName())})); 10723b776128SPiotr Padlewski } 10731f685e01SPiotr Padlewski GlobalsToImport.insert(&F); 107401e32130SMehdi Amini } 107501e32130SMehdi Amini } 10761f685e01SPiotr Padlewski for (GlobalVariable &GV : SrcModule->globals()) { 10772d28f7aaSMehdi Amini if (!GV.hasName()) 10782d28f7aaSMehdi Amini continue; 10792d28f7aaSMehdi Amini auto GUID = GV.getGUID(); 10802d28f7aaSMehdi Amini auto Import = ImportGUIDs.count(GUID); 1081d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing global " 1082d34e60caSNicola Zaghen << GUID << " " << GV.getName() << " from " 1083aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 10842d28f7aaSMehdi Amini if (Import) { 10857f00d0a1SPeter Collingbourne if (Error Err = GV.materialize()) 10867f00d0a1SPeter Collingbourne return std::move(Err); 108719e23874SEugene Leviant ImportedGVCount += GlobalsToImport.insert(&GV); 10882d28f7aaSMehdi Amini } 10892d28f7aaSMehdi Amini } 10901f685e01SPiotr Padlewski for (GlobalAlias &GA : SrcModule->aliases()) { 10911f685e01SPiotr Padlewski if (!GA.hasName()) 109201e32130SMehdi Amini continue; 10931f685e01SPiotr Padlewski auto GUID = GA.getGUID(); 109481bbf742STeresa Johnson auto Import = ImportGUIDs.count(GUID); 1095d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing alias " 1096d34e60caSNicola Zaghen << GUID << " " << GA.getName() << " from " 1097aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 109881bbf742STeresa Johnson if (Import) { 109981bbf742STeresa Johnson if (Error Err = GA.materialize()) 110081bbf742STeresa Johnson return std::move(Err); 110181bbf742STeresa Johnson // Import alias as a copy of its aliasee. 110281bbf742STeresa Johnson GlobalObject *Base = GA.getBaseObject(); 110381bbf742STeresa Johnson if (Error Err = Base->materialize()) 110481bbf742STeresa Johnson return std::move(Err); 110581bbf742STeresa Johnson auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA); 1106d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << Base->getGUID() 110781bbf742STeresa Johnson << " " << Base->getName() << " from " 110881bbf742STeresa Johnson << SrcModule->getSourceFileName() << "\n"); 110981bbf742STeresa Johnson if (EnableImportMetadata) { 111081bbf742STeresa Johnson // Add 'thinlto_src_module' metadata for statistics and debugging. 111181bbf742STeresa Johnson Fn->setMetadata( 111281bbf742STeresa Johnson "thinlto_src_module", 111381bbf742STeresa Johnson MDNode::get(DestModule.getContext(), 111481bbf742STeresa Johnson {MDString::get(DestModule.getContext(), 111581bbf742STeresa Johnson SrcModule->getSourceFileName())})); 111601e32130SMehdi Amini } 111781bbf742STeresa Johnson GlobalsToImport.insert(Fn); 111881bbf742STeresa Johnson } 111981bbf742STeresa Johnson } 112001e32130SMehdi Amini 112119ef4fadSMehdi Amini // Upgrade debug info after we're done materializing all the globals and we 112219ef4fadSMehdi Amini // have loaded all the required metadata! 112319ef4fadSMehdi Amini UpgradeDebugInfo(*SrcModule); 112419ef4fadSMehdi Amini 11257e88d0daSMehdi Amini // Link in the specified functions. 112601e32130SMehdi Amini if (renameModuleForThinLTO(*SrcModule, Index, &GlobalsToImport)) 11278d05185aSMehdi Amini return true; 11288d05185aSMehdi Amini 1129d29478f7STeresa Johnson if (PrintImports) { 1130d29478f7STeresa Johnson for (const auto *GV : GlobalsToImport) 1131d29478f7STeresa Johnson dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName() 1132d29478f7STeresa Johnson << " from " << SrcModule->getSourceFileName() << "\n"; 1133d29478f7STeresa Johnson } 1134d29478f7STeresa Johnson 11356d8f817fSPeter Collingbourne if (Mover.move(std::move(SrcModule), GlobalsToImport.getArrayRef(), 11366d8f817fSPeter Collingbourne [](GlobalValue &, IRMover::ValueAdder) {}, 1137e6fd9ff9SPeter Collingbourne /*IsPerformingImport=*/true)) 11387e88d0daSMehdi Amini report_fatal_error("Function Import: link error"); 11397e88d0daSMehdi Amini 114001e32130SMehdi Amini ImportedCount += GlobalsToImport.size(); 11416c475a75STeresa Johnson NumImportedModules++; 11427e88d0daSMehdi Amini } 1143e5a61917STeresa Johnson 114419e23874SEugene Leviant NumImportedFunctions += (ImportedCount - ImportedGVCount); 114519e23874SEugene Leviant NumImportedGlobalVars += ImportedGVCount; 1146d29478f7STeresa Johnson 1147d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount 1148d34e60caSNicola Zaghen << " functions for Module " 1149d34e60caSNicola Zaghen << DestModule.getModuleIdentifier() << "\n"); 1150d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount 115119e23874SEugene Leviant << " global variables for Module " 1152c8c55170SMehdi Amini << DestModule.getModuleIdentifier() << "\n"); 1153c8c55170SMehdi Amini return ImportedCount; 115442418abaSMehdi Amini } 115542418abaSMehdi Amini 1156598bd2a2SPeter Collingbourne static bool doImportingForModule(Module &M) { 1157598bd2a2SPeter Collingbourne if (SummaryFile.empty()) 1158598bd2a2SPeter Collingbourne report_fatal_error("error: -function-import requires -summary-file\n"); 11596de481a3SPeter Collingbourne Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr = 11606de481a3SPeter Collingbourne getModuleSummaryIndexForFile(SummaryFile); 11616de481a3SPeter Collingbourne if (!IndexPtrOrErr) { 11626de481a3SPeter Collingbourne logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(), 11636de481a3SPeter Collingbourne "Error loading file '" + SummaryFile + "': "); 116442418abaSMehdi Amini return false; 116542418abaSMehdi Amini } 1166598bd2a2SPeter Collingbourne std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr); 116742418abaSMehdi Amini 1168c86af334STeresa Johnson // First step is collecting the import list. 1169c86af334STeresa Johnson FunctionImporter::ImportMapTy ImportList; 117081bbf742STeresa Johnson // If requested, simply import all functions in the index. This is used 117181bbf742STeresa Johnson // when testing distributed backend handling via the opt tool, when 117281bbf742STeresa Johnson // we have distributed indexes containing exactly the summaries to import. 117381bbf742STeresa Johnson if (ImportAllIndex) 117481bbf742STeresa Johnson ComputeCrossModuleImportForModuleFromIndex(M.getModuleIdentifier(), *Index, 117581bbf742STeresa Johnson ImportList); 117681bbf742STeresa Johnson else 1177c86af334STeresa Johnson ComputeCrossModuleImportForModule(M.getModuleIdentifier(), *Index, 1178c86af334STeresa Johnson ImportList); 117901e32130SMehdi Amini 11804fef68cbSTeresa Johnson // Conservatively mark all internal values as promoted. This interface is 11814fef68cbSTeresa Johnson // only used when doing importing via the function importing pass. The pass 11824fef68cbSTeresa Johnson // is only enabled when testing importing via the 'opt' tool, which does 11834fef68cbSTeresa Johnson // not do the ThinLink that would normally determine what values to promote. 11844fef68cbSTeresa Johnson for (auto &I : *Index) { 11859667b91bSPeter Collingbourne for (auto &S : I.second.SummaryList) { 11864fef68cbSTeresa Johnson if (GlobalValue::isLocalLinkage(S->linkage())) 11874fef68cbSTeresa Johnson S->setLinkage(GlobalValue::ExternalLinkage); 11884fef68cbSTeresa Johnson } 11894fef68cbSTeresa Johnson } 11904fef68cbSTeresa Johnson 119101e32130SMehdi Amini // Next we need to promote to global scope and rename any local values that 11921b00f2d9STeresa Johnson // are potentially exported to other modules. 119301e32130SMehdi Amini if (renameModuleForThinLTO(M, *Index, nullptr)) { 11941b00f2d9STeresa Johnson errs() << "Error renaming module\n"; 11951b00f2d9STeresa Johnson return false; 11961b00f2d9STeresa Johnson } 11971b00f2d9STeresa Johnson 119842418abaSMehdi Amini // Perform the import now. 1199d16c8065SMehdi Amini auto ModuleLoader = [&M](StringRef Identifier) { 1200d16c8065SMehdi Amini return loadFile(Identifier, M.getContext()); 1201d16c8065SMehdi Amini }; 12029d2bfc48SRafael Espindola FunctionImporter Importer(*Index, ModuleLoader); 120337e24591SPeter Collingbourne Expected<bool> Result = Importer.importFunctions(M, ImportList); 12047f00d0a1SPeter Collingbourne 12057f00d0a1SPeter Collingbourne // FIXME: Probably need to propagate Errors through the pass manager. 12067f00d0a1SPeter Collingbourne if (!Result) { 12077f00d0a1SPeter Collingbourne logAllUnhandledErrors(Result.takeError(), errs(), 12087f00d0a1SPeter Collingbourne "Error importing module: "); 12097f00d0a1SPeter Collingbourne return false; 12107f00d0a1SPeter Collingbourne } 12117f00d0a1SPeter Collingbourne 12127f00d0a1SPeter Collingbourne return *Result; 121321241571STeresa Johnson } 121421241571STeresa Johnson 121521241571STeresa Johnson namespace { 1216e9ea08a0SEugene Zelenko 121721241571STeresa Johnson /// Pass that performs cross-module function import provided a summary file. 121821241571STeresa Johnson class FunctionImportLegacyPass : public ModulePass { 121921241571STeresa Johnson public: 122021241571STeresa Johnson /// Pass identification, replacement for typeid 122121241571STeresa Johnson static char ID; 122221241571STeresa Johnson 1223e9ea08a0SEugene Zelenko explicit FunctionImportLegacyPass() : ModulePass(ID) {} 1224e9ea08a0SEugene Zelenko 122521241571STeresa Johnson /// Specify pass name for debug output 1226117296c0SMehdi Amini StringRef getPassName() const override { return "Function Importing"; } 122721241571STeresa Johnson 122821241571STeresa Johnson bool runOnModule(Module &M) override { 122921241571STeresa Johnson if (skipModule(M)) 123021241571STeresa Johnson return false; 123121241571STeresa Johnson 1232598bd2a2SPeter Collingbourne return doImportingForModule(M); 123342418abaSMehdi Amini } 123442418abaSMehdi Amini }; 1235e9ea08a0SEugene Zelenko 1236e9ea08a0SEugene Zelenko } // end anonymous namespace 123742418abaSMehdi Amini 123821241571STeresa Johnson PreservedAnalyses FunctionImportPass::run(Module &M, 1239fd03ac6aSSean Silva ModuleAnalysisManager &AM) { 1240598bd2a2SPeter Collingbourne if (!doImportingForModule(M)) 124121241571STeresa Johnson return PreservedAnalyses::all(); 124221241571STeresa Johnson 124321241571STeresa Johnson return PreservedAnalyses::none(); 124421241571STeresa Johnson } 124521241571STeresa Johnson 124621241571STeresa Johnson char FunctionImportLegacyPass::ID = 0; 124721241571STeresa Johnson INITIALIZE_PASS(FunctionImportLegacyPass, "function-import", 124842418abaSMehdi Amini "Summary Based Function Import", false, false) 124942418abaSMehdi Amini 125042418abaSMehdi Amini namespace llvm { 1251e9ea08a0SEugene Zelenko 1252598bd2a2SPeter Collingbourne Pass *createFunctionImportPass() { 1253598bd2a2SPeter Collingbourne return new FunctionImportLegacyPass(); 12545fcbdb71STeresa Johnson } 1255e9ea08a0SEugene Zelenko 1256e9ea08a0SEugene Zelenko } // end namespace llvm 1257