142418abaSMehdi Amini //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===// 242418abaSMehdi Amini // 342418abaSMehdi Amini // The LLVM Compiler Infrastructure 442418abaSMehdi Amini // 542418abaSMehdi Amini // This file is distributed under the University of Illinois Open Source 642418abaSMehdi Amini // License. See LICENSE.TXT for details. 742418abaSMehdi Amini // 842418abaSMehdi Amini //===----------------------------------------------------------------------===// 942418abaSMehdi Amini // 1042418abaSMehdi Amini // This file implements Function import based on summaries. 1142418abaSMehdi Amini // 1242418abaSMehdi Amini //===----------------------------------------------------------------------===// 1342418abaSMehdi Amini 1442418abaSMehdi Amini #include "llvm/Transforms/IPO/FunctionImport.h" 15e9ea08a0SEugene Zelenko #include "llvm/ADT/ArrayRef.h" 16e9ea08a0SEugene Zelenko #include "llvm/ADT/STLExtras.h" 17e9ea08a0SEugene Zelenko #include "llvm/ADT/SetVector.h" 1801e32130SMehdi Amini #include "llvm/ADT/SmallVector.h" 19d29478f7STeresa Johnson #include "llvm/ADT/Statistic.h" 20e9ea08a0SEugene Zelenko #include "llvm/ADT/StringMap.h" 210efed325SSimon Pilgrim #include "llvm/ADT/StringRef.h" 22b040fcc6SCharles Saternos #include "llvm/ADT/StringSet.h" 23c15d60b7SPeter Collingbourne #include "llvm/Bitcode/BitcodeReader.h" 2442418abaSMehdi Amini #include "llvm/IR/AutoUpgrade.h" 2581bbf742STeresa Johnson #include "llvm/IR/Constants.h" 26e9ea08a0SEugene Zelenko #include "llvm/IR/Function.h" 27e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalAlias.h" 28e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalObject.h" 29e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalValue.h" 30e9ea08a0SEugene Zelenko #include "llvm/IR/GlobalVariable.h" 31e9ea08a0SEugene Zelenko #include "llvm/IR/Metadata.h" 3242418abaSMehdi Amini #include "llvm/IR/Module.h" 33e9ea08a0SEugene Zelenko #include "llvm/IR/ModuleSummaryIndex.h" 3442418abaSMehdi Amini #include "llvm/IRReader/IRReader.h" 35e9ea08a0SEugene Zelenko #include "llvm/Linker/IRMover.h" 36e9ea08a0SEugene Zelenko #include "llvm/Object/ModuleSymbolTable.h" 37e9ea08a0SEugene Zelenko #include "llvm/Object/SymbolicFile.h" 38e9ea08a0SEugene Zelenko #include "llvm/Pass.h" 39e9ea08a0SEugene Zelenko #include "llvm/Support/Casting.h" 4042418abaSMehdi Amini #include "llvm/Support/CommandLine.h" 4142418abaSMehdi Amini #include "llvm/Support/Debug.h" 42e9ea08a0SEugene Zelenko #include "llvm/Support/Error.h" 43e9ea08a0SEugene Zelenko #include "llvm/Support/ErrorHandling.h" 44e9ea08a0SEugene Zelenko #include "llvm/Support/FileSystem.h" 4542418abaSMehdi Amini #include "llvm/Support/SourceMgr.h" 46e9ea08a0SEugene Zelenko #include "llvm/Support/raw_ostream.h" 4704c9a2d6STeresa Johnson #include "llvm/Transforms/IPO/Internalize.h" 4881bbf742STeresa Johnson #include "llvm/Transforms/Utils/Cloning.h" 49488a800aSTeresa Johnson #include "llvm/Transforms/Utils/FunctionImportUtils.h" 5081bbf742STeresa Johnson #include "llvm/Transforms/Utils/ValueMapper.h" 51e9ea08a0SEugene Zelenko #include <cassert> 52e9ea08a0SEugene Zelenko #include <memory> 53e9ea08a0SEugene Zelenko #include <set> 54e9ea08a0SEugene Zelenko #include <string> 55e9ea08a0SEugene Zelenko #include <system_error> 56e9ea08a0SEugene Zelenko #include <tuple> 57e9ea08a0SEugene Zelenko #include <utility> 587e88d0daSMehdi Amini 5942418abaSMehdi Amini using namespace llvm; 6042418abaSMehdi Amini 61e9ea08a0SEugene Zelenko #define DEBUG_TYPE "function-import" 62e9ea08a0SEugene Zelenko 636c475a75STeresa Johnson STATISTIC(NumImportedFunctions, "Number of functions imported"); 6419e23874SEugene Leviant STATISTIC(NumImportedGlobalVars, "Number of global variables imported"); 656c475a75STeresa Johnson STATISTIC(NumImportedModules, "Number of modules imported from"); 666c475a75STeresa Johnson STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index"); 676c475a75STeresa Johnson STATISTIC(NumLiveSymbols, "Number of live symbols in index"); 68d29478f7STeresa Johnson 6939303619STeresa Johnson /// Limit on instruction count of imported functions. 7039303619STeresa Johnson static cl::opt<unsigned> ImportInstrLimit( 7139303619STeresa Johnson "import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), 7239303619STeresa Johnson cl::desc("Only import functions with less than N instructions")); 7339303619STeresa Johnson 74974706ebSTeresa Johnson static cl::opt<int> ImportCutoff( 75974706ebSTeresa Johnson "import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), 76974706ebSTeresa Johnson cl::desc("Only import first N functions if N>=0 (default -1)")); 77974706ebSTeresa Johnson 7840641748SMehdi Amini static cl::opt<float> 7940641748SMehdi Amini ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), 8040641748SMehdi Amini cl::Hidden, cl::value_desc("x"), 8140641748SMehdi Amini cl::desc("As we import functions, multiply the " 8240641748SMehdi Amini "`import-instr-limit` threshold by this factor " 8340641748SMehdi Amini "before processing newly imported functions")); 84ba72b95fSPiotr Padlewski 85d2869473SPiotr Padlewski static cl::opt<float> ImportHotInstrFactor( 86d2869473SPiotr Padlewski "import-hot-evolution-factor", cl::init(1.0), cl::Hidden, 87d2869473SPiotr Padlewski cl::value_desc("x"), 88d2869473SPiotr Padlewski cl::desc("As we import functions called from hot callsite, multiply the " 89d2869473SPiotr Padlewski "`import-instr-limit` threshold by this factor " 90d2869473SPiotr Padlewski "before processing newly imported functions")); 91d2869473SPiotr Padlewski 92d9830eb7SPiotr Padlewski static cl::opt<float> ImportHotMultiplier( 938260d665SDehao Chen "import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), 94ba72b95fSPiotr Padlewski cl::desc("Multiply the `import-instr-limit` threshold for hot callsites")); 95ba72b95fSPiotr Padlewski 9664c46574SDehao Chen static cl::opt<float> ImportCriticalMultiplier( 9764c46574SDehao Chen "import-critical-multiplier", cl::init(100.0), cl::Hidden, 9864c46574SDehao Chen cl::value_desc("x"), 9964c46574SDehao Chen cl::desc( 10064c46574SDehao Chen "Multiply the `import-instr-limit` threshold for critical callsites")); 10164c46574SDehao Chen 102ba72b95fSPiotr Padlewski // FIXME: This multiplier was not really tuned up. 103ba72b95fSPiotr Padlewski static cl::opt<float> ImportColdMultiplier( 104ba72b95fSPiotr Padlewski "import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), 105ba72b95fSPiotr Padlewski cl::desc("Multiply the `import-instr-limit` threshold for cold callsites")); 10640641748SMehdi Amini 107d29478f7STeresa Johnson static cl::opt<bool> PrintImports("print-imports", cl::init(false), cl::Hidden, 108d29478f7STeresa Johnson cl::desc("Print imported functions")); 109d29478f7STeresa Johnson 110cb9a82fcSTeresa Johnson static cl::opt<bool> PrintImportFailures( 111cb9a82fcSTeresa Johnson "print-import-failures", cl::init(false), cl::Hidden, 112cb9a82fcSTeresa Johnson cl::desc("Print information for functions rejected for importing")); 113cb9a82fcSTeresa Johnson 1146c475a75STeresa Johnson static cl::opt<bool> ComputeDead("compute-dead", cl::init(true), cl::Hidden, 1156c475a75STeresa Johnson cl::desc("Compute dead symbols")); 1166c475a75STeresa Johnson 1173b776128SPiotr Padlewski static cl::opt<bool> EnableImportMetadata( 1183b776128SPiotr Padlewski "enable-import-metadata", cl::init( 1193b776128SPiotr Padlewski #if !defined(NDEBUG) 1203b776128SPiotr Padlewski true /*Enabled with asserts.*/ 1213b776128SPiotr Padlewski #else 1223b776128SPiotr Padlewski false 1233b776128SPiotr Padlewski #endif 1243b776128SPiotr Padlewski ), 1253b776128SPiotr Padlewski cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module'")); 1263b776128SPiotr Padlewski 127e9ea08a0SEugene Zelenko /// Summary file to use for function importing when using -function-import from 128e9ea08a0SEugene Zelenko /// the command line. 129e9ea08a0SEugene Zelenko static cl::opt<std::string> 130e9ea08a0SEugene Zelenko SummaryFile("summary-file", 131e9ea08a0SEugene Zelenko cl::desc("The summary file to use for function importing.")); 132e9ea08a0SEugene Zelenko 13381bbf742STeresa Johnson /// Used when testing importing from distributed indexes via opt 13481bbf742STeresa Johnson // -function-import. 13581bbf742STeresa Johnson static cl::opt<bool> 13681bbf742STeresa Johnson ImportAllIndex("import-all-index", 13781bbf742STeresa Johnson cl::desc("Import all external functions in index.")); 13881bbf742STeresa Johnson 13942418abaSMehdi Amini // Load lazily a module from \p FileName in \p Context. 14042418abaSMehdi Amini static std::unique_ptr<Module> loadFile(const std::string &FileName, 14142418abaSMehdi Amini LLVMContext &Context) { 14242418abaSMehdi Amini SMDiagnostic Err; 143d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n"); 1446cba37ceSTeresa Johnson // Metadata isn't loaded until functions are imported, to minimize 1456cba37ceSTeresa Johnson // the memory overhead. 146a1080ee6STeresa Johnson std::unique_ptr<Module> Result = 147a1080ee6STeresa Johnson getLazyIRFileModule(FileName, Err, Context, 148a1080ee6STeresa Johnson /* ShouldLazyLoadMetadata = */ true); 14942418abaSMehdi Amini if (!Result) { 15042418abaSMehdi Amini Err.print("function-import", errs()); 151d7ad221cSMehdi Amini report_fatal_error("Abort"); 15242418abaSMehdi Amini } 15342418abaSMehdi Amini 15442418abaSMehdi Amini return Result; 15542418abaSMehdi Amini } 15642418abaSMehdi Amini 15701e32130SMehdi Amini /// Given a list of possible callee implementation for a call site, select one 15801e32130SMehdi Amini /// that fits the \p Threshold. 15901e32130SMehdi Amini /// 16001e32130SMehdi Amini /// FIXME: select "best" instead of first that fits. But what is "best"? 16101e32130SMehdi Amini /// - The smallest: more likely to be inlined. 16201e32130SMehdi Amini /// - The one with the least outgoing edges (already well optimized). 16301e32130SMehdi Amini /// - One from a module already being imported from in order to reduce the 16401e32130SMehdi Amini /// number of source modules parsed/linked. 16501e32130SMehdi Amini /// - One that has PGO data attached. 16601e32130SMehdi Amini /// - [insert you fancy metric here] 1672d28f7aaSMehdi Amini static const GlobalValueSummary * 168b4e1e829SMehdi Amini selectCallee(const ModuleSummaryIndex &Index, 1699667b91bSPeter Collingbourne ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList, 170cb9a82fcSTeresa Johnson unsigned Threshold, StringRef CallerModulePath, 171cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason &Reason, 172cb9a82fcSTeresa Johnson GlobalValue::GUID GUID) { 173cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::None; 17401e32130SMehdi Amini auto It = llvm::find_if( 17528e457bcSTeresa Johnson CalleeSummaryList, 17628e457bcSTeresa Johnson [&](const std::unique_ptr<GlobalValueSummary> &SummaryPtr) { 17728e457bcSTeresa Johnson auto *GVSummary = SummaryPtr.get(); 178cb9a82fcSTeresa Johnson if (!Index.isGlobalValueLive(GVSummary)) { 179cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::NotLive; 180eaf5172cSGeorge Rimar return false; 181cb9a82fcSTeresa Johnson } 182eaf5172cSGeorge Rimar 18373305f82STeresa Johnson // For SamplePGO, in computeImportForFunction the OriginalId 18473305f82STeresa Johnson // may have been used to locate the callee summary list (See 18573305f82STeresa Johnson // comment there). 18673305f82STeresa Johnson // The mapping from OriginalId to GUID may return a GUID 18773305f82STeresa Johnson // that corresponds to a static variable. Filter it out here. 18873305f82STeresa Johnson // This can happen when 18973305f82STeresa Johnson // 1) There is a call to a library function which is not defined 19073305f82STeresa Johnson // in the index. 19173305f82STeresa Johnson // 2) There is a static variable with the OriginalGUID identical 19273305f82STeresa Johnson // to the GUID of the library function in 1); 19373305f82STeresa Johnson // When this happens, the logic for SamplePGO kicks in and 19473305f82STeresa Johnson // the static variable in 2) will be found, which needs to be 19573305f82STeresa Johnson // filtered out. 196cb9a82fcSTeresa Johnson if (GVSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind) { 197cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::GlobalVar; 19873305f82STeresa Johnson return false; 199cb9a82fcSTeresa Johnson } 200cb9a82fcSTeresa Johnson if (GlobalValue::isInterposableLinkage(GVSummary->linkage())) { 201cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::InterposableLinkage; 2025b85d8d6SMehdi Amini // There is no point in importing these, we can't inline them 20301e32130SMehdi Amini return false; 204cb9a82fcSTeresa Johnson } 2052c719cc1SMehdi Amini 20681bbf742STeresa Johnson auto *Summary = cast<FunctionSummary>(GVSummary->getBaseObject()); 2077e88d0daSMehdi Amini 20883aaf358STeresa Johnson // If this is a local function, make sure we import the copy 20983aaf358STeresa Johnson // in the caller's module. The only time a local function can 21083aaf358STeresa Johnson // share an entry in the index is if there is a local with the same name 21183aaf358STeresa Johnson // in another module that had the same source file name (in a different 21283aaf358STeresa Johnson // directory), where each was compiled in their own directory so there 21383aaf358STeresa Johnson // was not distinguishing path. 21483aaf358STeresa Johnson // However, do the import from another module if there is only one 21583aaf358STeresa Johnson // entry in the list - in that case this must be a reference due 21683aaf358STeresa Johnson // to indirect call profile data, since a function pointer can point to 21783aaf358STeresa Johnson // a local in another module. 21883aaf358STeresa Johnson if (GlobalValue::isLocalLinkage(Summary->linkage()) && 21983aaf358STeresa Johnson CalleeSummaryList.size() > 1 && 220cb9a82fcSTeresa Johnson Summary->modulePath() != CallerModulePath) { 221cb9a82fcSTeresa Johnson Reason = 222cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason::LocalLinkageNotInModule; 22383aaf358STeresa Johnson return false; 224cb9a82fcSTeresa Johnson } 22583aaf358STeresa Johnson 226cb9a82fcSTeresa Johnson if (Summary->instCount() > Threshold) { 227cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::TooLarge; 228f9dc3deaSTeresa Johnson return false; 229cb9a82fcSTeresa Johnson } 230f9dc3deaSTeresa Johnson 231cb9a82fcSTeresa Johnson if (Summary->notEligibleToImport()) { 232cb9a82fcSTeresa Johnson Reason = FunctionImporter::ImportFailureReason::NotEligible; 233b4e1e829SMehdi Amini return false; 234cb9a82fcSTeresa Johnson } 235b4e1e829SMehdi Amini 23601e32130SMehdi Amini return true; 23701e32130SMehdi Amini }); 23828e457bcSTeresa Johnson if (It == CalleeSummaryList.end()) 23901e32130SMehdi Amini return nullptr; 2407e88d0daSMehdi Amini 241f9dc3deaSTeresa Johnson return cast<GlobalValueSummary>(It->get()); 242434e9561SRafael Espindola } 2437e88d0daSMehdi Amini 244e9ea08a0SEugene Zelenko namespace { 245e9ea08a0SEugene Zelenko 246475b51a7STeresa Johnson using EdgeInfo = std::tuple<const FunctionSummary *, unsigned /* Threshold */, 247475b51a7STeresa Johnson GlobalValue::GUID>; 24801e32130SMehdi Amini 249e9ea08a0SEugene Zelenko } // anonymous namespace 250e9ea08a0SEugene Zelenko 2511958083dSTeresa Johnson static ValueInfo 2521958083dSTeresa Johnson updateValueInfoForIndirectCalls(const ModuleSummaryIndex &Index, ValueInfo VI) { 2531958083dSTeresa Johnson if (!VI.getSummaryList().empty()) 2541958083dSTeresa Johnson return VI; 2551958083dSTeresa Johnson // For SamplePGO, the indirect call targets for local functions will 2561958083dSTeresa Johnson // have its original name annotated in profile. We try to find the 2571958083dSTeresa Johnson // corresponding PGOFuncName as the GUID. 2581958083dSTeresa Johnson // FIXME: Consider updating the edges in the graph after building 2591958083dSTeresa Johnson // it, rather than needing to perform this mapping on each walk. 2601958083dSTeresa Johnson auto GUID = Index.getGUIDFromOriginalID(VI.getGUID()); 2611958083dSTeresa Johnson if (GUID == 0) 26228d8a49fSEugene Leviant return ValueInfo(); 2631958083dSTeresa Johnson return Index.getValueInfo(GUID); 2641958083dSTeresa Johnson } 2651958083dSTeresa Johnson 26619e23874SEugene Leviant static void computeImportForReferencedGlobals( 26719e23874SEugene Leviant const FunctionSummary &Summary, const GVSummaryMapTy &DefinedGVSummaries, 26819e23874SEugene Leviant FunctionImporter::ImportMapTy &ImportList, 26919e23874SEugene Leviant StringMap<FunctionImporter::ExportSetTy> *ExportLists) { 27019e23874SEugene Leviant for (auto &VI : Summary.refs()) { 27119e23874SEugene Leviant if (DefinedGVSummaries.count(VI.getGUID())) { 272d34e60caSNicola Zaghen LLVM_DEBUG( 273d34e60caSNicola Zaghen dbgs() << "Ref ignored! Target already in destination module.\n"); 27419e23874SEugene Leviant continue; 27519e23874SEugene Leviant } 27619e23874SEugene Leviant 2777e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << " ref -> " << VI << "\n"); 27819e23874SEugene Leviant 27919e23874SEugene Leviant for (auto &RefSummary : VI.getSummaryList()) 28019e23874SEugene Leviant if (RefSummary->getSummaryKind() == GlobalValueSummary::GlobalVarKind && 281*eddf6b5dSEugene Leviant !RefSummary->notEligibleToImport() && 28219e23874SEugene Leviant !GlobalValue::isInterposableLinkage(RefSummary->linkage()) && 28319e23874SEugene Leviant RefSummary->refs().empty()) { 284d68935c5STeresa Johnson ImportList[RefSummary->modulePath()].insert(VI.getGUID()); 28519e23874SEugene Leviant if (ExportLists) 28619e23874SEugene Leviant (*ExportLists)[RefSummary->modulePath()].insert(VI.getGUID()); 28719e23874SEugene Leviant break; 28819e23874SEugene Leviant } 28919e23874SEugene Leviant } 29019e23874SEugene Leviant } 29119e23874SEugene Leviant 292cb9a82fcSTeresa Johnson static const char * 293cb9a82fcSTeresa Johnson getFailureName(FunctionImporter::ImportFailureReason Reason) { 294cb9a82fcSTeresa Johnson switch (Reason) { 295cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::None: 296cb9a82fcSTeresa Johnson return "None"; 297cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::GlobalVar: 298cb9a82fcSTeresa Johnson return "GlobalVar"; 299cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::NotLive: 300cb9a82fcSTeresa Johnson return "NotLive"; 301cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::TooLarge: 302cb9a82fcSTeresa Johnson return "TooLarge"; 303cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::InterposableLinkage: 304cb9a82fcSTeresa Johnson return "InterposableLinkage"; 305cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule: 306cb9a82fcSTeresa Johnson return "LocalLinkageNotInModule"; 307cb9a82fcSTeresa Johnson case FunctionImporter::ImportFailureReason::NotEligible: 308cb9a82fcSTeresa Johnson return "NotEligible"; 309cb9a82fcSTeresa Johnson } 310cb9a82fcSTeresa Johnson llvm_unreachable("invalid reason"); 311cb9a82fcSTeresa Johnson } 312cb9a82fcSTeresa Johnson 31301e32130SMehdi Amini /// Compute the list of functions to import for a given caller. Mark these 31401e32130SMehdi Amini /// imported functions and the symbols they reference in their source module as 31501e32130SMehdi Amini /// exported from their source module. 31601e32130SMehdi Amini static void computeImportForFunction( 3173255eec1STeresa Johnson const FunctionSummary &Summary, const ModuleSummaryIndex &Index, 318d9830eb7SPiotr Padlewski const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, 31901e32130SMehdi Amini SmallVectorImpl<EdgeInfo> &Worklist, 3209b490f10SMehdi Amini FunctionImporter::ImportMapTy &ImportList, 321d68935c5STeresa Johnson StringMap<FunctionImporter::ExportSetTy> *ExportLists, 322d68935c5STeresa Johnson FunctionImporter::ImportThresholdsTy &ImportThresholds) { 32319e23874SEugene Leviant computeImportForReferencedGlobals(Summary, DefinedGVSummaries, ImportList, 32419e23874SEugene Leviant ExportLists); 325974706ebSTeresa Johnson static int ImportCount = 0; 32601e32130SMehdi Amini for (auto &Edge : Summary.calls()) { 3279667b91bSPeter Collingbourne ValueInfo VI = Edge.first; 3287e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold 3297e7b13d0STeresa Johnson << "\n"); 33001e32130SMehdi Amini 331974706ebSTeresa Johnson if (ImportCutoff >= 0 && ImportCount >= ImportCutoff) { 332d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ignored! import-cutoff value of " << ImportCutoff 333974706ebSTeresa Johnson << " reached.\n"); 334974706ebSTeresa Johnson continue; 335974706ebSTeresa Johnson } 336974706ebSTeresa Johnson 3371958083dSTeresa Johnson VI = updateValueInfoForIndirectCalls(Index, VI); 3389667b91bSPeter Collingbourne if (!VI) 3399667b91bSPeter Collingbourne continue; 3404a435e08SDehao Chen 3419667b91bSPeter Collingbourne if (DefinedGVSummaries.count(VI.getGUID())) { 342d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n"); 3437e88d0daSMehdi Amini continue; 344d450da32STeresa Johnson } 34540641748SMehdi Amini 346ba72b95fSPiotr Padlewski auto GetBonusMultiplier = [](CalleeInfo::HotnessType Hotness) -> float { 347ba72b95fSPiotr Padlewski if (Hotness == CalleeInfo::HotnessType::Hot) 348ba72b95fSPiotr Padlewski return ImportHotMultiplier; 349ba72b95fSPiotr Padlewski if (Hotness == CalleeInfo::HotnessType::Cold) 350ba72b95fSPiotr Padlewski return ImportColdMultiplier; 35164c46574SDehao Chen if (Hotness == CalleeInfo::HotnessType::Critical) 35264c46574SDehao Chen return ImportCriticalMultiplier; 353ba72b95fSPiotr Padlewski return 1.0; 354ba72b95fSPiotr Padlewski }; 355ba72b95fSPiotr Padlewski 356d9830eb7SPiotr Padlewski const auto NewThreshold = 357c73cec84SEaswaran Raman Threshold * GetBonusMultiplier(Edge.second.getHotness()); 358d2869473SPiotr Padlewski 359cb9a82fcSTeresa Johnson auto IT = ImportThresholds.insert(std::make_pair( 360cb9a82fcSTeresa Johnson VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr))); 361d68935c5STeresa Johnson bool PreviouslyVisited = !IT.second; 362cb9a82fcSTeresa Johnson auto &ProcessedThreshold = std::get<0>(IT.first->second); 363cb9a82fcSTeresa Johnson auto &CalleeSummary = std::get<1>(IT.first->second); 364cb9a82fcSTeresa Johnson auto &FailureInfo = std::get<2>(IT.first->second); 365d68935c5STeresa Johnson 366d68935c5STeresa Johnson const FunctionSummary *ResolvedCalleeSummary = nullptr; 367d68935c5STeresa Johnson if (CalleeSummary) { 368d68935c5STeresa Johnson assert(PreviouslyVisited); 369d68935c5STeresa Johnson // Since the traversal of the call graph is DFS, we can revisit a function 370d68935c5STeresa Johnson // a second time with a higher threshold. In this case, it is added back 371d68935c5STeresa Johnson // to the worklist with the new threshold (so that its own callee chains 372d68935c5STeresa Johnson // can be considered with the higher threshold). 373d68935c5STeresa Johnson if (NewThreshold <= ProcessedThreshold) { 374d68935c5STeresa Johnson LLVM_DEBUG( 375d68935c5STeresa Johnson dbgs() << "ignored! Target was already imported with Threshold " 376d68935c5STeresa Johnson << ProcessedThreshold << "\n"); 377d68935c5STeresa Johnson continue; 378d68935c5STeresa Johnson } 379d68935c5STeresa Johnson // Update with new larger threshold. 380d68935c5STeresa Johnson ProcessedThreshold = NewThreshold; 381d68935c5STeresa Johnson ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 382d68935c5STeresa Johnson } else { 383d68935c5STeresa Johnson // If we already rejected importing a callee at the same or higher 384d68935c5STeresa Johnson // threshold, don't waste time calling selectCallee. 385d68935c5STeresa Johnson if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) { 386d68935c5STeresa Johnson LLVM_DEBUG( 387d68935c5STeresa Johnson dbgs() << "ignored! Target was already rejected with Threshold " 388d68935c5STeresa Johnson << ProcessedThreshold << "\n"); 389cb9a82fcSTeresa Johnson if (PrintImportFailures) { 390cb9a82fcSTeresa Johnson assert(FailureInfo && 391cb9a82fcSTeresa Johnson "Expected FailureInfo for previously rejected candidate"); 392cb9a82fcSTeresa Johnson FailureInfo->Attempts++; 393cb9a82fcSTeresa Johnson } 394d68935c5STeresa Johnson continue; 395d68935c5STeresa Johnson } 396d68935c5STeresa Johnson 397cb9a82fcSTeresa Johnson FunctionImporter::ImportFailureReason Reason; 398d68935c5STeresa Johnson CalleeSummary = selectCallee(Index, VI.getSummaryList(), NewThreshold, 399cb9a82fcSTeresa Johnson Summary.modulePath(), Reason, VI.getGUID()); 40001e32130SMehdi Amini if (!CalleeSummary) { 401d68935c5STeresa Johnson // Update with new larger threshold if this was a retry (otherwise 402cb9a82fcSTeresa Johnson // we would have already inserted with NewThreshold above). Also 403cb9a82fcSTeresa Johnson // update failure info if requested. 404cb9a82fcSTeresa Johnson if (PreviouslyVisited) { 405d68935c5STeresa Johnson ProcessedThreshold = NewThreshold; 406cb9a82fcSTeresa Johnson if (PrintImportFailures) { 407cb9a82fcSTeresa Johnson assert(FailureInfo && 408cb9a82fcSTeresa Johnson "Expected FailureInfo for previously rejected candidate"); 409cb9a82fcSTeresa Johnson FailureInfo->Reason = Reason; 410cb9a82fcSTeresa Johnson FailureInfo->Attempts++; 411cb9a82fcSTeresa Johnson FailureInfo->MaxHotness = 412cb9a82fcSTeresa Johnson std::max(FailureInfo->MaxHotness, Edge.second.getHotness()); 413cb9a82fcSTeresa Johnson } 414cb9a82fcSTeresa Johnson } else if (PrintImportFailures) { 415cb9a82fcSTeresa Johnson assert(!FailureInfo && 416cb9a82fcSTeresa Johnson "Expected no FailureInfo for newly rejected candidate"); 417cb9a82fcSTeresa Johnson FailureInfo = llvm::make_unique<FunctionImporter::ImportFailureInfo>( 418cb9a82fcSTeresa Johnson VI, Edge.second.getHotness(), Reason, 1); 419cb9a82fcSTeresa Johnson } 420d34e60caSNicola Zaghen LLVM_DEBUG( 421d34e60caSNicola Zaghen dbgs() << "ignored! No qualifying callee with summary found.\n"); 4227e88d0daSMehdi Amini continue; 4237e88d0daSMehdi Amini } 4242f0cc477SDavid Blaikie 4252f0cc477SDavid Blaikie // "Resolve" the summary 426d68935c5STeresa Johnson CalleeSummary = CalleeSummary->getBaseObject(); 427d68935c5STeresa Johnson ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary); 4282d28f7aaSMehdi Amini 429d9830eb7SPiotr Padlewski assert(ResolvedCalleeSummary->instCount() <= NewThreshold && 43001e32130SMehdi Amini "selectCallee() didn't honor the threshold"); 43101e32130SMehdi Amini 4321b859a23STeresa Johnson auto ExportModulePath = ResolvedCalleeSummary->modulePath(); 433d68935c5STeresa Johnson auto ILI = ImportList[ExportModulePath].insert(VI.getGUID()); 434d68935c5STeresa Johnson // We previously decided to import this GUID definition if it was already 435d68935c5STeresa Johnson // inserted in the set of imports from the exporting module. 436d68935c5STeresa Johnson bool PreviouslyImported = !ILI.second; 437974706ebSTeresa Johnson 4381b859a23STeresa Johnson // Make exports in the source module. 4391b859a23STeresa Johnson if (ExportLists) { 4401b859a23STeresa Johnson auto &ExportList = (*ExportLists)[ExportModulePath]; 4419667b91bSPeter Collingbourne ExportList.insert(VI.getGUID()); 44219f2aa78STeresa Johnson if (!PreviouslyImported) { 44319f2aa78STeresa Johnson // This is the first time this function was exported from its source 44419f2aa78STeresa Johnson // module, so mark all functions and globals it references as exported 4451b859a23STeresa Johnson // to the outside if they are defined in the same source module. 446edddca22STeresa Johnson // For efficiency, we unconditionally add all the referenced GUIDs 447edddca22STeresa Johnson // to the ExportList for this module, and will prune out any not 448edddca22STeresa Johnson // defined in the module later in a single pass. 4491b859a23STeresa Johnson for (auto &Edge : ResolvedCalleeSummary->calls()) { 4501b859a23STeresa Johnson auto CalleeGUID = Edge.first.getGUID(); 451edddca22STeresa Johnson ExportList.insert(CalleeGUID); 4521b859a23STeresa Johnson } 4531b859a23STeresa Johnson for (auto &Ref : ResolvedCalleeSummary->refs()) { 4541b859a23STeresa Johnson auto GUID = Ref.getGUID(); 455edddca22STeresa Johnson ExportList.insert(GUID); 4561b859a23STeresa Johnson } 4571b859a23STeresa Johnson } 45819f2aa78STeresa Johnson } 459d68935c5STeresa Johnson } 460d68935c5STeresa Johnson 461d68935c5STeresa Johnson auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) { 462d68935c5STeresa Johnson // Adjust the threshold for next level of imported functions. 463d68935c5STeresa Johnson // The threshold is different for hot callsites because we can then 464d68935c5STeresa Johnson // inline chains of hot calls. 465d68935c5STeresa Johnson if (IsHotCallsite) 466d68935c5STeresa Johnson return Threshold * ImportHotInstrFactor; 467d68935c5STeresa Johnson return Threshold * ImportInstrFactor; 468d68935c5STeresa Johnson }; 469d68935c5STeresa Johnson 470d68935c5STeresa Johnson bool IsHotCallsite = 471d68935c5STeresa Johnson Edge.second.getHotness() == CalleeInfo::HotnessType::Hot; 472d68935c5STeresa Johnson const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite); 473d68935c5STeresa Johnson 474d68935c5STeresa Johnson ImportCount++; 475d2869473SPiotr Padlewski 47601e32130SMehdi Amini // Insert the newly imported function to the worklist. 4779667b91bSPeter Collingbourne Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold, VI.getGUID()); 478d450da32STeresa Johnson } 479d450da32STeresa Johnson } 480d450da32STeresa Johnson 48101e32130SMehdi Amini /// Given the list of globals defined in a module, compute the list of imports 48201e32130SMehdi Amini /// as well as the list of "exports", i.e. the list of symbols referenced from 48301e32130SMehdi Amini /// another module (that may require promotion). 48401e32130SMehdi Amini static void ComputeImportForModule( 485c851d216STeresa Johnson const GVSummaryMapTy &DefinedGVSummaries, const ModuleSummaryIndex &Index, 486cb9a82fcSTeresa Johnson StringRef ModName, FunctionImporter::ImportMapTy &ImportList, 48756584bbfSEvgeniy Stepanov StringMap<FunctionImporter::ExportSetTy> *ExportLists = nullptr) { 48801e32130SMehdi Amini // Worklist contains the list of function imported in this module, for which 48901e32130SMehdi Amini // we will analyse the callees and may import further down the callgraph. 49001e32130SMehdi Amini SmallVector<EdgeInfo, 128> Worklist; 491d68935c5STeresa Johnson FunctionImporter::ImportThresholdsTy ImportThresholds; 49201e32130SMehdi Amini 49301e32130SMehdi Amini // Populate the worklist with the import for the functions in the current 49401e32130SMehdi Amini // module 49528e457bcSTeresa Johnson for (auto &GVSummary : DefinedGVSummaries) { 4967e7b13d0STeresa Johnson #ifndef NDEBUG 4977e7b13d0STeresa Johnson // FIXME: Change the GVSummaryMapTy to hold ValueInfo instead of GUID 4987e7b13d0STeresa Johnson // so this map look up (and possibly others) can be avoided. 4997e7b13d0STeresa Johnson auto VI = Index.getValueInfo(GVSummary.first); 5007e7b13d0STeresa Johnson #endif 50156584bbfSEvgeniy Stepanov if (!Index.isGlobalValueLive(GVSummary.second)) { 5027e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n"); 5036c475a75STeresa Johnson continue; 5046c475a75STeresa Johnson } 505cfbd0892SPeter Collingbourne auto *FuncSummary = 506cfbd0892SPeter Collingbourne dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject()); 5071aafabf7SMehdi Amini if (!FuncSummary) 5081aafabf7SMehdi Amini // Skip import for global variables 5091aafabf7SMehdi Amini continue; 5107e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n"); 5112d28f7aaSMehdi Amini computeImportForFunction(*FuncSummary, Index, ImportInstrLimit, 5129b490f10SMehdi Amini DefinedGVSummaries, Worklist, ImportList, 513d68935c5STeresa Johnson ExportLists, ImportThresholds); 51401e32130SMehdi Amini } 51501e32130SMehdi Amini 516d2869473SPiotr Padlewski // Process the newly imported functions and add callees to the worklist. 51742418abaSMehdi Amini while (!Worklist.empty()) { 51801e32130SMehdi Amini auto FuncInfo = Worklist.pop_back_val(); 519475b51a7STeresa Johnson auto *Summary = std::get<0>(FuncInfo); 520475b51a7STeresa Johnson auto Threshold = std::get<1>(FuncInfo); 52142418abaSMehdi Amini 5221aafabf7SMehdi Amini computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries, 523d68935c5STeresa Johnson Worklist, ImportList, ExportLists, 524d68935c5STeresa Johnson ImportThresholds); 525c8c55170SMehdi Amini } 526cb9a82fcSTeresa Johnson 527cb9a82fcSTeresa Johnson // Print stats about functions considered but rejected for importing 528cb9a82fcSTeresa Johnson // when requested. 529cb9a82fcSTeresa Johnson if (PrintImportFailures) { 530cb9a82fcSTeresa Johnson dbgs() << "Missed imports into module " << ModName << "\n"; 531cb9a82fcSTeresa Johnson for (auto &I : ImportThresholds) { 532cb9a82fcSTeresa Johnson auto &ProcessedThreshold = std::get<0>(I.second); 533cb9a82fcSTeresa Johnson auto &CalleeSummary = std::get<1>(I.second); 534cb9a82fcSTeresa Johnson auto &FailureInfo = std::get<2>(I.second); 535cb9a82fcSTeresa Johnson if (CalleeSummary) 536cb9a82fcSTeresa Johnson continue; // We are going to import. 537cb9a82fcSTeresa Johnson assert(FailureInfo); 538cb9a82fcSTeresa Johnson FunctionSummary *FS = nullptr; 539cb9a82fcSTeresa Johnson if (!FailureInfo->VI.getSummaryList().empty()) 540cb9a82fcSTeresa Johnson FS = dyn_cast<FunctionSummary>( 541cb9a82fcSTeresa Johnson FailureInfo->VI.getSummaryList()[0]->getBaseObject()); 542cb9a82fcSTeresa Johnson dbgs() << FailureInfo->VI 543cb9a82fcSTeresa Johnson << ": Reason = " << getFailureName(FailureInfo->Reason) 544cb9a82fcSTeresa Johnson << ", Threshold = " << ProcessedThreshold 545cb9a82fcSTeresa Johnson << ", Size = " << (FS ? (int)FS->instCount() : -1) 546cb9a82fcSTeresa Johnson << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness) 547cb9a82fcSTeresa Johnson << ", Attempts = " << FailureInfo->Attempts << "\n"; 548cb9a82fcSTeresa Johnson } 549cb9a82fcSTeresa Johnson } 55042418abaSMehdi Amini } 551ffe2e4aaSMehdi Amini 55219e23874SEugene Leviant #ifndef NDEBUG 55319e23874SEugene Leviant static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, 55419e23874SEugene Leviant GlobalValue::GUID G) { 55519e23874SEugene Leviant if (const auto &VI = Index.getValueInfo(G)) { 55619e23874SEugene Leviant auto SL = VI.getSummaryList(); 55719e23874SEugene Leviant if (!SL.empty()) 55819e23874SEugene Leviant return SL[0]->getSummaryKind() == GlobalValueSummary::GlobalVarKind; 55919e23874SEugene Leviant } 56019e23874SEugene Leviant return false; 56119e23874SEugene Leviant } 56219e23874SEugene Leviant 56319e23874SEugene Leviant static GlobalValue::GUID getGUID(GlobalValue::GUID G) { return G; } 56419e23874SEugene Leviant 56519e23874SEugene Leviant template <class T> 5661fc0da48SBenjamin Kramer static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, 5671fc0da48SBenjamin Kramer T &Cont) { 56819e23874SEugene Leviant unsigned NumGVS = 0; 56919e23874SEugene Leviant for (auto &V : Cont) 57019e23874SEugene Leviant if (isGlobalVarSummary(Index, getGUID(V))) 57119e23874SEugene Leviant ++NumGVS; 57219e23874SEugene Leviant return NumGVS; 57319e23874SEugene Leviant } 57419e23874SEugene Leviant #endif 57519e23874SEugene Leviant 576c86af334STeresa Johnson /// Compute all the import and export for every module using the Index. 57701e32130SMehdi Amini void llvm::ComputeCrossModuleImport( 57801e32130SMehdi Amini const ModuleSummaryIndex &Index, 579c851d216STeresa Johnson const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 58001e32130SMehdi Amini StringMap<FunctionImporter::ImportMapTy> &ImportLists, 58156584bbfSEvgeniy Stepanov StringMap<FunctionImporter::ExportSetTy> &ExportLists) { 58201e32130SMehdi Amini // For each module that has function defined, compute the import/export lists. 5831aafabf7SMehdi Amini for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) { 5849b490f10SMehdi Amini auto &ImportList = ImportLists[DefinedGVSummaries.first()]; 585d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Computing import for Module '" 5861aafabf7SMehdi Amini << DefinedGVSummaries.first() << "'\n"); 587cb9a82fcSTeresa Johnson ComputeImportForModule(DefinedGVSummaries.second, Index, 588cb9a82fcSTeresa Johnson DefinedGVSummaries.first(), ImportList, 58956584bbfSEvgeniy Stepanov &ExportLists); 59001e32130SMehdi Amini } 59101e32130SMehdi Amini 592edddca22STeresa Johnson // When computing imports we added all GUIDs referenced by anything 593edddca22STeresa Johnson // imported from the module to its ExportList. Now we prune each ExportList 594edddca22STeresa Johnson // of any not defined in that module. This is more efficient than checking 595edddca22STeresa Johnson // while computing imports because some of the summary lists may be long 596edddca22STeresa Johnson // due to linkonce (comdat) copies. 597edddca22STeresa Johnson for (auto &ELI : ExportLists) { 598edddca22STeresa Johnson const auto &DefinedGVSummaries = 599edddca22STeresa Johnson ModuleToDefinedGVSummaries.lookup(ELI.first()); 600edddca22STeresa Johnson for (auto EI = ELI.second.begin(); EI != ELI.second.end();) { 601edddca22STeresa Johnson if (!DefinedGVSummaries.count(*EI)) 602edddca22STeresa Johnson EI = ELI.second.erase(EI); 603edddca22STeresa Johnson else 604edddca22STeresa Johnson ++EI; 605edddca22STeresa Johnson } 606edddca22STeresa Johnson } 607edddca22STeresa Johnson 60801e32130SMehdi Amini #ifndef NDEBUG 609d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size() 61001e32130SMehdi Amini << " modules:\n"); 61101e32130SMehdi Amini for (auto &ModuleImports : ImportLists) { 61201e32130SMehdi Amini auto ModName = ModuleImports.first(); 61301e32130SMehdi Amini auto &Exports = ExportLists[ModName]; 61419e23874SEugene Leviant unsigned NumGVS = numGlobalVarSummaries(Index, Exports); 615d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports " 61619e23874SEugene Leviant << Exports.size() - NumGVS << " functions and " << NumGVS 617d34e60caSNicola Zaghen << " vars. Imports from " << ModuleImports.second.size() 618d34e60caSNicola Zaghen << " modules.\n"); 61901e32130SMehdi Amini for (auto &Src : ModuleImports.second) { 62001e32130SMehdi Amini auto SrcModName = Src.first(); 62119e23874SEugene Leviant unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 622d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 62319e23874SEugene Leviant << " functions imported from " << SrcModName << "\n"); 624d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod 625d34e60caSNicola Zaghen << " global vars imported from " << SrcModName << "\n"); 62601e32130SMehdi Amini } 62701e32130SMehdi Amini } 62801e32130SMehdi Amini #endif 62901e32130SMehdi Amini } 63001e32130SMehdi Amini 63181bbf742STeresa Johnson #ifndef NDEBUG 63219e23874SEugene Leviant static void dumpImportListForModule(const ModuleSummaryIndex &Index, 63319e23874SEugene Leviant StringRef ModulePath, 63481bbf742STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 635d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from " 63681bbf742STeresa Johnson << ImportList.size() << " modules.\n"); 63781bbf742STeresa Johnson for (auto &Src : ImportList) { 63881bbf742STeresa Johnson auto SrcModName = Src.first(); 63919e23874SEugene Leviant unsigned NumGVSPerMod = numGlobalVarSummaries(Index, Src.second); 640d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << Src.second.size() - NumGVSPerMod 64119e23874SEugene Leviant << " functions imported from " << SrcModName << "\n"); 642d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << " - " << NumGVSPerMod << " vars imported from " 64381bbf742STeresa Johnson << SrcModName << "\n"); 64481bbf742STeresa Johnson } 64581bbf742STeresa Johnson } 64669b2de84STeresa Johnson #endif 64781bbf742STeresa Johnson 648c86af334STeresa Johnson /// Compute all the imports for the given module in the Index. 649c86af334STeresa Johnson void llvm::ComputeCrossModuleImportForModule( 650c86af334STeresa Johnson StringRef ModulePath, const ModuleSummaryIndex &Index, 651c86af334STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 652c86af334STeresa Johnson // Collect the list of functions this module defines. 653c86af334STeresa Johnson // GUID -> Summary 654c851d216STeresa Johnson GVSummaryMapTy FunctionSummaryMap; 65528e457bcSTeresa Johnson Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap); 656c86af334STeresa Johnson 657c86af334STeresa Johnson // Compute the import list for this module. 658d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n"); 659cb9a82fcSTeresa Johnson ComputeImportForModule(FunctionSummaryMap, Index, ModulePath, ImportList); 660c86af334STeresa Johnson 661c86af334STeresa Johnson #ifndef NDEBUG 66219e23874SEugene Leviant dumpImportListForModule(Index, ModulePath, ImportList); 66381bbf742STeresa Johnson #endif 664c86af334STeresa Johnson } 66581bbf742STeresa Johnson 66681bbf742STeresa Johnson // Mark all external summaries in Index for import into the given module. 66781bbf742STeresa Johnson // Used for distributed builds using a distributed index. 66881bbf742STeresa Johnson void llvm::ComputeCrossModuleImportForModuleFromIndex( 66981bbf742STeresa Johnson StringRef ModulePath, const ModuleSummaryIndex &Index, 67081bbf742STeresa Johnson FunctionImporter::ImportMapTy &ImportList) { 67181bbf742STeresa Johnson for (auto &GlobalList : Index) { 67281bbf742STeresa Johnson // Ignore entries for undefined references. 67381bbf742STeresa Johnson if (GlobalList.second.SummaryList.empty()) 67481bbf742STeresa Johnson continue; 67581bbf742STeresa Johnson 67681bbf742STeresa Johnson auto GUID = GlobalList.first; 67781bbf742STeresa Johnson assert(GlobalList.second.SummaryList.size() == 1 && 67881bbf742STeresa Johnson "Expected individual combined index to have one summary per GUID"); 67981bbf742STeresa Johnson auto &Summary = GlobalList.second.SummaryList[0]; 68081bbf742STeresa Johnson // Skip the summaries for the importing module. These are included to 68181bbf742STeresa Johnson // e.g. record required linkage changes. 68281bbf742STeresa Johnson if (Summary->modulePath() == ModulePath) 68381bbf742STeresa Johnson continue; 684d68935c5STeresa Johnson // Add an entry to provoke importing by thinBackend. 685d68935c5STeresa Johnson ImportList[Summary->modulePath()].insert(GUID); 68681bbf742STeresa Johnson } 68781bbf742STeresa Johnson #ifndef NDEBUG 68819e23874SEugene Leviant dumpImportListForModule(Index, ModulePath, ImportList); 689c86af334STeresa Johnson #endif 690c86af334STeresa Johnson } 691c86af334STeresa Johnson 69256584bbfSEvgeniy Stepanov void llvm::computeDeadSymbols( 69356584bbfSEvgeniy Stepanov ModuleSummaryIndex &Index, 694eaf5172cSGeorge Rimar const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, 695eaf5172cSGeorge Rimar function_ref<PrevailingType(GlobalValue::GUID)> isPrevailing) { 69656584bbfSEvgeniy Stepanov assert(!Index.withGlobalValueDeadStripping()); 6976c475a75STeresa Johnson if (!ComputeDead) 69856584bbfSEvgeniy Stepanov return; 6996c475a75STeresa Johnson if (GUIDPreservedSymbols.empty()) 7006c475a75STeresa Johnson // Don't do anything when nothing is live, this is friendly with tests. 70156584bbfSEvgeniy Stepanov return; 70256584bbfSEvgeniy Stepanov unsigned LiveSymbols = 0; 7039667b91bSPeter Collingbourne SmallVector<ValueInfo, 128> Worklist; 7049667b91bSPeter Collingbourne Worklist.reserve(GUIDPreservedSymbols.size() * 2); 7059667b91bSPeter Collingbourne for (auto GUID : GUIDPreservedSymbols) { 7069667b91bSPeter Collingbourne ValueInfo VI = Index.getValueInfo(GUID); 7079667b91bSPeter Collingbourne if (!VI) 7089667b91bSPeter Collingbourne continue; 70956584bbfSEvgeniy Stepanov for (auto &S : VI.getSummaryList()) 71056584bbfSEvgeniy Stepanov S->setLive(true); 7116c475a75STeresa Johnson } 71256584bbfSEvgeniy Stepanov 7136c475a75STeresa Johnson // Add values flagged in the index as live roots to the worklist. 7147e7b13d0STeresa Johnson for (const auto &Entry : Index) { 7157e7b13d0STeresa Johnson auto VI = Index.getValueInfo(Entry); 71656584bbfSEvgeniy Stepanov for (auto &S : Entry.second.SummaryList) 71756584bbfSEvgeniy Stepanov if (S->isLive()) { 7187e7b13d0STeresa Johnson LLVM_DEBUG(dbgs() << "Live root: " << VI << "\n"); 7197e7b13d0STeresa Johnson Worklist.push_back(VI); 72056584bbfSEvgeniy Stepanov ++LiveSymbols; 72156584bbfSEvgeniy Stepanov break; 7226c475a75STeresa Johnson } 7237e7b13d0STeresa Johnson } 7246c475a75STeresa Johnson 72556584bbfSEvgeniy Stepanov // Make value live and add it to the worklist if it was not live before. 72656584bbfSEvgeniy Stepanov auto visit = [&](ValueInfo VI) { 7271958083dSTeresa Johnson // FIXME: If we knew which edges were created for indirect call profiles, 7281958083dSTeresa Johnson // we could skip them here. Any that are live should be reached via 7291958083dSTeresa Johnson // other edges, e.g. reference edges. Otherwise, using a profile collected 7301958083dSTeresa Johnson // on a slightly different binary might provoke preserving, importing 7311958083dSTeresa Johnson // and ultimately promoting calls to functions not linked into this 7321958083dSTeresa Johnson // binary, which increases the binary size unnecessarily. Note that 7331958083dSTeresa Johnson // if this code changes, the importer needs to change so that edges 7341958083dSTeresa Johnson // to functions marked dead are skipped. 7351958083dSTeresa Johnson VI = updateValueInfoForIndirectCalls(Index, VI); 7361958083dSTeresa Johnson if (!VI) 7371958083dSTeresa Johnson return; 73856584bbfSEvgeniy Stepanov for (auto &S : VI.getSummaryList()) 739f625118eSTeresa Johnson if (S->isLive()) 740f625118eSTeresa Johnson return; 741eaf5172cSGeorge Rimar 742aab60006SVlad Tsyrklevich // We only keep live symbols that are known to be non-prevailing if any are 743bfdad33bSXin Tong // available_externally, linkonceodr, weakodr. Those symbols are discarded 744bfdad33bSXin Tong // later in the EliminateAvailableExternally pass and setting them to 745bfdad33bSXin Tong // not-live could break downstreams users of liveness information (PR36483) 746bfdad33bSXin Tong // or limit optimization opportunities. 747aab60006SVlad Tsyrklevich if (isPrevailing(VI.getGUID()) == PrevailingType::No) { 748bfdad33bSXin Tong bool KeepAliveLinkage = false; 749aab60006SVlad Tsyrklevich bool Interposable = false; 750aab60006SVlad Tsyrklevich for (auto &S : VI.getSummaryList()) { 751bfdad33bSXin Tong if (S->linkage() == GlobalValue::AvailableExternallyLinkage || 752bfdad33bSXin Tong S->linkage() == GlobalValue::WeakODRLinkage || 753bfdad33bSXin Tong S->linkage() == GlobalValue::LinkOnceODRLinkage) 754bfdad33bSXin Tong KeepAliveLinkage = true; 755aab60006SVlad Tsyrklevich else if (GlobalValue::isInterposableLinkage(S->linkage())) 756aab60006SVlad Tsyrklevich Interposable = true; 757aab60006SVlad Tsyrklevich } 758aab60006SVlad Tsyrklevich 759bfdad33bSXin Tong if (!KeepAliveLinkage) 760eaf5172cSGeorge Rimar return; 761eaf5172cSGeorge Rimar 762aab60006SVlad Tsyrklevich if (Interposable) 763bfdad33bSXin Tong report_fatal_error( 764bfdad33bSXin Tong "Interposable and available_externally/linkonce_odr/weak_odr symbol"); 765aab60006SVlad Tsyrklevich } 766aab60006SVlad Tsyrklevich 767f625118eSTeresa Johnson for (auto &S : VI.getSummaryList()) 76856584bbfSEvgeniy Stepanov S->setLive(true); 76956584bbfSEvgeniy Stepanov ++LiveSymbols; 77056584bbfSEvgeniy Stepanov Worklist.push_back(VI); 77156584bbfSEvgeniy Stepanov }; 77256584bbfSEvgeniy Stepanov 7736c475a75STeresa Johnson while (!Worklist.empty()) { 7749667b91bSPeter Collingbourne auto VI = Worklist.pop_back_val(); 7759667b91bSPeter Collingbourne for (auto &Summary : VI.getSummaryList()) { 776cfbd0892SPeter Collingbourne GlobalValueSummary *Base = Summary->getBaseObject(); 777eaf5172cSGeorge Rimar // Set base value live in case it is an alias. 778eaf5172cSGeorge Rimar Base->setLive(true); 779cfbd0892SPeter Collingbourne for (auto Ref : Base->refs()) 78056584bbfSEvgeniy Stepanov visit(Ref); 781cfbd0892SPeter Collingbourne if (auto *FS = dyn_cast<FunctionSummary>(Base)) 78256584bbfSEvgeniy Stepanov for (auto Call : FS->calls()) 78356584bbfSEvgeniy Stepanov visit(Call.first); 7846c475a75STeresa Johnson } 7856c475a75STeresa Johnson } 78656584bbfSEvgeniy Stepanov Index.setWithGlobalValueDeadStripping(); 78756584bbfSEvgeniy Stepanov 78856584bbfSEvgeniy Stepanov unsigned DeadSymbols = Index.size() - LiveSymbols; 789d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols 79056584bbfSEvgeniy Stepanov << " symbols Dead \n"); 79156584bbfSEvgeniy Stepanov NumDeadSymbols += DeadSymbols; 79256584bbfSEvgeniy Stepanov NumLiveSymbols += LiveSymbols; 7936c475a75STeresa Johnson } 7946c475a75STeresa Johnson 79584174c37STeresa Johnson /// Compute the set of summaries needed for a ThinLTO backend compilation of 79684174c37STeresa Johnson /// \p ModulePath. 79784174c37STeresa Johnson void llvm::gatherImportedSummariesForModule( 79884174c37STeresa Johnson StringRef ModulePath, 79984174c37STeresa Johnson const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries, 800cdbcbf74SMehdi Amini const FunctionImporter::ImportMapTy &ImportList, 80184174c37STeresa Johnson std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 80284174c37STeresa Johnson // Include all summaries from the importing module. 80384174c37STeresa Johnson ModuleToSummariesForIndex[ModulePath] = 80484174c37STeresa Johnson ModuleToDefinedGVSummaries.lookup(ModulePath); 80584174c37STeresa Johnson // Include summaries for imports. 80688c491ddSMehdi Amini for (auto &ILI : ImportList) { 80784174c37STeresa Johnson auto &SummariesForIndex = ModuleToSummariesForIndex[ILI.first()]; 80884174c37STeresa Johnson const auto &DefinedGVSummaries = 80984174c37STeresa Johnson ModuleToDefinedGVSummaries.lookup(ILI.first()); 81084174c37STeresa Johnson for (auto &GI : ILI.second) { 811d68935c5STeresa Johnson const auto &DS = DefinedGVSummaries.find(GI); 81284174c37STeresa Johnson assert(DS != DefinedGVSummaries.end() && 81384174c37STeresa Johnson "Expected a defined summary for imported global value"); 814d68935c5STeresa Johnson SummariesForIndex[GI] = DS->second; 81584174c37STeresa Johnson } 81684174c37STeresa Johnson } 81784174c37STeresa Johnson } 81884174c37STeresa Johnson 8198570fe47STeresa Johnson /// Emit the files \p ModulePath will import from into \p OutputFilename. 820c0320ef4STeresa Johnson std::error_code llvm::EmitImportsFiles( 821c0320ef4STeresa Johnson StringRef ModulePath, StringRef OutputFilename, 822c0320ef4STeresa Johnson const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { 8238570fe47STeresa Johnson std::error_code EC; 8248570fe47STeresa Johnson raw_fd_ostream ImportsOS(OutputFilename, EC, sys::fs::OpenFlags::F_None); 8258570fe47STeresa Johnson if (EC) 8268570fe47STeresa Johnson return EC; 827c0320ef4STeresa Johnson for (auto &ILI : ModuleToSummariesForIndex) 828c0320ef4STeresa Johnson // The ModuleToSummariesForIndex map includes an entry for the current 829c0320ef4STeresa Johnson // Module (needed for writing out the index files). We don't want to 830c0320ef4STeresa Johnson // include it in the imports file, however, so filter it out. 831c0320ef4STeresa Johnson if (ILI.first != ModulePath) 832c0320ef4STeresa Johnson ImportsOS << ILI.first << "\n"; 8338570fe47STeresa Johnson return std::error_code(); 8348570fe47STeresa Johnson } 8358570fe47STeresa Johnson 8365a95c477STeresa Johnson bool llvm::convertToDeclaration(GlobalValue &GV) { 837d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName() 838d34e60caSNicola Zaghen << "\n"); 8394566c6dbSTeresa Johnson if (Function *F = dyn_cast<Function>(&GV)) { 8404566c6dbSTeresa Johnson F->deleteBody(); 8414566c6dbSTeresa Johnson F->clearMetadata(); 8427873669bSPeter Collingbourne F->setComdat(nullptr); 8434566c6dbSTeresa Johnson } else if (GlobalVariable *V = dyn_cast<GlobalVariable>(&GV)) { 8444566c6dbSTeresa Johnson V->setInitializer(nullptr); 8454566c6dbSTeresa Johnson V->setLinkage(GlobalValue::ExternalLinkage); 8464566c6dbSTeresa Johnson V->clearMetadata(); 8477873669bSPeter Collingbourne V->setComdat(nullptr); 8485a95c477STeresa Johnson } else { 8495a95c477STeresa Johnson GlobalValue *NewGV; 8505a95c477STeresa Johnson if (GV.getValueType()->isFunctionTy()) 8515a95c477STeresa Johnson NewGV = 8525a95c477STeresa Johnson Function::Create(cast<FunctionType>(GV.getValueType()), 8535a95c477STeresa Johnson GlobalValue::ExternalLinkage, "", GV.getParent()); 8545a95c477STeresa Johnson else 8555a95c477STeresa Johnson NewGV = 8565a95c477STeresa Johnson new GlobalVariable(*GV.getParent(), GV.getValueType(), 8575a95c477STeresa Johnson /*isConstant*/ false, GlobalValue::ExternalLinkage, 8585a95c477STeresa Johnson /*init*/ nullptr, "", 8595a95c477STeresa Johnson /*insertbefore*/ nullptr, GV.getThreadLocalMode(), 8605a95c477STeresa Johnson GV.getType()->getAddressSpace()); 8615a95c477STeresa Johnson NewGV->takeName(&GV); 8625a95c477STeresa Johnson GV.replaceAllUsesWith(NewGV); 8635a95c477STeresa Johnson return false; 8645a95c477STeresa Johnson } 8655a95c477STeresa Johnson return true; 866eaf5172cSGeorge Rimar } 8674566c6dbSTeresa Johnson 868eaf5172cSGeorge Rimar /// Fixup WeakForLinker linkages in \p TheModule based on summary analysis. 869eaf5172cSGeorge Rimar void llvm::thinLTOResolveWeakForLinkerModule( 870eaf5172cSGeorge Rimar Module &TheModule, const GVSummaryMapTy &DefinedGlobals) { 87104c9a2d6STeresa Johnson auto updateLinkage = [&](GlobalValue &GV) { 87204c9a2d6STeresa Johnson // See if the global summary analysis computed a new resolved linkage. 87304c9a2d6STeresa Johnson const auto &GS = DefinedGlobals.find(GV.getGUID()); 87404c9a2d6STeresa Johnson if (GS == DefinedGlobals.end()) 87504c9a2d6STeresa Johnson return; 87604c9a2d6STeresa Johnson auto NewLinkage = GS->second->linkage(); 87704c9a2d6STeresa Johnson if (NewLinkage == GV.getLinkage()) 87804c9a2d6STeresa Johnson return; 8796a5fbe52SDavide Italiano 8806a5fbe52SDavide Italiano // Switch the linkage to weakany if asked for, e.g. we do this for 8816a5fbe52SDavide Italiano // linker redefined symbols (via --wrap or --defsym). 882f4891d29SDavide Italiano // We record that the visibility should be changed here in `addThinLTO` 883f4891d29SDavide Italiano // as we need access to the resolution vectors for each input file in 884f4891d29SDavide Italiano // order to find which symbols have been redefined. 885f4891d29SDavide Italiano // We may consider reorganizing this code and moving the linkage recording 886f4891d29SDavide Italiano // somewhere else, e.g. in thinLTOResolveWeakForLinkerInIndex. 8876a5fbe52SDavide Italiano if (NewLinkage == GlobalValue::WeakAnyLinkage) { 8886a5fbe52SDavide Italiano GV.setLinkage(NewLinkage); 8896a5fbe52SDavide Italiano return; 8906a5fbe52SDavide Italiano } 8916a5fbe52SDavide Italiano 8926a5fbe52SDavide Italiano if (!GlobalValue::isWeakForLinker(GV.getLinkage())) 8936a5fbe52SDavide Italiano return; 8944566c6dbSTeresa Johnson // Check for a non-prevailing def that has interposable linkage 8954566c6dbSTeresa Johnson // (e.g. non-odr weak or linkonce). In that case we can't simply 8964566c6dbSTeresa Johnson // convert to available_externally, since it would lose the 8974566c6dbSTeresa Johnson // interposable property and possibly get inlined. Simply drop 8984566c6dbSTeresa Johnson // the definition in that case. 8994566c6dbSTeresa Johnson if (GlobalValue::isAvailableExternallyLinkage(NewLinkage) && 9005a95c477STeresa Johnson GlobalValue::isInterposableLinkage(GV.getLinkage())) { 9015a95c477STeresa Johnson if (!convertToDeclaration(GV)) 9025a95c477STeresa Johnson // FIXME: Change this to collect replaced GVs and later erase 9035a95c477STeresa Johnson // them from the parent module once thinLTOResolveWeakForLinkerGUID is 9045a95c477STeresa Johnson // changed to enable this for aliases. 9055a95c477STeresa Johnson llvm_unreachable("Expected GV to be converted"); 9065a95c477STeresa Johnson } else { 90733ba93c2SSteven Wu // If the original symbols has global unnamed addr and linkonce_odr linkage, 90833ba93c2SSteven Wu // it should be an auto hide symbol. Add hidden visibility to the symbol to 90933ba93c2SSteven Wu // preserve the property. 91033ba93c2SSteven Wu if (GV.hasLinkOnceODRLinkage() && GV.hasGlobalUnnamedAddr() && 91133ba93c2SSteven Wu NewLinkage == GlobalValue::WeakODRLinkage) 91233ba93c2SSteven Wu GV.setVisibility(GlobalValue::HiddenVisibility); 91333ba93c2SSteven Wu 914d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "ODR fixing up linkage for `" << GV.getName() 915d34e60caSNicola Zaghen << "` from " << GV.getLinkage() << " to " << NewLinkage 916d34e60caSNicola Zaghen << "\n"); 91704c9a2d6STeresa Johnson GV.setLinkage(NewLinkage); 9184566c6dbSTeresa Johnson } 9194566c6dbSTeresa Johnson // Remove declarations from comdats, including available_externally 9206107a419STeresa Johnson // as this is a declaration for the linker, and will be dropped eventually. 9216107a419STeresa Johnson // It is illegal for comdats to contain declarations. 9226107a419STeresa Johnson auto *GO = dyn_cast_or_null<GlobalObject>(&GV); 9234566c6dbSTeresa Johnson if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) 9246107a419STeresa Johnson GO->setComdat(nullptr); 92504c9a2d6STeresa Johnson }; 92604c9a2d6STeresa Johnson 92704c9a2d6STeresa Johnson // Process functions and global now 92804c9a2d6STeresa Johnson for (auto &GV : TheModule) 92904c9a2d6STeresa Johnson updateLinkage(GV); 93004c9a2d6STeresa Johnson for (auto &GV : TheModule.globals()) 93104c9a2d6STeresa Johnson updateLinkage(GV); 93204c9a2d6STeresa Johnson for (auto &GV : TheModule.aliases()) 93304c9a2d6STeresa Johnson updateLinkage(GV); 93404c9a2d6STeresa Johnson } 93504c9a2d6STeresa Johnson 93604c9a2d6STeresa Johnson /// Run internalization on \p TheModule based on symmary analysis. 93704c9a2d6STeresa Johnson void llvm::thinLTOInternalizeModule(Module &TheModule, 93804c9a2d6STeresa Johnson const GVSummaryMapTy &DefinedGlobals) { 93904c9a2d6STeresa Johnson // Declare a callback for the internalize pass that will ask for every 94004c9a2d6STeresa Johnson // candidate GlobalValue if it can be internalized or not. 94104c9a2d6STeresa Johnson auto MustPreserveGV = [&](const GlobalValue &GV) -> bool { 94204c9a2d6STeresa Johnson // Lookup the linkage recorded in the summaries during global analysis. 943c3d677f9SPeter Collingbourne auto GS = DefinedGlobals.find(GV.getGUID()); 94404c9a2d6STeresa Johnson if (GS == DefinedGlobals.end()) { 94504c9a2d6STeresa Johnson // Must have been promoted (possibly conservatively). Find original 94604c9a2d6STeresa Johnson // name so that we can access the correct summary and see if it can 94704c9a2d6STeresa Johnson // be internalized again. 94804c9a2d6STeresa Johnson // FIXME: Eventually we should control promotion instead of promoting 94904c9a2d6STeresa Johnson // and internalizing again. 95004c9a2d6STeresa Johnson StringRef OrigName = 95104c9a2d6STeresa Johnson ModuleSummaryIndex::getOriginalNameBeforePromote(GV.getName()); 95204c9a2d6STeresa Johnson std::string OrigId = GlobalValue::getGlobalIdentifier( 95304c9a2d6STeresa Johnson OrigName, GlobalValue::InternalLinkage, 95404c9a2d6STeresa Johnson TheModule.getSourceFileName()); 955c3d677f9SPeter Collingbourne GS = DefinedGlobals.find(GlobalValue::getGUID(OrigId)); 9567ab1f692STeresa Johnson if (GS == DefinedGlobals.end()) { 9577ab1f692STeresa Johnson // Also check the original non-promoted non-globalized name. In some 9587ab1f692STeresa Johnson // cases a preempted weak value is linked in as a local copy because 9597ab1f692STeresa Johnson // it is referenced by an alias (IRLinker::linkGlobalValueProto). 9607ab1f692STeresa Johnson // In that case, since it was originally not a local value, it was 9617ab1f692STeresa Johnson // recorded in the index using the original name. 9627ab1f692STeresa Johnson // FIXME: This may not be needed once PR27866 is fixed. 963c3d677f9SPeter Collingbourne GS = DefinedGlobals.find(GlobalValue::getGUID(OrigName)); 96404c9a2d6STeresa Johnson assert(GS != DefinedGlobals.end()); 9657ab1f692STeresa Johnson } 966c3d677f9SPeter Collingbourne } 967c3d677f9SPeter Collingbourne return !GlobalValue::isLocalLinkage(GS->second->linkage()); 96804c9a2d6STeresa Johnson }; 96904c9a2d6STeresa Johnson 97004c9a2d6STeresa Johnson // FIXME: See if we can just internalize directly here via linkage changes 97104c9a2d6STeresa Johnson // based on the index, rather than invoking internalizeModule. 972e9ea08a0SEugene Zelenko internalizeModule(TheModule, MustPreserveGV); 97304c9a2d6STeresa Johnson } 97404c9a2d6STeresa Johnson 97581bbf742STeresa Johnson /// Make alias a clone of its aliasee. 97681bbf742STeresa Johnson static Function *replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA) { 97781bbf742STeresa Johnson Function *Fn = cast<Function>(GA->getBaseObject()); 97881bbf742STeresa Johnson 97981bbf742STeresa Johnson ValueToValueMapTy VMap; 98081bbf742STeresa Johnson Function *NewFn = CloneFunction(Fn, VMap); 98181bbf742STeresa Johnson // Clone should use the original alias's linkage and name, and we ensure 98281bbf742STeresa Johnson // all uses of alias instead use the new clone (casted if necessary). 98381bbf742STeresa Johnson NewFn->setLinkage(GA->getLinkage()); 98481bbf742STeresa Johnson GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewFn, GA->getType())); 98581bbf742STeresa Johnson NewFn->takeName(GA); 98681bbf742STeresa Johnson return NewFn; 98781bbf742STeresa Johnson } 98881bbf742STeresa Johnson 989c8c55170SMehdi Amini // Automatically import functions in Module \p DestModule based on the summaries 990c8c55170SMehdi Amini // index. 9917f00d0a1SPeter Collingbourne Expected<bool> FunctionImporter::importFunctions( 99266043797SAdrian Prantl Module &DestModule, const FunctionImporter::ImportMapTy &ImportList) { 993d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Starting import for Module " 994311fef6eSMehdi Amini << DestModule.getModuleIdentifier() << "\n"); 99519e23874SEugene Leviant unsigned ImportedCount = 0, ImportedGVCount = 0; 996c8c55170SMehdi Amini 9976d8f817fSPeter Collingbourne IRMover Mover(DestModule); 9987e88d0daSMehdi Amini // Do the actual import of functions now, one Module at a time 99901e32130SMehdi Amini std::set<StringRef> ModuleNameOrderedList; 100001e32130SMehdi Amini for (auto &FunctionsToImportPerModule : ImportList) { 100101e32130SMehdi Amini ModuleNameOrderedList.insert(FunctionsToImportPerModule.first()); 100201e32130SMehdi Amini } 100301e32130SMehdi Amini for (auto &Name : ModuleNameOrderedList) { 10047e88d0daSMehdi Amini // Get the module for the import 100501e32130SMehdi Amini const auto &FunctionsToImportPerModule = ImportList.find(Name); 100601e32130SMehdi Amini assert(FunctionsToImportPerModule != ImportList.end()); 1007d9445c49SPeter Collingbourne Expected<std::unique_ptr<Module>> SrcModuleOrErr = ModuleLoader(Name); 1008d9445c49SPeter Collingbourne if (!SrcModuleOrErr) 1009d9445c49SPeter Collingbourne return SrcModuleOrErr.takeError(); 1010d9445c49SPeter Collingbourne std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr); 10117e88d0daSMehdi Amini assert(&DestModule.getContext() == &SrcModule->getContext() && 10127e88d0daSMehdi Amini "Context mismatch"); 10137e88d0daSMehdi Amini 10146cba37ceSTeresa Johnson // If modules were created with lazy metadata loading, materialize it 10156cba37ceSTeresa Johnson // now, before linking it (otherwise this will be a noop). 10167f00d0a1SPeter Collingbourne if (Error Err = SrcModule->materializeMetadata()) 10177f00d0a1SPeter Collingbourne return std::move(Err); 1018e5a61917STeresa Johnson 101901e32130SMehdi Amini auto &ImportGUIDs = FunctionsToImportPerModule->second; 102001e32130SMehdi Amini // Find the globals to import 10216d8f817fSPeter Collingbourne SetVector<GlobalValue *> GlobalsToImport; 10221f685e01SPiotr Padlewski for (Function &F : *SrcModule) { 10231f685e01SPiotr Padlewski if (!F.hasName()) 10240beb858eSTeresa Johnson continue; 10251f685e01SPiotr Padlewski auto GUID = F.getGUID(); 10260beb858eSTeresa Johnson auto Import = ImportGUIDs.count(GUID); 1027d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing function " 1028d34e60caSNicola Zaghen << GUID << " " << F.getName() << " from " 1029aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 10300beb858eSTeresa Johnson if (Import) { 10317f00d0a1SPeter Collingbourne if (Error Err = F.materialize()) 10327f00d0a1SPeter Collingbourne return std::move(Err); 10333b776128SPiotr Padlewski if (EnableImportMetadata) { 10346deaa6afSPiotr Padlewski // Add 'thinlto_src_module' metadata for statistics and debugging. 10353b776128SPiotr Padlewski F.setMetadata( 10363b776128SPiotr Padlewski "thinlto_src_module", 1037e9ea08a0SEugene Zelenko MDNode::get(DestModule.getContext(), 1038e9ea08a0SEugene Zelenko {MDString::get(DestModule.getContext(), 10396deaa6afSPiotr Padlewski SrcModule->getSourceFileName())})); 10403b776128SPiotr Padlewski } 10411f685e01SPiotr Padlewski GlobalsToImport.insert(&F); 104201e32130SMehdi Amini } 104301e32130SMehdi Amini } 10441f685e01SPiotr Padlewski for (GlobalVariable &GV : SrcModule->globals()) { 10452d28f7aaSMehdi Amini if (!GV.hasName()) 10462d28f7aaSMehdi Amini continue; 10472d28f7aaSMehdi Amini auto GUID = GV.getGUID(); 10482d28f7aaSMehdi Amini auto Import = ImportGUIDs.count(GUID); 1049d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing global " 1050d34e60caSNicola Zaghen << GUID << " " << GV.getName() << " from " 1051aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 10522d28f7aaSMehdi Amini if (Import) { 10537f00d0a1SPeter Collingbourne if (Error Err = GV.materialize()) 10547f00d0a1SPeter Collingbourne return std::move(Err); 105519e23874SEugene Leviant ImportedGVCount += GlobalsToImport.insert(&GV); 10562d28f7aaSMehdi Amini } 10572d28f7aaSMehdi Amini } 10581f685e01SPiotr Padlewski for (GlobalAlias &GA : SrcModule->aliases()) { 10591f685e01SPiotr Padlewski if (!GA.hasName()) 106001e32130SMehdi Amini continue; 10611f685e01SPiotr Padlewski auto GUID = GA.getGUID(); 106281bbf742STeresa Johnson auto Import = ImportGUIDs.count(GUID); 1063d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << (Import ? "Is" : "Not") << " importing alias " 1064d34e60caSNicola Zaghen << GUID << " " << GA.getName() << " from " 1065aeb1e59bSMehdi Amini << SrcModule->getSourceFileName() << "\n"); 106681bbf742STeresa Johnson if (Import) { 106781bbf742STeresa Johnson if (Error Err = GA.materialize()) 106881bbf742STeresa Johnson return std::move(Err); 106981bbf742STeresa Johnson // Import alias as a copy of its aliasee. 107081bbf742STeresa Johnson GlobalObject *Base = GA.getBaseObject(); 107181bbf742STeresa Johnson if (Error Err = Base->materialize()) 107281bbf742STeresa Johnson return std::move(Err); 107381bbf742STeresa Johnson auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA); 1074d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << Base->getGUID() 107581bbf742STeresa Johnson << " " << Base->getName() << " from " 107681bbf742STeresa Johnson << SrcModule->getSourceFileName() << "\n"); 107781bbf742STeresa Johnson if (EnableImportMetadata) { 107881bbf742STeresa Johnson // Add 'thinlto_src_module' metadata for statistics and debugging. 107981bbf742STeresa Johnson Fn->setMetadata( 108081bbf742STeresa Johnson "thinlto_src_module", 108181bbf742STeresa Johnson MDNode::get(DestModule.getContext(), 108281bbf742STeresa Johnson {MDString::get(DestModule.getContext(), 108381bbf742STeresa Johnson SrcModule->getSourceFileName())})); 108401e32130SMehdi Amini } 108581bbf742STeresa Johnson GlobalsToImport.insert(Fn); 108681bbf742STeresa Johnson } 108781bbf742STeresa Johnson } 108801e32130SMehdi Amini 108919ef4fadSMehdi Amini // Upgrade debug info after we're done materializing all the globals and we 109019ef4fadSMehdi Amini // have loaded all the required metadata! 109119ef4fadSMehdi Amini UpgradeDebugInfo(*SrcModule); 109219ef4fadSMehdi Amini 10937e88d0daSMehdi Amini // Link in the specified functions. 109401e32130SMehdi Amini if (renameModuleForThinLTO(*SrcModule, Index, &GlobalsToImport)) 10958d05185aSMehdi Amini return true; 10968d05185aSMehdi Amini 1097d29478f7STeresa Johnson if (PrintImports) { 1098d29478f7STeresa Johnson for (const auto *GV : GlobalsToImport) 1099d29478f7STeresa Johnson dbgs() << DestModule.getSourceFileName() << ": Import " << GV->getName() 1100d29478f7STeresa Johnson << " from " << SrcModule->getSourceFileName() << "\n"; 1101d29478f7STeresa Johnson } 1102d29478f7STeresa Johnson 11036d8f817fSPeter Collingbourne if (Mover.move(std::move(SrcModule), GlobalsToImport.getArrayRef(), 11046d8f817fSPeter Collingbourne [](GlobalValue &, IRMover::ValueAdder) {}, 1105e6fd9ff9SPeter Collingbourne /*IsPerformingImport=*/true)) 11067e88d0daSMehdi Amini report_fatal_error("Function Import: link error"); 11077e88d0daSMehdi Amini 110801e32130SMehdi Amini ImportedCount += GlobalsToImport.size(); 11096c475a75STeresa Johnson NumImportedModules++; 11107e88d0daSMehdi Amini } 1111e5a61917STeresa Johnson 111219e23874SEugene Leviant NumImportedFunctions += (ImportedCount - ImportedGVCount); 111319e23874SEugene Leviant NumImportedGlobalVars += ImportedGVCount; 1114d29478f7STeresa Johnson 1115d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount 1116d34e60caSNicola Zaghen << " functions for Module " 1117d34e60caSNicola Zaghen << DestModule.getModuleIdentifier() << "\n"); 1118d34e60caSNicola Zaghen LLVM_DEBUG(dbgs() << "Imported " << ImportedGVCount 111919e23874SEugene Leviant << " global variables for Module " 1120c8c55170SMehdi Amini << DestModule.getModuleIdentifier() << "\n"); 1121c8c55170SMehdi Amini return ImportedCount; 112242418abaSMehdi Amini } 112342418abaSMehdi Amini 1124598bd2a2SPeter Collingbourne static bool doImportingForModule(Module &M) { 1125598bd2a2SPeter Collingbourne if (SummaryFile.empty()) 1126598bd2a2SPeter Collingbourne report_fatal_error("error: -function-import requires -summary-file\n"); 11276de481a3SPeter Collingbourne Expected<std::unique_ptr<ModuleSummaryIndex>> IndexPtrOrErr = 11286de481a3SPeter Collingbourne getModuleSummaryIndexForFile(SummaryFile); 11296de481a3SPeter Collingbourne if (!IndexPtrOrErr) { 11306de481a3SPeter Collingbourne logAllUnhandledErrors(IndexPtrOrErr.takeError(), errs(), 11316de481a3SPeter Collingbourne "Error loading file '" + SummaryFile + "': "); 113242418abaSMehdi Amini return false; 113342418abaSMehdi Amini } 1134598bd2a2SPeter Collingbourne std::unique_ptr<ModuleSummaryIndex> Index = std::move(*IndexPtrOrErr); 113542418abaSMehdi Amini 1136c86af334STeresa Johnson // First step is collecting the import list. 1137c86af334STeresa Johnson FunctionImporter::ImportMapTy ImportList; 113881bbf742STeresa Johnson // If requested, simply import all functions in the index. This is used 113981bbf742STeresa Johnson // when testing distributed backend handling via the opt tool, when 114081bbf742STeresa Johnson // we have distributed indexes containing exactly the summaries to import. 114181bbf742STeresa Johnson if (ImportAllIndex) 114281bbf742STeresa Johnson ComputeCrossModuleImportForModuleFromIndex(M.getModuleIdentifier(), *Index, 114381bbf742STeresa Johnson ImportList); 114481bbf742STeresa Johnson else 1145c86af334STeresa Johnson ComputeCrossModuleImportForModule(M.getModuleIdentifier(), *Index, 1146c86af334STeresa Johnson ImportList); 114701e32130SMehdi Amini 11484fef68cbSTeresa Johnson // Conservatively mark all internal values as promoted. This interface is 11494fef68cbSTeresa Johnson // only used when doing importing via the function importing pass. The pass 11504fef68cbSTeresa Johnson // is only enabled when testing importing via the 'opt' tool, which does 11514fef68cbSTeresa Johnson // not do the ThinLink that would normally determine what values to promote. 11524fef68cbSTeresa Johnson for (auto &I : *Index) { 11539667b91bSPeter Collingbourne for (auto &S : I.second.SummaryList) { 11544fef68cbSTeresa Johnson if (GlobalValue::isLocalLinkage(S->linkage())) 11554fef68cbSTeresa Johnson S->setLinkage(GlobalValue::ExternalLinkage); 11564fef68cbSTeresa Johnson } 11574fef68cbSTeresa Johnson } 11584fef68cbSTeresa Johnson 115901e32130SMehdi Amini // Next we need to promote to global scope and rename any local values that 11601b00f2d9STeresa Johnson // are potentially exported to other modules. 116101e32130SMehdi Amini if (renameModuleForThinLTO(M, *Index, nullptr)) { 11621b00f2d9STeresa Johnson errs() << "Error renaming module\n"; 11631b00f2d9STeresa Johnson return false; 11641b00f2d9STeresa Johnson } 11651b00f2d9STeresa Johnson 116642418abaSMehdi Amini // Perform the import now. 1167d16c8065SMehdi Amini auto ModuleLoader = [&M](StringRef Identifier) { 1168d16c8065SMehdi Amini return loadFile(Identifier, M.getContext()); 1169d16c8065SMehdi Amini }; 11709d2bfc48SRafael Espindola FunctionImporter Importer(*Index, ModuleLoader); 117137e24591SPeter Collingbourne Expected<bool> Result = Importer.importFunctions(M, ImportList); 11727f00d0a1SPeter Collingbourne 11737f00d0a1SPeter Collingbourne // FIXME: Probably need to propagate Errors through the pass manager. 11747f00d0a1SPeter Collingbourne if (!Result) { 11757f00d0a1SPeter Collingbourne logAllUnhandledErrors(Result.takeError(), errs(), 11767f00d0a1SPeter Collingbourne "Error importing module: "); 11777f00d0a1SPeter Collingbourne return false; 11787f00d0a1SPeter Collingbourne } 11797f00d0a1SPeter Collingbourne 11807f00d0a1SPeter Collingbourne return *Result; 118121241571STeresa Johnson } 118221241571STeresa Johnson 118321241571STeresa Johnson namespace { 1184e9ea08a0SEugene Zelenko 118521241571STeresa Johnson /// Pass that performs cross-module function import provided a summary file. 118621241571STeresa Johnson class FunctionImportLegacyPass : public ModulePass { 118721241571STeresa Johnson public: 118821241571STeresa Johnson /// Pass identification, replacement for typeid 118921241571STeresa Johnson static char ID; 119021241571STeresa Johnson 1191e9ea08a0SEugene Zelenko explicit FunctionImportLegacyPass() : ModulePass(ID) {} 1192e9ea08a0SEugene Zelenko 119321241571STeresa Johnson /// Specify pass name for debug output 1194117296c0SMehdi Amini StringRef getPassName() const override { return "Function Importing"; } 119521241571STeresa Johnson 119621241571STeresa Johnson bool runOnModule(Module &M) override { 119721241571STeresa Johnson if (skipModule(M)) 119821241571STeresa Johnson return false; 119921241571STeresa Johnson 1200598bd2a2SPeter Collingbourne return doImportingForModule(M); 120142418abaSMehdi Amini } 120242418abaSMehdi Amini }; 1203e9ea08a0SEugene Zelenko 1204e9ea08a0SEugene Zelenko } // end anonymous namespace 120542418abaSMehdi Amini 120621241571STeresa Johnson PreservedAnalyses FunctionImportPass::run(Module &M, 1207fd03ac6aSSean Silva ModuleAnalysisManager &AM) { 1208598bd2a2SPeter Collingbourne if (!doImportingForModule(M)) 120921241571STeresa Johnson return PreservedAnalyses::all(); 121021241571STeresa Johnson 121121241571STeresa Johnson return PreservedAnalyses::none(); 121221241571STeresa Johnson } 121321241571STeresa Johnson 121421241571STeresa Johnson char FunctionImportLegacyPass::ID = 0; 121521241571STeresa Johnson INITIALIZE_PASS(FunctionImportLegacyPass, "function-import", 121642418abaSMehdi Amini "Summary Based Function Import", false, false) 121742418abaSMehdi Amini 121842418abaSMehdi Amini namespace llvm { 1219e9ea08a0SEugene Zelenko 1220598bd2a2SPeter Collingbourne Pass *createFunctionImportPass() { 1221598bd2a2SPeter Collingbourne return new FunctionImportLegacyPass(); 12225fcbdb71STeresa Johnson } 1223e9ea08a0SEugene Zelenko 1224e9ea08a0SEugene Zelenko } // end namespace llvm 1225