11398a32eSPeter Collingbourne //===- ThinLTOBitcodeWriter.cpp - Bitcode writing pass for ThinLTO --------===// 21398a32eSPeter Collingbourne // 32946cd70SChandler Carruth // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 42946cd70SChandler Carruth // See https://llvm.org/LICENSE.txt for license information. 52946cd70SChandler Carruth // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 61398a32eSPeter Collingbourne // 71398a32eSPeter Collingbourne //===----------------------------------------------------------------------===// 81398a32eSPeter Collingbourne 96b411418STim Shen #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" 10002c2d53SPeter Collingbourne #include "llvm/Analysis/BasicAliasAnalysis.h" 11384ca190SMehdi Amini #include "llvm/Analysis/ModuleSummaryAnalysis.h" 1294624acaSTeresa Johnson #include "llvm/Analysis/ProfileSummaryInfo.h" 131398a32eSPeter Collingbourne #include "llvm/Analysis/TypeMetadataUtils.h" 141398a32eSPeter Collingbourne #include "llvm/Bitcode/BitcodeWriter.h" 151398a32eSPeter Collingbourne #include "llvm/IR/Constants.h" 1628ffd326SPeter Collingbourne #include "llvm/IR/DebugInfo.h" 179ed8e0caSdfukalov #include "llvm/IR/Instructions.h" 181398a32eSPeter Collingbourne #include "llvm/IR/Intrinsics.h" 191398a32eSPeter Collingbourne #include "llvm/IR/Module.h" 201398a32eSPeter Collingbourne #include "llvm/IR/PassManager.h" 2105da2fe5SReid Kleckner #include "llvm/InitializePasses.h" 22230b2567SVlad Tsyrklevich #include "llvm/Object/ModuleSymbolTable.h" 231398a32eSPeter Collingbourne #include "llvm/Pass.h" 241398a32eSPeter Collingbourne #include "llvm/Support/ScopedPrinter.h" 250c6a4ff8STeresa Johnson #include "llvm/Support/raw_ostream.h" 260c6a4ff8STeresa Johnson #include "llvm/Transforms/IPO.h" 27002c2d53SPeter Collingbourne #include "llvm/Transforms/IPO/FunctionAttrs.h" 28d3704f67SGeorge Rimar #include "llvm/Transforms/IPO/FunctionImport.h" 290e497d15SPeter Collingbourne #include "llvm/Transforms/IPO/LowerTypeTests.h" 301398a32eSPeter Collingbourne #include "llvm/Transforms/Utils/Cloning.h" 31964f4663SEvgeniy Stepanov #include "llvm/Transforms/Utils/ModuleUtils.h" 321398a32eSPeter Collingbourne using namespace llvm; 331398a32eSPeter Collingbourne 341398a32eSPeter Collingbourne namespace { 351398a32eSPeter Collingbourne 367ce1c4daSSami Tolvanen // Determine if a promotion alias should be created for a symbol name. 377ce1c4daSSami Tolvanen static bool allowPromotionAlias(const std::string &Name) { 387ce1c4daSSami Tolvanen // Promotion aliases are used only in inline assembly. It's safe to 397ce1c4daSSami Tolvanen // simply skip unusual names. Subset of MCAsmInfo::isAcceptableChar() 407ce1c4daSSami Tolvanen // and MCAsmInfoXCOFF::isAcceptableChar(). 417ce1c4daSSami Tolvanen for (const char &C : Name) { 427ce1c4daSSami Tolvanen if (isAlnum(C) || C == '_' || C == '.') 437ce1c4daSSami Tolvanen continue; 447ce1c4daSSami Tolvanen return false; 457ce1c4daSSami Tolvanen } 467ce1c4daSSami Tolvanen return true; 477ce1c4daSSami Tolvanen } 487ce1c4daSSami Tolvanen 491398a32eSPeter Collingbourne // Promote each local-linkage entity defined by ExportM and used by ImportM by 501398a32eSPeter Collingbourne // changing visibility and appending the given ModuleId. 514d4ee93dSEvgeniy Stepanov void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId, 524d4ee93dSEvgeniy Stepanov SetVector<GlobalValue *> &PromoteExtra) { 534075ccc7SBob Haarman DenseMap<const Comdat *, Comdat *> RenamedComdats; 546b193966SPeter Collingbourne for (auto &ExportGV : ExportM.global_values()) { 551398a32eSPeter Collingbourne if (!ExportGV.hasLocalLinkage()) 566b193966SPeter Collingbourne continue; 571398a32eSPeter Collingbourne 584075ccc7SBob Haarman auto Name = ExportGV.getName(); 591f034226SPeter Collingbourne GlobalValue *ImportGV = nullptr; 601f034226SPeter Collingbourne if (!PromoteExtra.count(&ExportGV)) { 611f034226SPeter Collingbourne ImportGV = ImportM.getNamedValue(Name); 621f034226SPeter Collingbourne if (!ImportGV) 636b193966SPeter Collingbourne continue; 641f034226SPeter Collingbourne ImportGV->removeDeadConstantUsers(); 651f034226SPeter Collingbourne if (ImportGV->use_empty()) { 661f034226SPeter Collingbourne ImportGV->eraseFromParent(); 671f034226SPeter Collingbourne continue; 681f034226SPeter Collingbourne } 691f034226SPeter Collingbourne } 701398a32eSPeter Collingbourne 717ce1c4daSSami Tolvanen std::string OldName = Name.str(); 724075ccc7SBob Haarman std::string NewName = (Name + ModuleId).str(); 734075ccc7SBob Haarman 744075ccc7SBob Haarman if (const auto *C = ExportGV.getComdat()) 754075ccc7SBob Haarman if (C->getName() == Name) 764075ccc7SBob Haarman RenamedComdats.try_emplace(C, ExportM.getOrInsertComdat(NewName)); 771398a32eSPeter Collingbourne 781398a32eSPeter Collingbourne ExportGV.setName(NewName); 791398a32eSPeter Collingbourne ExportGV.setLinkage(GlobalValue::ExternalLinkage); 801398a32eSPeter Collingbourne ExportGV.setVisibility(GlobalValue::HiddenVisibility); 811398a32eSPeter Collingbourne 824d4ee93dSEvgeniy Stepanov if (ImportGV) { 831398a32eSPeter Collingbourne ImportGV->setName(NewName); 841398a32eSPeter Collingbourne ImportGV->setVisibility(GlobalValue::HiddenVisibility); 856b193966SPeter Collingbourne } 867ce1c4daSSami Tolvanen 877ce1c4daSSami Tolvanen if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) { 887ce1c4daSSami Tolvanen // Create a local alias with the original name to avoid breaking 897ce1c4daSSami Tolvanen // references from inline assembly. 907ce1c4daSSami Tolvanen std::string Alias = ".set " + OldName + "," + NewName + "\n"; 917ce1c4daSSami Tolvanen ExportM.appendModuleInlineAsm(Alias); 927ce1c4daSSami Tolvanen } 934d4ee93dSEvgeniy Stepanov } 944075ccc7SBob Haarman 954075ccc7SBob Haarman if (!RenamedComdats.empty()) 964075ccc7SBob Haarman for (auto &GO : ExportM.global_objects()) 974075ccc7SBob Haarman if (auto *C = GO.getComdat()) { 984075ccc7SBob Haarman auto Replacement = RenamedComdats.find(C); 994075ccc7SBob Haarman if (Replacement != RenamedComdats.end()) 1004075ccc7SBob Haarman GO.setComdat(Replacement->second); 1014075ccc7SBob Haarman } 1021398a32eSPeter Collingbourne } 1031398a32eSPeter Collingbourne 1041398a32eSPeter Collingbourne // Promote all internal (i.e. distinct) type ids used by the module by replacing 1051398a32eSPeter Collingbourne // them with external type ids formed using the module id. 1061398a32eSPeter Collingbourne // 1071398a32eSPeter Collingbourne // Note that this needs to be done before we clone the module because each clone 1081398a32eSPeter Collingbourne // will receive its own set of distinct metadata nodes. 1091398a32eSPeter Collingbourne void promoteTypeIds(Module &M, StringRef ModuleId) { 1101398a32eSPeter Collingbourne DenseMap<Metadata *, Metadata *> LocalToGlobal; 1111398a32eSPeter Collingbourne auto ExternalizeTypeId = [&](CallInst *CI, unsigned ArgNo) { 1121398a32eSPeter Collingbourne Metadata *MD = 1131398a32eSPeter Collingbourne cast<MetadataAsValue>(CI->getArgOperand(ArgNo))->getMetadata(); 1141398a32eSPeter Collingbourne 1151398a32eSPeter Collingbourne if (isa<MDNode>(MD) && cast<MDNode>(MD)->isDistinct()) { 1161398a32eSPeter Collingbourne Metadata *&GlobalMD = LocalToGlobal[MD]; 1171398a32eSPeter Collingbourne if (!GlobalMD) { 1183a13ed60SBenjamin Kramer std::string NewName = (Twine(LocalToGlobal.size()) + ModuleId).str(); 1191398a32eSPeter Collingbourne GlobalMD = MDString::get(M.getContext(), NewName); 1201398a32eSPeter Collingbourne } 1211398a32eSPeter Collingbourne 1221398a32eSPeter Collingbourne CI->setArgOperand(ArgNo, 1231398a32eSPeter Collingbourne MetadataAsValue::get(M.getContext(), GlobalMD)); 1241398a32eSPeter Collingbourne } 1251398a32eSPeter Collingbourne }; 1261398a32eSPeter Collingbourne 1271398a32eSPeter Collingbourne if (Function *TypeTestFunc = 1281398a32eSPeter Collingbourne M.getFunction(Intrinsic::getName(Intrinsic::type_test))) { 1291398a32eSPeter Collingbourne for (const Use &U : TypeTestFunc->uses()) { 1301398a32eSPeter Collingbourne auto CI = cast<CallInst>(U.getUser()); 1311398a32eSPeter Collingbourne ExternalizeTypeId(CI, 1); 1321398a32eSPeter Collingbourne } 1331398a32eSPeter Collingbourne } 1341398a32eSPeter Collingbourne 1351398a32eSPeter Collingbourne if (Function *TypeCheckedLoadFunc = 1361398a32eSPeter Collingbourne M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) { 1371398a32eSPeter Collingbourne for (const Use &U : TypeCheckedLoadFunc->uses()) { 1381398a32eSPeter Collingbourne auto CI = cast<CallInst>(U.getUser()); 1391398a32eSPeter Collingbourne ExternalizeTypeId(CI, 2); 1401398a32eSPeter Collingbourne } 1411398a32eSPeter Collingbourne } 1421398a32eSPeter Collingbourne 1431398a32eSPeter Collingbourne for (GlobalObject &GO : M.global_objects()) { 1441398a32eSPeter Collingbourne SmallVector<MDNode *, 1> MDs; 1451398a32eSPeter Collingbourne GO.getMetadata(LLVMContext::MD_type, MDs); 1461398a32eSPeter Collingbourne 1471398a32eSPeter Collingbourne GO.eraseMetadata(LLVMContext::MD_type); 1481398a32eSPeter Collingbourne for (auto MD : MDs) { 1491398a32eSPeter Collingbourne auto I = LocalToGlobal.find(MD->getOperand(1)); 1501398a32eSPeter Collingbourne if (I == LocalToGlobal.end()) { 1511398a32eSPeter Collingbourne GO.addMetadata(LLVMContext::MD_type, *MD); 1521398a32eSPeter Collingbourne continue; 1531398a32eSPeter Collingbourne } 1541398a32eSPeter Collingbourne GO.addMetadata( 1551398a32eSPeter Collingbourne LLVMContext::MD_type, 1560deb9a9aSBenjamin Kramer *MDNode::get(M.getContext(), {MD->getOperand(0), I->second})); 1571398a32eSPeter Collingbourne } 1581398a32eSPeter Collingbourne } 1591398a32eSPeter Collingbourne } 1601398a32eSPeter Collingbourne 1611398a32eSPeter Collingbourne // Drop unused globals, and drop type information from function declarations. 1621398a32eSPeter Collingbourne // FIXME: If we made functions typeless then there would be no need to do this. 1631398a32eSPeter Collingbourne void simplifyExternals(Module &M) { 1641398a32eSPeter Collingbourne FunctionType *EmptyFT = 1651398a32eSPeter Collingbourne FunctionType::get(Type::getVoidTy(M.getContext()), false); 1661398a32eSPeter Collingbourne 167d9e46beaSKazu Hirata for (Function &F : llvm::make_early_inc_range(M)) { 1681398a32eSPeter Collingbourne if (F.isDeclaration() && F.use_empty()) { 1691398a32eSPeter Collingbourne F.eraseFromParent(); 1701398a32eSPeter Collingbourne continue; 1711398a32eSPeter Collingbourne } 1721398a32eSPeter Collingbourne 17393fdaca5SPeter Collingbourne if (!F.isDeclaration() || F.getFunctionType() == EmptyFT || 17493fdaca5SPeter Collingbourne // Changing the type of an intrinsic may invalidate the IR. 17593fdaca5SPeter Collingbourne F.getName().startswith("llvm.")) 1761398a32eSPeter Collingbourne continue; 1771398a32eSPeter Collingbourne 1781398a32eSPeter Collingbourne Function *NewF = 179f920da00SDylan McKay Function::Create(EmptyFT, GlobalValue::ExternalLinkage, 180f920da00SDylan McKay F.getAddressSpace(), "", &M); 181e28435caSZequan Wu NewF->copyAttributesFrom(&F); 182e28435caSZequan Wu // Only copy function attribtues. 18380ea2bb5SArthur Eubanks NewF->setAttributes(AttributeList::get(M.getContext(), 18480ea2bb5SArthur Eubanks AttributeList::FunctionIndex, 18580ea2bb5SArthur Eubanks F.getAttributes().getFnAttrs())); 1861398a32eSPeter Collingbourne NewF->takeName(&F); 1871398a32eSPeter Collingbourne F.replaceAllUsesWith(ConstantExpr::getBitCast(NewF, F.getType())); 1881398a32eSPeter Collingbourne F.eraseFromParent(); 1891398a32eSPeter Collingbourne } 1901398a32eSPeter Collingbourne 191*24c8eaecSKazu Hirata for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) { 1921398a32eSPeter Collingbourne if (GV.isDeclaration() && GV.use_empty()) { 1931398a32eSPeter Collingbourne GV.eraseFromParent(); 1941398a32eSPeter Collingbourne continue; 1951398a32eSPeter Collingbourne } 1961398a32eSPeter Collingbourne } 1971398a32eSPeter Collingbourne } 1981398a32eSPeter Collingbourne 199d3704f67SGeorge Rimar static void 200d3704f67SGeorge Rimar filterModule(Module *M, 201d3704f67SGeorge Rimar function_ref<bool(const GlobalValue *)> ShouldKeepDefinition) { 202d3704f67SGeorge Rimar std::vector<GlobalValue *> V; 203d3704f67SGeorge Rimar for (GlobalValue &GV : M->global_values()) 204d3704f67SGeorge Rimar if (!ShouldKeepDefinition(&GV)) 205d3704f67SGeorge Rimar V.push_back(&GV); 20654565249SGeorge Rimar 207d3704f67SGeorge Rimar for (GlobalValue *GV : V) 208d3704f67SGeorge Rimar if (!convertToDeclaration(*GV)) 209d3704f67SGeorge Rimar GV->eraseFromParent(); 2101398a32eSPeter Collingbourne } 2111398a32eSPeter Collingbourne 212002c2d53SPeter Collingbourne void forEachVirtualFunction(Constant *C, function_ref<void(Function *)> Fn) { 213002c2d53SPeter Collingbourne if (auto *F = dyn_cast<Function>(C)) 214002c2d53SPeter Collingbourne return Fn(F); 2153baa72afSPeter Collingbourne if (isa<GlobalValue>(C)) 2163baa72afSPeter Collingbourne return; 217002c2d53SPeter Collingbourne for (Value *Op : C->operands()) 218002c2d53SPeter Collingbourne forEachVirtualFunction(cast<Constant>(Op), Fn); 219002c2d53SPeter Collingbourne } 220002c2d53SPeter Collingbourne 221fde55a9cSTeresa Johnson // Clone any @llvm[.compiler].used over to the new module and append 222fde55a9cSTeresa Johnson // values whose defs were cloned into that module. 223fde55a9cSTeresa Johnson static void cloneUsedGlobalVariables(const Module &SrcM, Module &DestM, 224fde55a9cSTeresa Johnson bool CompilerUsed) { 2253adb89bbSFangrui Song SmallVector<GlobalValue *, 4> Used, NewUsed; 226fde55a9cSTeresa Johnson // First collect those in the llvm[.compiler].used set. 227fde55a9cSTeresa Johnson collectUsedGlobalVariables(SrcM, Used, CompilerUsed); 228fde55a9cSTeresa Johnson // Next build a set of the equivalent values defined in DestM. 229fde55a9cSTeresa Johnson for (auto *V : Used) { 230fde55a9cSTeresa Johnson auto *GV = DestM.getNamedValue(V->getName()); 231fde55a9cSTeresa Johnson if (GV && !GV->isDeclaration()) 2323adb89bbSFangrui Song NewUsed.push_back(GV); 233fde55a9cSTeresa Johnson } 234fde55a9cSTeresa Johnson // Finally, add them to a llvm[.compiler].used variable in DestM. 235fde55a9cSTeresa Johnson if (CompilerUsed) 2363adb89bbSFangrui Song appendToCompilerUsed(DestM, NewUsed); 237fde55a9cSTeresa Johnson else 2383adb89bbSFangrui Song appendToUsed(DestM, NewUsed); 239fde55a9cSTeresa Johnson } 240fde55a9cSTeresa Johnson 2411398a32eSPeter Collingbourne // If it's possible to split M into regular and thin LTO parts, do so and write 2421398a32eSPeter Collingbourne // a multi-module bitcode file with the two parts to OS. Otherwise, write only a 2431398a32eSPeter Collingbourne // regular LTO bitcode file to OS. 244002c2d53SPeter Collingbourne void splitAndWriteThinLTOBitcode( 2450c6a4ff8STeresa Johnson raw_ostream &OS, raw_ostream *ThinLinkOS, 2460c6a4ff8STeresa Johnson function_ref<AAResults &(Function &)> AARGetter, Module &M) { 247964f4663SEvgeniy Stepanov std::string ModuleId = getUniqueModuleId(&M); 2481398a32eSPeter Collingbourne if (ModuleId.empty()) { 2496867ab7cSVlad Tsyrklevich // We couldn't generate a module ID for this module, write it out as a 2506867ab7cSVlad Tsyrklevich // regular LTO module with an index for summary-based dead stripping. 2516867ab7cSVlad Tsyrklevich ProfileSummaryInfo PSI(M); 2526867ab7cSVlad Tsyrklevich M.addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); 2536867ab7cSVlad Tsyrklevich ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI); 2546867ab7cSVlad Tsyrklevich WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, &Index); 2556867ab7cSVlad Tsyrklevich 2560c6a4ff8STeresa Johnson if (ThinLinkOS) 2570c6a4ff8STeresa Johnson // We don't have a ThinLTO part, but still write the module to the 2580c6a4ff8STeresa Johnson // ThinLinkOS if requested so that the expected output file is produced. 2596867ab7cSVlad Tsyrklevich WriteBitcodeToFile(M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false, 2606867ab7cSVlad Tsyrklevich &Index); 2616867ab7cSVlad Tsyrklevich 2621398a32eSPeter Collingbourne return; 2631398a32eSPeter Collingbourne } 2641398a32eSPeter Collingbourne 2651398a32eSPeter Collingbourne promoteTypeIds(M, ModuleId); 2661398a32eSPeter Collingbourne 267dd968219SPeter Collingbourne // Returns whether a global or its associated global has attached type 268dd968219SPeter Collingbourne // metadata. The former may participate in CFI or whole-program 269dd968219SPeter Collingbourne // devirtualization, so they need to appear in the merged module instead of 270dd968219SPeter Collingbourne // the thin LTO module. Similarly, globals that are associated with globals 271dd968219SPeter Collingbourne // with type metadata need to appear in the merged module because they will 272dd968219SPeter Collingbourne // reference the global's section directly. 2730deb9a9aSBenjamin Kramer auto HasTypeMetadata = [](const GlobalObject *GO) { 274dd968219SPeter Collingbourne if (MDNode *MD = GO->getMetadata(LLVMContext::MD_associated)) 275dd968219SPeter Collingbourne if (auto *AssocVM = dyn_cast_or_null<ValueAsMetadata>(MD->getOperand(0))) 276dd968219SPeter Collingbourne if (auto *AssocGO = dyn_cast<GlobalObject>(AssocVM->getValue())) 277dd968219SPeter Collingbourne if (AssocGO->hasMetadata(LLVMContext::MD_type)) 278dd968219SPeter Collingbourne return true; 2790deb9a9aSBenjamin Kramer return GO->hasMetadata(LLVMContext::MD_type); 2801398a32eSPeter Collingbourne }; 2811398a32eSPeter Collingbourne 282002c2d53SPeter Collingbourne // Collect the set of virtual functions that are eligible for virtual constant 283002c2d53SPeter Collingbourne // propagation. Each eligible function must not access memory, must return 284002c2d53SPeter Collingbourne // an integer of width <=64 bits, must take at least one argument, must not 285002c2d53SPeter Collingbourne // use its first argument (assumed to be "this") and all arguments other than 286002c2d53SPeter Collingbourne // the first one must be of <=64 bit integer type. 287002c2d53SPeter Collingbourne // 288002c2d53SPeter Collingbourne // Note that we test whether this copy of the function is readnone, rather 289002c2d53SPeter Collingbourne // than testing function attributes, which must hold for any copy of the 290002c2d53SPeter Collingbourne // function, even a less optimized version substituted at link time. This is 291002c2d53SPeter Collingbourne // sound because the virtual constant propagation optimizations effectively 292002c2d53SPeter Collingbourne // inline all implementations of the virtual function into each call site, 293002c2d53SPeter Collingbourne // rather than using function attributes to perform local optimization. 294aa09a82bSGeorge Burgess IV DenseSet<const Function *> EligibleVirtualFns; 2954075ccc7SBob Haarman // If any member of a comdat lives in MergedM, put all members of that 2964075ccc7SBob Haarman // comdat in MergedM to keep the comdat together. 2974075ccc7SBob Haarman DenseSet<const Comdat *> MergedMComdats; 298002c2d53SPeter Collingbourne for (GlobalVariable &GV : M.globals()) 2994075ccc7SBob Haarman if (HasTypeMetadata(&GV)) { 3004075ccc7SBob Haarman if (const auto *C = GV.getComdat()) 3014075ccc7SBob Haarman MergedMComdats.insert(C); 302002c2d53SPeter Collingbourne forEachVirtualFunction(GV.getInitializer(), [&](Function *F) { 303002c2d53SPeter Collingbourne auto *RT = dyn_cast<IntegerType>(F->getReturnType()); 304002c2d53SPeter Collingbourne if (!RT || RT->getBitWidth() > 64 || F->arg_empty() || 305002c2d53SPeter Collingbourne !F->arg_begin()->use_empty()) 306002c2d53SPeter Collingbourne return; 30723b0ab2aSKazu Hirata for (auto &Arg : drop_begin(F->args())) { 308002c2d53SPeter Collingbourne auto *ArgT = dyn_cast<IntegerType>(Arg.getType()); 309002c2d53SPeter Collingbourne if (!ArgT || ArgT->getBitWidth() > 64) 310002c2d53SPeter Collingbourne return; 311002c2d53SPeter Collingbourne } 31201f0c8a8SChandler Carruth if (!F->isDeclaration() && 31301f0c8a8SChandler Carruth computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == MAK_ReadNone) 314002c2d53SPeter Collingbourne EligibleVirtualFns.insert(F); 315002c2d53SPeter Collingbourne }); 3164075ccc7SBob Haarman } 317002c2d53SPeter Collingbourne 3181398a32eSPeter Collingbourne ValueToValueMapTy VMap; 319002c2d53SPeter Collingbourne std::unique_ptr<Module> MergedM( 32071867532SRafael Espindola CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool { 3214075ccc7SBob Haarman if (const auto *C = GV->getComdat()) 3224075ccc7SBob Haarman if (MergedMComdats.count(C)) 3234075ccc7SBob Haarman return true; 324002c2d53SPeter Collingbourne if (auto *F = dyn_cast<Function>(GV)) 325002c2d53SPeter Collingbourne return EligibleVirtualFns.count(F); 326002c2d53SPeter Collingbourne if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getBaseObject())) 327002c2d53SPeter Collingbourne return HasTypeMetadata(GVar); 328002c2d53SPeter Collingbourne return false; 329002c2d53SPeter Collingbourne })); 33028ffd326SPeter Collingbourne StripDebugInfo(*MergedM); 33129c6f483SPeter Collingbourne MergedM->setModuleInlineAsm(""); 3321398a32eSPeter Collingbourne 333fde55a9cSTeresa Johnson // Clone any llvm.*used globals to ensure the included values are 334fde55a9cSTeresa Johnson // not deleted. 335fde55a9cSTeresa Johnson cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ false); 336fde55a9cSTeresa Johnson cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true); 337fde55a9cSTeresa Johnson 338002c2d53SPeter Collingbourne for (Function &F : *MergedM) 339002c2d53SPeter Collingbourne if (!F.isDeclaration()) { 340002c2d53SPeter Collingbourne // Reset the linkage of all functions eligible for virtual constant 341002c2d53SPeter Collingbourne // propagation. The canonical definitions live in the thin LTO module so 342002c2d53SPeter Collingbourne // that they can be imported. 343002c2d53SPeter Collingbourne F.setLinkage(GlobalValue::AvailableExternallyLinkage); 344002c2d53SPeter Collingbourne F.setComdat(nullptr); 345002c2d53SPeter Collingbourne } 346002c2d53SPeter Collingbourne 3474d4ee93dSEvgeniy Stepanov SetVector<GlobalValue *> CfiFunctions; 3484d4ee93dSEvgeniy Stepanov for (auto &F : M) 3494d4ee93dSEvgeniy Stepanov if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F)) 3504d4ee93dSEvgeniy Stepanov CfiFunctions.insert(&F); 3514d4ee93dSEvgeniy Stepanov 3524075ccc7SBob Haarman // Remove all globals with type metadata, globals with comdats that live in 3534075ccc7SBob Haarman // MergedM, and aliases pointing to such globals from the thin LTO module. 354002c2d53SPeter Collingbourne filterModule(&M, [&](const GlobalValue *GV) { 355002c2d53SPeter Collingbourne if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getBaseObject())) 3564075ccc7SBob Haarman if (HasTypeMetadata(GVar)) 3574075ccc7SBob Haarman return false; 3584075ccc7SBob Haarman if (const auto *C = GV->getComdat()) 3594075ccc7SBob Haarman if (MergedMComdats.count(C)) 3604075ccc7SBob Haarman return false; 361002c2d53SPeter Collingbourne return true; 362002c2d53SPeter Collingbourne }); 3631398a32eSPeter Collingbourne 3644d4ee93dSEvgeniy Stepanov promoteInternals(*MergedM, M, ModuleId, CfiFunctions); 3654d4ee93dSEvgeniy Stepanov promoteInternals(M, *MergedM, ModuleId, CfiFunctions); 3664d4ee93dSEvgeniy Stepanov 367230b2567SVlad Tsyrklevich auto &Ctx = MergedM->getContext(); 3684d4ee93dSEvgeniy Stepanov SmallVector<MDNode *, 8> CfiFunctionMDs; 3694d4ee93dSEvgeniy Stepanov for (auto V : CfiFunctions) { 3704d4ee93dSEvgeniy Stepanov Function &F = *cast<Function>(V); 3714d4ee93dSEvgeniy Stepanov SmallVector<MDNode *, 2> Types; 3724d4ee93dSEvgeniy Stepanov F.getMetadata(LLVMContext::MD_type, Types); 3734d4ee93dSEvgeniy Stepanov 3744d4ee93dSEvgeniy Stepanov SmallVector<Metadata *, 4> Elts; 3754d4ee93dSEvgeniy Stepanov Elts.push_back(MDString::get(Ctx, F.getName())); 3764d4ee93dSEvgeniy Stepanov CfiFunctionLinkage Linkage; 3770e497d15SPeter Collingbourne if (lowertypetests::isJumpTableCanonical(&F)) 3784d4ee93dSEvgeniy Stepanov Linkage = CFL_Definition; 3790e497d15SPeter Collingbourne else if (F.hasExternalWeakLinkage()) 3804d4ee93dSEvgeniy Stepanov Linkage = CFL_WeakDeclaration; 3814d4ee93dSEvgeniy Stepanov else 3824d4ee93dSEvgeniy Stepanov Linkage = CFL_Declaration; 3834d4ee93dSEvgeniy Stepanov Elts.push_back(ConstantAsMetadata::get( 3844d4ee93dSEvgeniy Stepanov llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage))); 385e53472deSKazu Hirata append_range(Elts, Types); 3864d4ee93dSEvgeniy Stepanov CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts)); 3874d4ee93dSEvgeniy Stepanov } 3884d4ee93dSEvgeniy Stepanov 3894d4ee93dSEvgeniy Stepanov if(!CfiFunctionMDs.empty()) { 3904d4ee93dSEvgeniy Stepanov NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions"); 3914d4ee93dSEvgeniy Stepanov for (auto MD : CfiFunctionMDs) 3924d4ee93dSEvgeniy Stepanov NMD->addOperand(MD); 3934d4ee93dSEvgeniy Stepanov } 3941398a32eSPeter Collingbourne 395cdec22efSVlad Tsyrklevich SmallVector<MDNode *, 8> FunctionAliases; 396cdec22efSVlad Tsyrklevich for (auto &A : M.aliases()) { 397cdec22efSVlad Tsyrklevich if (!isa<Function>(A.getAliasee())) 398cdec22efSVlad Tsyrklevich continue; 399cdec22efSVlad Tsyrklevich 400cdec22efSVlad Tsyrklevich auto *F = cast<Function>(A.getAliasee()); 401cdec22efSVlad Tsyrklevich 4020deb9a9aSBenjamin Kramer Metadata *Elts[] = { 4030deb9a9aSBenjamin Kramer MDString::get(Ctx, A.getName()), 4040deb9a9aSBenjamin Kramer MDString::get(Ctx, F->getName()), 4050deb9a9aSBenjamin Kramer ConstantAsMetadata::get( 4060deb9a9aSBenjamin Kramer ConstantInt::get(Type::getInt8Ty(Ctx), A.getVisibility())), 4070deb9a9aSBenjamin Kramer ConstantAsMetadata::get( 4080deb9a9aSBenjamin Kramer ConstantInt::get(Type::getInt8Ty(Ctx), A.isWeakForLinker())), 4090deb9a9aSBenjamin Kramer }; 410cdec22efSVlad Tsyrklevich 411cdec22efSVlad Tsyrklevich FunctionAliases.push_back(MDTuple::get(Ctx, Elts)); 412cdec22efSVlad Tsyrklevich } 413cdec22efSVlad Tsyrklevich 414cdec22efSVlad Tsyrklevich if (!FunctionAliases.empty()) { 415cdec22efSVlad Tsyrklevich NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("aliases"); 416cdec22efSVlad Tsyrklevich for (auto MD : FunctionAliases) 417cdec22efSVlad Tsyrklevich NMD->addOperand(MD); 418cdec22efSVlad Tsyrklevich } 419cdec22efSVlad Tsyrklevich 420230b2567SVlad Tsyrklevich SmallVector<MDNode *, 8> Symvers; 421230b2567SVlad Tsyrklevich ModuleSymbolTable::CollectAsmSymvers(M, [&](StringRef Name, StringRef Alias) { 422230b2567SVlad Tsyrklevich Function *F = M.getFunction(Name); 423230b2567SVlad Tsyrklevich if (!F || F->use_empty()) 424230b2567SVlad Tsyrklevich return; 425230b2567SVlad Tsyrklevich 4260deb9a9aSBenjamin Kramer Symvers.push_back(MDTuple::get( 4270deb9a9aSBenjamin Kramer Ctx, {MDString::get(Ctx, Name), MDString::get(Ctx, Alias)})); 428230b2567SVlad Tsyrklevich }); 429230b2567SVlad Tsyrklevich 430230b2567SVlad Tsyrklevich if (!Symvers.empty()) { 431230b2567SVlad Tsyrklevich NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("symvers"); 432230b2567SVlad Tsyrklevich for (auto MD : Symvers) 433230b2567SVlad Tsyrklevich NMD->addOperand(MD); 434230b2567SVlad Tsyrklevich } 435230b2567SVlad Tsyrklevich 4361398a32eSPeter Collingbourne simplifyExternals(*MergedM); 4371398a32eSPeter Collingbourne 4381398a32eSPeter Collingbourne // FIXME: Try to re-use BSI and PFI from the original module here. 43994624acaSTeresa Johnson ProfileSummaryInfo PSI(M); 44094624acaSTeresa Johnson ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI); 4410c6a4ff8STeresa Johnson 442e357fbd2SPeter Collingbourne // Mark the merged module as requiring full LTO. We still want an index for 443e357fbd2SPeter Collingbourne // it though, so that it can participate in summary-based dead stripping. 444e357fbd2SPeter Collingbourne MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); 445e357fbd2SPeter Collingbourne ModuleSummaryIndex MergedMIndex = 446e357fbd2SPeter Collingbourne buildModuleSummaryIndex(*MergedM, nullptr, &PSI); 447e357fbd2SPeter Collingbourne 4480c6a4ff8STeresa Johnson SmallVector<char, 0> Buffer; 4490c6a4ff8STeresa Johnson 4500c6a4ff8STeresa Johnson BitcodeWriter W(Buffer); 4510c6a4ff8STeresa Johnson // Save the module hash produced for the full bitcode, which will 4520c6a4ff8STeresa Johnson // be used in the backends, and use that in the minimized bitcode 4530c6a4ff8STeresa Johnson // produced for the full link. 4540c6a4ff8STeresa Johnson ModuleHash ModHash = {{0}}; 4556a86e25dSRafael Espindola W.writeModule(M, /*ShouldPreserveUseListOrder=*/false, &Index, 4560c6a4ff8STeresa Johnson /*GenerateHash=*/true, &ModHash); 4576a86e25dSRafael Espindola W.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, &MergedMIndex); 45892648c25SPeter Collingbourne W.writeSymtab(); 459a0f371a1SPeter Collingbourne W.writeStrtab(); 4601398a32eSPeter Collingbourne OS << Buffer; 4610c6a4ff8STeresa Johnson 4621dec57d5SHaojie Wang // If a minimized bitcode module was requested for the thin link, only 4631dec57d5SHaojie Wang // the information that is needed by thin link will be written in the 4641dec57d5SHaojie Wang // given OS (the merged module will be written as usual). 4650c6a4ff8STeresa Johnson if (ThinLinkOS) { 4660c6a4ff8STeresa Johnson Buffer.clear(); 4670c6a4ff8STeresa Johnson BitcodeWriter W2(Buffer); 4680c6a4ff8STeresa Johnson StripDebugInfo(M); 4696a86e25dSRafael Espindola W2.writeThinLinkBitcode(M, Index, ModHash); 4706a86e25dSRafael Espindola W2.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, 471e357fbd2SPeter Collingbourne &MergedMIndex); 47292648c25SPeter Collingbourne W2.writeSymtab(); 473a0f371a1SPeter Collingbourne W2.writeStrtab(); 4740c6a4ff8STeresa Johnson *ThinLinkOS << Buffer; 4750c6a4ff8STeresa Johnson } 4761398a32eSPeter Collingbourne } 4771398a32eSPeter Collingbourne 478a7004363STeresa Johnson // Check if the LTO Unit splitting has been enabled. 479a7004363STeresa Johnson bool enableSplitLTOUnit(Module &M) { 480290a8398STeresa Johnson bool EnableSplitLTOUnit = false; 481290a8398STeresa Johnson if (auto *MD = mdconst::extract_or_null<ConstantInt>( 482290a8398STeresa Johnson M.getModuleFlag("EnableSplitLTOUnit"))) 483290a8398STeresa Johnson EnableSplitLTOUnit = MD->getZExtValue(); 484a7004363STeresa Johnson return EnableSplitLTOUnit; 485a7004363STeresa Johnson } 486290a8398STeresa Johnson 487a7004363STeresa Johnson // Returns whether this module needs to be split because it uses type metadata. 488a7004363STeresa Johnson bool hasTypeMetadata(Module &M) { 4891398a32eSPeter Collingbourne for (auto &GO : M.global_objects()) { 4900deb9a9aSBenjamin Kramer if (GO.hasMetadata(LLVMContext::MD_type)) 4911398a32eSPeter Collingbourne return true; 4921398a32eSPeter Collingbourne } 4931398a32eSPeter Collingbourne return false; 4941398a32eSPeter Collingbourne } 4951398a32eSPeter Collingbourne 4960c6a4ff8STeresa Johnson void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, 497002c2d53SPeter Collingbourne function_ref<AAResults &(Function &)> AARGetter, 498002c2d53SPeter Collingbourne Module &M, const ModuleSummaryIndex *Index) { 499a7004363STeresa Johnson std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr; 500a7004363STeresa Johnson // See if this module has any type metadata. If so, we try to split it 501a7004363STeresa Johnson // or at least promote type ids to enable WPD. 502a7004363STeresa Johnson if (hasTypeMetadata(M)) { 503a7004363STeresa Johnson if (enableSplitLTOUnit(M)) 5040c6a4ff8STeresa Johnson return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M); 505a7004363STeresa Johnson // Promote type ids as needed for index-based WPD. 506a7004363STeresa Johnson std::string ModuleId = getUniqueModuleId(&M); 507a7004363STeresa Johnson if (!ModuleId.empty()) { 508a7004363STeresa Johnson promoteTypeIds(M, ModuleId); 509a7004363STeresa Johnson // Need to rebuild the index so that it contains type metadata 510a7004363STeresa Johnson // for the newly promoted type ids. 511a7004363STeresa Johnson // FIXME: Probably should not bother building the index at all 512a7004363STeresa Johnson // in the caller of writeThinLTOBitcode (which does so via the 513a7004363STeresa Johnson // ModuleSummaryIndexAnalysis pass), since we have to rebuild it 514a7004363STeresa Johnson // anyway whenever there is type metadata (here or in 515a7004363STeresa Johnson // splitAndWriteThinLTOBitcode). Just always build it once via the 516a7004363STeresa Johnson // buildModuleSummaryIndex when Module(s) are ready. 517a7004363STeresa Johnson ProfileSummaryInfo PSI(M); 5180eaee545SJonas Devlieghere NewIndex = std::make_unique<ModuleSummaryIndex>( 519a7004363STeresa Johnson buildModuleSummaryIndex(M, nullptr, &PSI)); 520a7004363STeresa Johnson Index = NewIndex.get(); 521a7004363STeresa Johnson } 522a7004363STeresa Johnson } 5231398a32eSPeter Collingbourne 524a7004363STeresa Johnson // Write it out as an unsplit ThinLTO module. 5250c6a4ff8STeresa Johnson 5260c6a4ff8STeresa Johnson // Save the module hash produced for the full bitcode, which will 5270c6a4ff8STeresa Johnson // be used in the backends, and use that in the minimized bitcode 5280c6a4ff8STeresa Johnson // produced for the full link. 5290c6a4ff8STeresa Johnson ModuleHash ModHash = {{0}}; 5306a86e25dSRafael Espindola WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, Index, 5310c6a4ff8STeresa Johnson /*GenerateHash=*/true, &ModHash); 5321dec57d5SHaojie Wang // If a minimized bitcode module was requested for the thin link, only 5331dec57d5SHaojie Wang // the information that is needed by thin link will be written in the 5341dec57d5SHaojie Wang // given OS. 5351dec57d5SHaojie Wang if (ThinLinkOS && Index) 5366a86e25dSRafael Espindola WriteThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash); 5371398a32eSPeter Collingbourne } 5381398a32eSPeter Collingbourne 5391398a32eSPeter Collingbourne class WriteThinLTOBitcode : public ModulePass { 5401398a32eSPeter Collingbourne raw_ostream &OS; // raw_ostream to print on 5410c6a4ff8STeresa Johnson // The output stream on which to emit a minimized module for use 5420c6a4ff8STeresa Johnson // just in the thin link, if requested. 5430c6a4ff8STeresa Johnson raw_ostream *ThinLinkOS; 5441398a32eSPeter Collingbourne 5451398a32eSPeter Collingbourne public: 5461398a32eSPeter Collingbourne static char ID; // Pass identification, replacement for typeid 5470c6a4ff8STeresa Johnson WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()), ThinLinkOS(nullptr) { 5481398a32eSPeter Collingbourne initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); 5491398a32eSPeter Collingbourne } 5501398a32eSPeter Collingbourne 5510c6a4ff8STeresa Johnson explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS) 5520c6a4ff8STeresa Johnson : ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) { 5531398a32eSPeter Collingbourne initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); 5541398a32eSPeter Collingbourne } 5551398a32eSPeter Collingbourne 5561398a32eSPeter Collingbourne StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; } 5571398a32eSPeter Collingbourne 5581398a32eSPeter Collingbourne bool runOnModule(Module &M) override { 5591398a32eSPeter Collingbourne const ModuleSummaryIndex *Index = 5601398a32eSPeter Collingbourne &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex()); 5610c6a4ff8STeresa Johnson writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index); 5621398a32eSPeter Collingbourne return true; 5631398a32eSPeter Collingbourne } 5641398a32eSPeter Collingbourne void getAnalysisUsage(AnalysisUsage &AU) const override { 5651398a32eSPeter Collingbourne AU.setPreservesAll(); 566002c2d53SPeter Collingbourne AU.addRequired<AssumptionCacheTracker>(); 5671398a32eSPeter Collingbourne AU.addRequired<ModuleSummaryIndexWrapperPass>(); 568002c2d53SPeter Collingbourne AU.addRequired<TargetLibraryInfoWrapperPass>(); 5691398a32eSPeter Collingbourne } 5701398a32eSPeter Collingbourne }; 5711398a32eSPeter Collingbourne } // anonymous namespace 5721398a32eSPeter Collingbourne 5731398a32eSPeter Collingbourne char WriteThinLTOBitcode::ID = 0; 5741398a32eSPeter Collingbourne INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode", 5751398a32eSPeter Collingbourne "Write ThinLTO Bitcode", false, true) 576002c2d53SPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 5771398a32eSPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass) 578002c2d53SPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 5791398a32eSPeter Collingbourne INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode", 5801398a32eSPeter Collingbourne "Write ThinLTO Bitcode", false, true) 5811398a32eSPeter Collingbourne 5820c6a4ff8STeresa Johnson ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str, 5830c6a4ff8STeresa Johnson raw_ostream *ThinLinkOS) { 5840c6a4ff8STeresa Johnson return new WriteThinLTOBitcode(Str, ThinLinkOS); 5851398a32eSPeter Collingbourne } 5866b411418STim Shen 5876b411418STim Shen PreservedAnalyses 5886b411418STim Shen llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { 5896b411418STim Shen FunctionAnalysisManager &FAM = 5906b411418STim Shen AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 5916b411418STim Shen writeThinLTOBitcode(OS, ThinLinkOS, 5926b411418STim Shen [&FAM](Function &F) -> AAResults & { 5936b411418STim Shen return FAM.getResult<AAManager>(F); 5946b411418STim Shen }, 5956b411418STim Shen M, &AM.getResult<ModuleSummaryIndexAnalysis>(M)); 5966b411418STim Shen return PreservedAnalyses::all(); 5976b411418STim Shen } 598