11398a32eSPeter Collingbourne //===- ThinLTOBitcodeWriter.cpp - Bitcode writing pass for ThinLTO --------===//
21398a32eSPeter Collingbourne //
32946cd70SChandler Carruth // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
42946cd70SChandler Carruth // See https://llvm.org/LICENSE.txt for license information.
52946cd70SChandler Carruth // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
61398a32eSPeter Collingbourne //
71398a32eSPeter Collingbourne //===----------------------------------------------------------------------===//
81398a32eSPeter Collingbourne
96b411418STim Shen #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
10002c2d53SPeter Collingbourne #include "llvm/Analysis/BasicAliasAnalysis.h"
11384ca190SMehdi Amini #include "llvm/Analysis/ModuleSummaryAnalysis.h"
1294624acaSTeresa Johnson #include "llvm/Analysis/ProfileSummaryInfo.h"
131398a32eSPeter Collingbourne #include "llvm/Analysis/TypeMetadataUtils.h"
141398a32eSPeter Collingbourne #include "llvm/Bitcode/BitcodeWriter.h"
151398a32eSPeter Collingbourne #include "llvm/IR/Constants.h"
1628ffd326SPeter Collingbourne #include "llvm/IR/DebugInfo.h"
179ed8e0caSdfukalov #include "llvm/IR/Instructions.h"
181398a32eSPeter Collingbourne #include "llvm/IR/Intrinsics.h"
191398a32eSPeter Collingbourne #include "llvm/IR/Module.h"
201398a32eSPeter Collingbourne #include "llvm/IR/PassManager.h"
2105da2fe5SReid Kleckner #include "llvm/InitializePasses.h"
22230b2567SVlad Tsyrklevich #include "llvm/Object/ModuleSymbolTable.h"
231398a32eSPeter Collingbourne #include "llvm/Pass.h"
240c6a4ff8STeresa Johnson #include "llvm/Support/raw_ostream.h"
250c6a4ff8STeresa Johnson #include "llvm/Transforms/IPO.h"
26002c2d53SPeter Collingbourne #include "llvm/Transforms/IPO/FunctionAttrs.h"
27d3704f67SGeorge Rimar #include "llvm/Transforms/IPO/FunctionImport.h"
280e497d15SPeter Collingbourne #include "llvm/Transforms/IPO/LowerTypeTests.h"
291398a32eSPeter Collingbourne #include "llvm/Transforms/Utils/Cloning.h"
30964f4663SEvgeniy Stepanov #include "llvm/Transforms/Utils/ModuleUtils.h"
311398a32eSPeter Collingbourne using namespace llvm;
321398a32eSPeter Collingbourne
331398a32eSPeter Collingbourne namespace {
341398a32eSPeter Collingbourne
357ce1c4daSSami Tolvanen // Determine if a promotion alias should be created for a symbol name.
allowPromotionAlias(const std::string & Name)367ce1c4daSSami Tolvanen static bool allowPromotionAlias(const std::string &Name) {
377ce1c4daSSami Tolvanen // Promotion aliases are used only in inline assembly. It's safe to
387ce1c4daSSami Tolvanen // simply skip unusual names. Subset of MCAsmInfo::isAcceptableChar()
397ce1c4daSSami Tolvanen // and MCAsmInfoXCOFF::isAcceptableChar().
407ce1c4daSSami Tolvanen for (const char &C : Name) {
417ce1c4daSSami Tolvanen if (isAlnum(C) || C == '_' || C == '.')
427ce1c4daSSami Tolvanen continue;
437ce1c4daSSami Tolvanen return false;
447ce1c4daSSami Tolvanen }
457ce1c4daSSami Tolvanen return true;
467ce1c4daSSami Tolvanen }
477ce1c4daSSami Tolvanen
481398a32eSPeter Collingbourne // Promote each local-linkage entity defined by ExportM and used by ImportM by
491398a32eSPeter Collingbourne // changing visibility and appending the given ModuleId.
promoteInternals(Module & ExportM,Module & ImportM,StringRef ModuleId,SetVector<GlobalValue * > & PromoteExtra)504d4ee93dSEvgeniy Stepanov void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId,
514d4ee93dSEvgeniy Stepanov SetVector<GlobalValue *> &PromoteExtra) {
524075ccc7SBob Haarman DenseMap<const Comdat *, Comdat *> RenamedComdats;
536b193966SPeter Collingbourne for (auto &ExportGV : ExportM.global_values()) {
541398a32eSPeter Collingbourne if (!ExportGV.hasLocalLinkage())
556b193966SPeter Collingbourne continue;
561398a32eSPeter Collingbourne
574075ccc7SBob Haarman auto Name = ExportGV.getName();
581f034226SPeter Collingbourne GlobalValue *ImportGV = nullptr;
591f034226SPeter Collingbourne if (!PromoteExtra.count(&ExportGV)) {
601f034226SPeter Collingbourne ImportGV = ImportM.getNamedValue(Name);
611f034226SPeter Collingbourne if (!ImportGV)
626b193966SPeter Collingbourne continue;
631f034226SPeter Collingbourne ImportGV->removeDeadConstantUsers();
641f034226SPeter Collingbourne if (ImportGV->use_empty()) {
651f034226SPeter Collingbourne ImportGV->eraseFromParent();
661f034226SPeter Collingbourne continue;
671f034226SPeter Collingbourne }
681f034226SPeter Collingbourne }
691398a32eSPeter Collingbourne
707ce1c4daSSami Tolvanen std::string OldName = Name.str();
714075ccc7SBob Haarman std::string NewName = (Name + ModuleId).str();
724075ccc7SBob Haarman
734075ccc7SBob Haarman if (const auto *C = ExportGV.getComdat())
744075ccc7SBob Haarman if (C->getName() == Name)
754075ccc7SBob Haarman RenamedComdats.try_emplace(C, ExportM.getOrInsertComdat(NewName));
761398a32eSPeter Collingbourne
771398a32eSPeter Collingbourne ExportGV.setName(NewName);
781398a32eSPeter Collingbourne ExportGV.setLinkage(GlobalValue::ExternalLinkage);
791398a32eSPeter Collingbourne ExportGV.setVisibility(GlobalValue::HiddenVisibility);
801398a32eSPeter Collingbourne
814d4ee93dSEvgeniy Stepanov if (ImportGV) {
821398a32eSPeter Collingbourne ImportGV->setName(NewName);
831398a32eSPeter Collingbourne ImportGV->setVisibility(GlobalValue::HiddenVisibility);
846b193966SPeter Collingbourne }
857ce1c4daSSami Tolvanen
867ce1c4daSSami Tolvanen if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) {
877ce1c4daSSami Tolvanen // Create a local alias with the original name to avoid breaking
887ce1c4daSSami Tolvanen // references from inline assembly.
899a74c753SSami Tolvanen std::string Alias =
909a74c753SSami Tolvanen ".lto_set_conditional " + OldName + "," + NewName + "\n";
917ce1c4daSSami Tolvanen ExportM.appendModuleInlineAsm(Alias);
927ce1c4daSSami Tolvanen }
934d4ee93dSEvgeniy Stepanov }
944075ccc7SBob Haarman
954075ccc7SBob Haarman if (!RenamedComdats.empty())
964075ccc7SBob Haarman for (auto &GO : ExportM.global_objects())
974075ccc7SBob Haarman if (auto *C = GO.getComdat()) {
984075ccc7SBob Haarman auto Replacement = RenamedComdats.find(C);
994075ccc7SBob Haarman if (Replacement != RenamedComdats.end())
1004075ccc7SBob Haarman GO.setComdat(Replacement->second);
1014075ccc7SBob Haarman }
1021398a32eSPeter Collingbourne }
1031398a32eSPeter Collingbourne
1041398a32eSPeter Collingbourne // Promote all internal (i.e. distinct) type ids used by the module by replacing
1051398a32eSPeter Collingbourne // them with external type ids formed using the module id.
1061398a32eSPeter Collingbourne //
1071398a32eSPeter Collingbourne // Note that this needs to be done before we clone the module because each clone
1081398a32eSPeter Collingbourne // will receive its own set of distinct metadata nodes.
promoteTypeIds(Module & M,StringRef ModuleId)1091398a32eSPeter Collingbourne void promoteTypeIds(Module &M, StringRef ModuleId) {
1101398a32eSPeter Collingbourne DenseMap<Metadata *, Metadata *> LocalToGlobal;
1111398a32eSPeter Collingbourne auto ExternalizeTypeId = [&](CallInst *CI, unsigned ArgNo) {
1121398a32eSPeter Collingbourne Metadata *MD =
1131398a32eSPeter Collingbourne cast<MetadataAsValue>(CI->getArgOperand(ArgNo))->getMetadata();
1141398a32eSPeter Collingbourne
1151398a32eSPeter Collingbourne if (isa<MDNode>(MD) && cast<MDNode>(MD)->isDistinct()) {
1161398a32eSPeter Collingbourne Metadata *&GlobalMD = LocalToGlobal[MD];
1171398a32eSPeter Collingbourne if (!GlobalMD) {
1183a13ed60SBenjamin Kramer std::string NewName = (Twine(LocalToGlobal.size()) + ModuleId).str();
1191398a32eSPeter Collingbourne GlobalMD = MDString::get(M.getContext(), NewName);
1201398a32eSPeter Collingbourne }
1211398a32eSPeter Collingbourne
1221398a32eSPeter Collingbourne CI->setArgOperand(ArgNo,
1231398a32eSPeter Collingbourne MetadataAsValue::get(M.getContext(), GlobalMD));
1241398a32eSPeter Collingbourne }
1251398a32eSPeter Collingbourne };
1261398a32eSPeter Collingbourne
1271398a32eSPeter Collingbourne if (Function *TypeTestFunc =
1281398a32eSPeter Collingbourne M.getFunction(Intrinsic::getName(Intrinsic::type_test))) {
1291398a32eSPeter Collingbourne for (const Use &U : TypeTestFunc->uses()) {
1301398a32eSPeter Collingbourne auto CI = cast<CallInst>(U.getUser());
1311398a32eSPeter Collingbourne ExternalizeTypeId(CI, 1);
1321398a32eSPeter Collingbourne }
1331398a32eSPeter Collingbourne }
1341398a32eSPeter Collingbourne
135*2eade1dbSArthur Eubanks if (Function *PublicTypeTestFunc =
136*2eade1dbSArthur Eubanks M.getFunction(Intrinsic::getName(Intrinsic::public_type_test))) {
137*2eade1dbSArthur Eubanks for (const Use &U : PublicTypeTestFunc->uses()) {
138*2eade1dbSArthur Eubanks auto CI = cast<CallInst>(U.getUser());
139*2eade1dbSArthur Eubanks ExternalizeTypeId(CI, 1);
140*2eade1dbSArthur Eubanks }
141*2eade1dbSArthur Eubanks }
142*2eade1dbSArthur Eubanks
1431398a32eSPeter Collingbourne if (Function *TypeCheckedLoadFunc =
1441398a32eSPeter Collingbourne M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) {
1451398a32eSPeter Collingbourne for (const Use &U : TypeCheckedLoadFunc->uses()) {
1461398a32eSPeter Collingbourne auto CI = cast<CallInst>(U.getUser());
1471398a32eSPeter Collingbourne ExternalizeTypeId(CI, 2);
1481398a32eSPeter Collingbourne }
1491398a32eSPeter Collingbourne }
1501398a32eSPeter Collingbourne
1511398a32eSPeter Collingbourne for (GlobalObject &GO : M.global_objects()) {
1521398a32eSPeter Collingbourne SmallVector<MDNode *, 1> MDs;
1531398a32eSPeter Collingbourne GO.getMetadata(LLVMContext::MD_type, MDs);
1541398a32eSPeter Collingbourne
1551398a32eSPeter Collingbourne GO.eraseMetadata(LLVMContext::MD_type);
1561398a32eSPeter Collingbourne for (auto MD : MDs) {
1571398a32eSPeter Collingbourne auto I = LocalToGlobal.find(MD->getOperand(1));
1581398a32eSPeter Collingbourne if (I == LocalToGlobal.end()) {
1591398a32eSPeter Collingbourne GO.addMetadata(LLVMContext::MD_type, *MD);
1601398a32eSPeter Collingbourne continue;
1611398a32eSPeter Collingbourne }
1621398a32eSPeter Collingbourne GO.addMetadata(
1631398a32eSPeter Collingbourne LLVMContext::MD_type,
1640deb9a9aSBenjamin Kramer *MDNode::get(M.getContext(), {MD->getOperand(0), I->second}));
1651398a32eSPeter Collingbourne }
1661398a32eSPeter Collingbourne }
1671398a32eSPeter Collingbourne }
1681398a32eSPeter Collingbourne
1691398a32eSPeter Collingbourne // Drop unused globals, and drop type information from function declarations.
1701398a32eSPeter Collingbourne // FIXME: If we made functions typeless then there would be no need to do this.
simplifyExternals(Module & M)1711398a32eSPeter Collingbourne void simplifyExternals(Module &M) {
1721398a32eSPeter Collingbourne FunctionType *EmptyFT =
1731398a32eSPeter Collingbourne FunctionType::get(Type::getVoidTy(M.getContext()), false);
1741398a32eSPeter Collingbourne
175d9e46beaSKazu Hirata for (Function &F : llvm::make_early_inc_range(M)) {
1761398a32eSPeter Collingbourne if (F.isDeclaration() && F.use_empty()) {
1771398a32eSPeter Collingbourne F.eraseFromParent();
1781398a32eSPeter Collingbourne continue;
1791398a32eSPeter Collingbourne }
1801398a32eSPeter Collingbourne
18193fdaca5SPeter Collingbourne if (!F.isDeclaration() || F.getFunctionType() == EmptyFT ||
18293fdaca5SPeter Collingbourne // Changing the type of an intrinsic may invalidate the IR.
18393fdaca5SPeter Collingbourne F.getName().startswith("llvm."))
1841398a32eSPeter Collingbourne continue;
1851398a32eSPeter Collingbourne
1861398a32eSPeter Collingbourne Function *NewF =
187f920da00SDylan McKay Function::Create(EmptyFT, GlobalValue::ExternalLinkage,
188f920da00SDylan McKay F.getAddressSpace(), "", &M);
189e28435caSZequan Wu NewF->copyAttributesFrom(&F);
190e28435caSZequan Wu // Only copy function attribtues.
19180ea2bb5SArthur Eubanks NewF->setAttributes(AttributeList::get(M.getContext(),
19280ea2bb5SArthur Eubanks AttributeList::FunctionIndex,
19380ea2bb5SArthur Eubanks F.getAttributes().getFnAttrs()));
1941398a32eSPeter Collingbourne NewF->takeName(&F);
1951398a32eSPeter Collingbourne F.replaceAllUsesWith(ConstantExpr::getBitCast(NewF, F.getType()));
1961398a32eSPeter Collingbourne F.eraseFromParent();
1971398a32eSPeter Collingbourne }
1981398a32eSPeter Collingbourne
19924c8eaecSKazu Hirata for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) {
2001398a32eSPeter Collingbourne if (GV.isDeclaration() && GV.use_empty()) {
2011398a32eSPeter Collingbourne GV.eraseFromParent();
2021398a32eSPeter Collingbourne continue;
2031398a32eSPeter Collingbourne }
2041398a32eSPeter Collingbourne }
2051398a32eSPeter Collingbourne }
2061398a32eSPeter Collingbourne
207d3704f67SGeorge Rimar static void
filterModule(Module * M,function_ref<bool (const GlobalValue *)> ShouldKeepDefinition)208d3704f67SGeorge Rimar filterModule(Module *M,
209d3704f67SGeorge Rimar function_ref<bool(const GlobalValue *)> ShouldKeepDefinition) {
210d3704f67SGeorge Rimar std::vector<GlobalValue *> V;
211d3704f67SGeorge Rimar for (GlobalValue &GV : M->global_values())
212d3704f67SGeorge Rimar if (!ShouldKeepDefinition(&GV))
213d3704f67SGeorge Rimar V.push_back(&GV);
21454565249SGeorge Rimar
215d3704f67SGeorge Rimar for (GlobalValue *GV : V)
216d3704f67SGeorge Rimar if (!convertToDeclaration(*GV))
217d3704f67SGeorge Rimar GV->eraseFromParent();
2181398a32eSPeter Collingbourne }
2191398a32eSPeter Collingbourne
forEachVirtualFunction(Constant * C,function_ref<void (Function *)> Fn)220002c2d53SPeter Collingbourne void forEachVirtualFunction(Constant *C, function_ref<void(Function *)> Fn) {
221002c2d53SPeter Collingbourne if (auto *F = dyn_cast<Function>(C))
222002c2d53SPeter Collingbourne return Fn(F);
2233baa72afSPeter Collingbourne if (isa<GlobalValue>(C))
2243baa72afSPeter Collingbourne return;
225002c2d53SPeter Collingbourne for (Value *Op : C->operands())
226002c2d53SPeter Collingbourne forEachVirtualFunction(cast<Constant>(Op), Fn);
227002c2d53SPeter Collingbourne }
228002c2d53SPeter Collingbourne
229fde55a9cSTeresa Johnson // Clone any @llvm[.compiler].used over to the new module and append
230fde55a9cSTeresa Johnson // values whose defs were cloned into that module.
cloneUsedGlobalVariables(const Module & SrcM,Module & DestM,bool CompilerUsed)231fde55a9cSTeresa Johnson static void cloneUsedGlobalVariables(const Module &SrcM, Module &DestM,
232fde55a9cSTeresa Johnson bool CompilerUsed) {
2333adb89bbSFangrui Song SmallVector<GlobalValue *, 4> Used, NewUsed;
234fde55a9cSTeresa Johnson // First collect those in the llvm[.compiler].used set.
235fde55a9cSTeresa Johnson collectUsedGlobalVariables(SrcM, Used, CompilerUsed);
236fde55a9cSTeresa Johnson // Next build a set of the equivalent values defined in DestM.
237fde55a9cSTeresa Johnson for (auto *V : Used) {
238fde55a9cSTeresa Johnson auto *GV = DestM.getNamedValue(V->getName());
239fde55a9cSTeresa Johnson if (GV && !GV->isDeclaration())
2403adb89bbSFangrui Song NewUsed.push_back(GV);
241fde55a9cSTeresa Johnson }
242fde55a9cSTeresa Johnson // Finally, add them to a llvm[.compiler].used variable in DestM.
243fde55a9cSTeresa Johnson if (CompilerUsed)
2443adb89bbSFangrui Song appendToCompilerUsed(DestM, NewUsed);
245fde55a9cSTeresa Johnson else
2463adb89bbSFangrui Song appendToUsed(DestM, NewUsed);
247fde55a9cSTeresa Johnson }
248fde55a9cSTeresa Johnson
2491398a32eSPeter Collingbourne // If it's possible to split M into regular and thin LTO parts, do so and write
2501398a32eSPeter Collingbourne // a multi-module bitcode file with the two parts to OS. Otherwise, write only a
2511398a32eSPeter Collingbourne // regular LTO bitcode file to OS.
splitAndWriteThinLTOBitcode(raw_ostream & OS,raw_ostream * ThinLinkOS,function_ref<AAResults & (Function &)> AARGetter,Module & M)252002c2d53SPeter Collingbourne void splitAndWriteThinLTOBitcode(
2530c6a4ff8STeresa Johnson raw_ostream &OS, raw_ostream *ThinLinkOS,
2540c6a4ff8STeresa Johnson function_ref<AAResults &(Function &)> AARGetter, Module &M) {
255964f4663SEvgeniy Stepanov std::string ModuleId = getUniqueModuleId(&M);
2561398a32eSPeter Collingbourne if (ModuleId.empty()) {
2576867ab7cSVlad Tsyrklevich // We couldn't generate a module ID for this module, write it out as a
2586867ab7cSVlad Tsyrklevich // regular LTO module with an index for summary-based dead stripping.
2596867ab7cSVlad Tsyrklevich ProfileSummaryInfo PSI(M);
2606867ab7cSVlad Tsyrklevich M.addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
2616867ab7cSVlad Tsyrklevich ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
2626867ab7cSVlad Tsyrklevich WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, &Index);
2636867ab7cSVlad Tsyrklevich
2640c6a4ff8STeresa Johnson if (ThinLinkOS)
2650c6a4ff8STeresa Johnson // We don't have a ThinLTO part, but still write the module to the
2660c6a4ff8STeresa Johnson // ThinLinkOS if requested so that the expected output file is produced.
2676867ab7cSVlad Tsyrklevich WriteBitcodeToFile(M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false,
2686867ab7cSVlad Tsyrklevich &Index);
2696867ab7cSVlad Tsyrklevich
2701398a32eSPeter Collingbourne return;
2711398a32eSPeter Collingbourne }
2721398a32eSPeter Collingbourne
2731398a32eSPeter Collingbourne promoteTypeIds(M, ModuleId);
2741398a32eSPeter Collingbourne
275dd968219SPeter Collingbourne // Returns whether a global or its associated global has attached type
276dd968219SPeter Collingbourne // metadata. The former may participate in CFI or whole-program
277dd968219SPeter Collingbourne // devirtualization, so they need to appear in the merged module instead of
278dd968219SPeter Collingbourne // the thin LTO module. Similarly, globals that are associated with globals
279dd968219SPeter Collingbourne // with type metadata need to appear in the merged module because they will
280dd968219SPeter Collingbourne // reference the global's section directly.
2810deb9a9aSBenjamin Kramer auto HasTypeMetadata = [](const GlobalObject *GO) {
282dd968219SPeter Collingbourne if (MDNode *MD = GO->getMetadata(LLVMContext::MD_associated))
283dd968219SPeter Collingbourne if (auto *AssocVM = dyn_cast_or_null<ValueAsMetadata>(MD->getOperand(0)))
284dd968219SPeter Collingbourne if (auto *AssocGO = dyn_cast<GlobalObject>(AssocVM->getValue()))
285dd968219SPeter Collingbourne if (AssocGO->hasMetadata(LLVMContext::MD_type))
286dd968219SPeter Collingbourne return true;
2870deb9a9aSBenjamin Kramer return GO->hasMetadata(LLVMContext::MD_type);
2881398a32eSPeter Collingbourne };
2891398a32eSPeter Collingbourne
290002c2d53SPeter Collingbourne // Collect the set of virtual functions that are eligible for virtual constant
291002c2d53SPeter Collingbourne // propagation. Each eligible function must not access memory, must return
292002c2d53SPeter Collingbourne // an integer of width <=64 bits, must take at least one argument, must not
293002c2d53SPeter Collingbourne // use its first argument (assumed to be "this") and all arguments other than
294002c2d53SPeter Collingbourne // the first one must be of <=64 bit integer type.
295002c2d53SPeter Collingbourne //
296002c2d53SPeter Collingbourne // Note that we test whether this copy of the function is readnone, rather
297002c2d53SPeter Collingbourne // than testing function attributes, which must hold for any copy of the
298002c2d53SPeter Collingbourne // function, even a less optimized version substituted at link time. This is
299002c2d53SPeter Collingbourne // sound because the virtual constant propagation optimizations effectively
300002c2d53SPeter Collingbourne // inline all implementations of the virtual function into each call site,
301002c2d53SPeter Collingbourne // rather than using function attributes to perform local optimization.
302aa09a82bSGeorge Burgess IV DenseSet<const Function *> EligibleVirtualFns;
3034075ccc7SBob Haarman // If any member of a comdat lives in MergedM, put all members of that
3044075ccc7SBob Haarman // comdat in MergedM to keep the comdat together.
3054075ccc7SBob Haarman DenseSet<const Comdat *> MergedMComdats;
306002c2d53SPeter Collingbourne for (GlobalVariable &GV : M.globals())
3074075ccc7SBob Haarman if (HasTypeMetadata(&GV)) {
3084075ccc7SBob Haarman if (const auto *C = GV.getComdat())
3094075ccc7SBob Haarman MergedMComdats.insert(C);
310002c2d53SPeter Collingbourne forEachVirtualFunction(GV.getInitializer(), [&](Function *F) {
311002c2d53SPeter Collingbourne auto *RT = dyn_cast<IntegerType>(F->getReturnType());
312002c2d53SPeter Collingbourne if (!RT || RT->getBitWidth() > 64 || F->arg_empty() ||
313002c2d53SPeter Collingbourne !F->arg_begin()->use_empty())
314002c2d53SPeter Collingbourne return;
31523b0ab2aSKazu Hirata for (auto &Arg : drop_begin(F->args())) {
316002c2d53SPeter Collingbourne auto *ArgT = dyn_cast<IntegerType>(Arg.getType());
317002c2d53SPeter Collingbourne if (!ArgT || ArgT->getBitWidth() > 64)
318002c2d53SPeter Collingbourne return;
319002c2d53SPeter Collingbourne }
32001f0c8a8SChandler Carruth if (!F->isDeclaration() &&
321014f5bcfSFlorian Hahn computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) ==
322014f5bcfSFlorian Hahn FMRB_DoesNotAccessMemory)
323002c2d53SPeter Collingbourne EligibleVirtualFns.insert(F);
324002c2d53SPeter Collingbourne });
3254075ccc7SBob Haarman }
326002c2d53SPeter Collingbourne
3271398a32eSPeter Collingbourne ValueToValueMapTy VMap;
328002c2d53SPeter Collingbourne std::unique_ptr<Module> MergedM(
32971867532SRafael Espindola CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool {
3304075ccc7SBob Haarman if (const auto *C = GV->getComdat())
3314075ccc7SBob Haarman if (MergedMComdats.count(C))
3324075ccc7SBob Haarman return true;
333002c2d53SPeter Collingbourne if (auto *F = dyn_cast<Function>(GV))
334002c2d53SPeter Collingbourne return EligibleVirtualFns.count(F);
33540ec1c0fSItay Bookstein if (auto *GVar =
33640ec1c0fSItay Bookstein dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
337002c2d53SPeter Collingbourne return HasTypeMetadata(GVar);
338002c2d53SPeter Collingbourne return false;
339002c2d53SPeter Collingbourne }));
34028ffd326SPeter Collingbourne StripDebugInfo(*MergedM);
34129c6f483SPeter Collingbourne MergedM->setModuleInlineAsm("");
3421398a32eSPeter Collingbourne
343fde55a9cSTeresa Johnson // Clone any llvm.*used globals to ensure the included values are
344fde55a9cSTeresa Johnson // not deleted.
345fde55a9cSTeresa Johnson cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ false);
346fde55a9cSTeresa Johnson cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true);
347fde55a9cSTeresa Johnson
348002c2d53SPeter Collingbourne for (Function &F : *MergedM)
349002c2d53SPeter Collingbourne if (!F.isDeclaration()) {
350002c2d53SPeter Collingbourne // Reset the linkage of all functions eligible for virtual constant
351002c2d53SPeter Collingbourne // propagation. The canonical definitions live in the thin LTO module so
352002c2d53SPeter Collingbourne // that they can be imported.
353002c2d53SPeter Collingbourne F.setLinkage(GlobalValue::AvailableExternallyLinkage);
354002c2d53SPeter Collingbourne F.setComdat(nullptr);
355002c2d53SPeter Collingbourne }
356002c2d53SPeter Collingbourne
3574d4ee93dSEvgeniy Stepanov SetVector<GlobalValue *> CfiFunctions;
3584d4ee93dSEvgeniy Stepanov for (auto &F : M)
3594d4ee93dSEvgeniy Stepanov if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F))
3604d4ee93dSEvgeniy Stepanov CfiFunctions.insert(&F);
3614d4ee93dSEvgeniy Stepanov
3624075ccc7SBob Haarman // Remove all globals with type metadata, globals with comdats that live in
3634075ccc7SBob Haarman // MergedM, and aliases pointing to such globals from the thin LTO module.
364002c2d53SPeter Collingbourne filterModule(&M, [&](const GlobalValue *GV) {
36540ec1c0fSItay Bookstein if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
3664075ccc7SBob Haarman if (HasTypeMetadata(GVar))
3674075ccc7SBob Haarman return false;
3684075ccc7SBob Haarman if (const auto *C = GV->getComdat())
3694075ccc7SBob Haarman if (MergedMComdats.count(C))
3704075ccc7SBob Haarman return false;
371002c2d53SPeter Collingbourne return true;
372002c2d53SPeter Collingbourne });
3731398a32eSPeter Collingbourne
3744d4ee93dSEvgeniy Stepanov promoteInternals(*MergedM, M, ModuleId, CfiFunctions);
3754d4ee93dSEvgeniy Stepanov promoteInternals(M, *MergedM, ModuleId, CfiFunctions);
3764d4ee93dSEvgeniy Stepanov
377230b2567SVlad Tsyrklevich auto &Ctx = MergedM->getContext();
3784d4ee93dSEvgeniy Stepanov SmallVector<MDNode *, 8> CfiFunctionMDs;
3794d4ee93dSEvgeniy Stepanov for (auto V : CfiFunctions) {
3804d4ee93dSEvgeniy Stepanov Function &F = *cast<Function>(V);
3814d4ee93dSEvgeniy Stepanov SmallVector<MDNode *, 2> Types;
3824d4ee93dSEvgeniy Stepanov F.getMetadata(LLVMContext::MD_type, Types);
3834d4ee93dSEvgeniy Stepanov
3844d4ee93dSEvgeniy Stepanov SmallVector<Metadata *, 4> Elts;
3854d4ee93dSEvgeniy Stepanov Elts.push_back(MDString::get(Ctx, F.getName()));
3864d4ee93dSEvgeniy Stepanov CfiFunctionLinkage Linkage;
3870e497d15SPeter Collingbourne if (lowertypetests::isJumpTableCanonical(&F))
3884d4ee93dSEvgeniy Stepanov Linkage = CFL_Definition;
3890e497d15SPeter Collingbourne else if (F.hasExternalWeakLinkage())
3904d4ee93dSEvgeniy Stepanov Linkage = CFL_WeakDeclaration;
3914d4ee93dSEvgeniy Stepanov else
3924d4ee93dSEvgeniy Stepanov Linkage = CFL_Declaration;
3934d4ee93dSEvgeniy Stepanov Elts.push_back(ConstantAsMetadata::get(
3944d4ee93dSEvgeniy Stepanov llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage)));
395e53472deSKazu Hirata append_range(Elts, Types);
3964d4ee93dSEvgeniy Stepanov CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts));
3974d4ee93dSEvgeniy Stepanov }
3984d4ee93dSEvgeniy Stepanov
3994d4ee93dSEvgeniy Stepanov if(!CfiFunctionMDs.empty()) {
4004d4ee93dSEvgeniy Stepanov NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions");
4014d4ee93dSEvgeniy Stepanov for (auto MD : CfiFunctionMDs)
4024d4ee93dSEvgeniy Stepanov NMD->addOperand(MD);
4034d4ee93dSEvgeniy Stepanov }
4041398a32eSPeter Collingbourne
405cdec22efSVlad Tsyrklevich SmallVector<MDNode *, 8> FunctionAliases;
406cdec22efSVlad Tsyrklevich for (auto &A : M.aliases()) {
407cdec22efSVlad Tsyrklevich if (!isa<Function>(A.getAliasee()))
408cdec22efSVlad Tsyrklevich continue;
409cdec22efSVlad Tsyrklevich
410cdec22efSVlad Tsyrklevich auto *F = cast<Function>(A.getAliasee());
411cdec22efSVlad Tsyrklevich
4120deb9a9aSBenjamin Kramer Metadata *Elts[] = {
4130deb9a9aSBenjamin Kramer MDString::get(Ctx, A.getName()),
4140deb9a9aSBenjamin Kramer MDString::get(Ctx, F->getName()),
4150deb9a9aSBenjamin Kramer ConstantAsMetadata::get(
4160deb9a9aSBenjamin Kramer ConstantInt::get(Type::getInt8Ty(Ctx), A.getVisibility())),
4170deb9a9aSBenjamin Kramer ConstantAsMetadata::get(
4180deb9a9aSBenjamin Kramer ConstantInt::get(Type::getInt8Ty(Ctx), A.isWeakForLinker())),
4190deb9a9aSBenjamin Kramer };
420cdec22efSVlad Tsyrklevich
421cdec22efSVlad Tsyrklevich FunctionAliases.push_back(MDTuple::get(Ctx, Elts));
422cdec22efSVlad Tsyrklevich }
423cdec22efSVlad Tsyrklevich
424cdec22efSVlad Tsyrklevich if (!FunctionAliases.empty()) {
425cdec22efSVlad Tsyrklevich NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("aliases");
426cdec22efSVlad Tsyrklevich for (auto MD : FunctionAliases)
427cdec22efSVlad Tsyrklevich NMD->addOperand(MD);
428cdec22efSVlad Tsyrklevich }
429cdec22efSVlad Tsyrklevich
430230b2567SVlad Tsyrklevich SmallVector<MDNode *, 8> Symvers;
431230b2567SVlad Tsyrklevich ModuleSymbolTable::CollectAsmSymvers(M, [&](StringRef Name, StringRef Alias) {
432230b2567SVlad Tsyrklevich Function *F = M.getFunction(Name);
433230b2567SVlad Tsyrklevich if (!F || F->use_empty())
434230b2567SVlad Tsyrklevich return;
435230b2567SVlad Tsyrklevich
4360deb9a9aSBenjamin Kramer Symvers.push_back(MDTuple::get(
4370deb9a9aSBenjamin Kramer Ctx, {MDString::get(Ctx, Name), MDString::get(Ctx, Alias)}));
438230b2567SVlad Tsyrklevich });
439230b2567SVlad Tsyrklevich
440230b2567SVlad Tsyrklevich if (!Symvers.empty()) {
441230b2567SVlad Tsyrklevich NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("symvers");
442230b2567SVlad Tsyrklevich for (auto MD : Symvers)
443230b2567SVlad Tsyrklevich NMD->addOperand(MD);
444230b2567SVlad Tsyrklevich }
445230b2567SVlad Tsyrklevich
4461398a32eSPeter Collingbourne simplifyExternals(*MergedM);
4471398a32eSPeter Collingbourne
4481398a32eSPeter Collingbourne // FIXME: Try to re-use BSI and PFI from the original module here.
44994624acaSTeresa Johnson ProfileSummaryInfo PSI(M);
45094624acaSTeresa Johnson ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
4510c6a4ff8STeresa Johnson
452e357fbd2SPeter Collingbourne // Mark the merged module as requiring full LTO. We still want an index for
453e357fbd2SPeter Collingbourne // it though, so that it can participate in summary-based dead stripping.
454e357fbd2SPeter Collingbourne MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
455e357fbd2SPeter Collingbourne ModuleSummaryIndex MergedMIndex =
456e357fbd2SPeter Collingbourne buildModuleSummaryIndex(*MergedM, nullptr, &PSI);
457e357fbd2SPeter Collingbourne
4580c6a4ff8STeresa Johnson SmallVector<char, 0> Buffer;
4590c6a4ff8STeresa Johnson
4600c6a4ff8STeresa Johnson BitcodeWriter W(Buffer);
4610c6a4ff8STeresa Johnson // Save the module hash produced for the full bitcode, which will
4620c6a4ff8STeresa Johnson // be used in the backends, and use that in the minimized bitcode
4630c6a4ff8STeresa Johnson // produced for the full link.
4640c6a4ff8STeresa Johnson ModuleHash ModHash = {{0}};
4656a86e25dSRafael Espindola W.writeModule(M, /*ShouldPreserveUseListOrder=*/false, &Index,
4660c6a4ff8STeresa Johnson /*GenerateHash=*/true, &ModHash);
4676a86e25dSRafael Espindola W.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, &MergedMIndex);
46892648c25SPeter Collingbourne W.writeSymtab();
469a0f371a1SPeter Collingbourne W.writeStrtab();
4701398a32eSPeter Collingbourne OS << Buffer;
4710c6a4ff8STeresa Johnson
4721dec57d5SHaojie Wang // If a minimized bitcode module was requested for the thin link, only
4731dec57d5SHaojie Wang // the information that is needed by thin link will be written in the
4741dec57d5SHaojie Wang // given OS (the merged module will be written as usual).
4750c6a4ff8STeresa Johnson if (ThinLinkOS) {
4760c6a4ff8STeresa Johnson Buffer.clear();
4770c6a4ff8STeresa Johnson BitcodeWriter W2(Buffer);
4780c6a4ff8STeresa Johnson StripDebugInfo(M);
4796a86e25dSRafael Espindola W2.writeThinLinkBitcode(M, Index, ModHash);
4806a86e25dSRafael Espindola W2.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false,
481e357fbd2SPeter Collingbourne &MergedMIndex);
48292648c25SPeter Collingbourne W2.writeSymtab();
483a0f371a1SPeter Collingbourne W2.writeStrtab();
4840c6a4ff8STeresa Johnson *ThinLinkOS << Buffer;
4850c6a4ff8STeresa Johnson }
4861398a32eSPeter Collingbourne }
4871398a32eSPeter Collingbourne
488a7004363STeresa Johnson // Check if the LTO Unit splitting has been enabled.
enableSplitLTOUnit(Module & M)489a7004363STeresa Johnson bool enableSplitLTOUnit(Module &M) {
490290a8398STeresa Johnson bool EnableSplitLTOUnit = false;
491290a8398STeresa Johnson if (auto *MD = mdconst::extract_or_null<ConstantInt>(
492290a8398STeresa Johnson M.getModuleFlag("EnableSplitLTOUnit")))
493290a8398STeresa Johnson EnableSplitLTOUnit = MD->getZExtValue();
494a7004363STeresa Johnson return EnableSplitLTOUnit;
495a7004363STeresa Johnson }
496290a8398STeresa Johnson
497a7004363STeresa Johnson // Returns whether this module needs to be split because it uses type metadata.
hasTypeMetadata(Module & M)498a7004363STeresa Johnson bool hasTypeMetadata(Module &M) {
4991398a32eSPeter Collingbourne for (auto &GO : M.global_objects()) {
5000deb9a9aSBenjamin Kramer if (GO.hasMetadata(LLVMContext::MD_type))
5011398a32eSPeter Collingbourne return true;
5021398a32eSPeter Collingbourne }
5031398a32eSPeter Collingbourne return false;
5041398a32eSPeter Collingbourne }
5051398a32eSPeter Collingbourne
writeThinLTOBitcode(raw_ostream & OS,raw_ostream * ThinLinkOS,function_ref<AAResults & (Function &)> AARGetter,Module & M,const ModuleSummaryIndex * Index)5060c6a4ff8STeresa Johnson void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
507002c2d53SPeter Collingbourne function_ref<AAResults &(Function &)> AARGetter,
508002c2d53SPeter Collingbourne Module &M, const ModuleSummaryIndex *Index) {
509a7004363STeresa Johnson std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr;
510a7004363STeresa Johnson // See if this module has any type metadata. If so, we try to split it
511a7004363STeresa Johnson // or at least promote type ids to enable WPD.
512a7004363STeresa Johnson if (hasTypeMetadata(M)) {
513a7004363STeresa Johnson if (enableSplitLTOUnit(M))
5140c6a4ff8STeresa Johnson return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
515a7004363STeresa Johnson // Promote type ids as needed for index-based WPD.
516a7004363STeresa Johnson std::string ModuleId = getUniqueModuleId(&M);
517a7004363STeresa Johnson if (!ModuleId.empty()) {
518a7004363STeresa Johnson promoteTypeIds(M, ModuleId);
519a7004363STeresa Johnson // Need to rebuild the index so that it contains type metadata
520a7004363STeresa Johnson // for the newly promoted type ids.
521a7004363STeresa Johnson // FIXME: Probably should not bother building the index at all
522a7004363STeresa Johnson // in the caller of writeThinLTOBitcode (which does so via the
523a7004363STeresa Johnson // ModuleSummaryIndexAnalysis pass), since we have to rebuild it
524a7004363STeresa Johnson // anyway whenever there is type metadata (here or in
525a7004363STeresa Johnson // splitAndWriteThinLTOBitcode). Just always build it once via the
526a7004363STeresa Johnson // buildModuleSummaryIndex when Module(s) are ready.
527a7004363STeresa Johnson ProfileSummaryInfo PSI(M);
5280eaee545SJonas Devlieghere NewIndex = std::make_unique<ModuleSummaryIndex>(
529a7004363STeresa Johnson buildModuleSummaryIndex(M, nullptr, &PSI));
530a7004363STeresa Johnson Index = NewIndex.get();
531a7004363STeresa Johnson }
532a7004363STeresa Johnson }
5331398a32eSPeter Collingbourne
534a7004363STeresa Johnson // Write it out as an unsplit ThinLTO module.
5350c6a4ff8STeresa Johnson
5360c6a4ff8STeresa Johnson // Save the module hash produced for the full bitcode, which will
5370c6a4ff8STeresa Johnson // be used in the backends, and use that in the minimized bitcode
5380c6a4ff8STeresa Johnson // produced for the full link.
5390c6a4ff8STeresa Johnson ModuleHash ModHash = {{0}};
5406a86e25dSRafael Espindola WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, Index,
5410c6a4ff8STeresa Johnson /*GenerateHash=*/true, &ModHash);
5421dec57d5SHaojie Wang // If a minimized bitcode module was requested for the thin link, only
5431dec57d5SHaojie Wang // the information that is needed by thin link will be written in the
5441dec57d5SHaojie Wang // given OS.
5451dec57d5SHaojie Wang if (ThinLinkOS && Index)
5467aaf024dSFangrui Song writeThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash);
5471398a32eSPeter Collingbourne }
5481398a32eSPeter Collingbourne
5491398a32eSPeter Collingbourne class WriteThinLTOBitcode : public ModulePass {
5501398a32eSPeter Collingbourne raw_ostream &OS; // raw_ostream to print on
5510c6a4ff8STeresa Johnson // The output stream on which to emit a minimized module for use
5520c6a4ff8STeresa Johnson // just in the thin link, if requested.
553726b2dd0SKazu Hirata raw_ostream *ThinLinkOS = nullptr;
5541398a32eSPeter Collingbourne
5551398a32eSPeter Collingbourne public:
5561398a32eSPeter Collingbourne static char ID; // Pass identification, replacement for typeid
WriteThinLTOBitcode()557726b2dd0SKazu Hirata WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()) {
5581398a32eSPeter Collingbourne initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
5591398a32eSPeter Collingbourne }
5601398a32eSPeter Collingbourne
WriteThinLTOBitcode(raw_ostream & o,raw_ostream * ThinLinkOS)5610c6a4ff8STeresa Johnson explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS)
5620c6a4ff8STeresa Johnson : ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) {
5631398a32eSPeter Collingbourne initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
5641398a32eSPeter Collingbourne }
5651398a32eSPeter Collingbourne
getPassName() const5661398a32eSPeter Collingbourne StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; }
5671398a32eSPeter Collingbourne
runOnModule(Module & M)5681398a32eSPeter Collingbourne bool runOnModule(Module &M) override {
5691398a32eSPeter Collingbourne const ModuleSummaryIndex *Index =
5701398a32eSPeter Collingbourne &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex());
5710c6a4ff8STeresa Johnson writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index);
5721398a32eSPeter Collingbourne return true;
5731398a32eSPeter Collingbourne }
getAnalysisUsage(AnalysisUsage & AU) const5741398a32eSPeter Collingbourne void getAnalysisUsage(AnalysisUsage &AU) const override {
5751398a32eSPeter Collingbourne AU.setPreservesAll();
576002c2d53SPeter Collingbourne AU.addRequired<AssumptionCacheTracker>();
5771398a32eSPeter Collingbourne AU.addRequired<ModuleSummaryIndexWrapperPass>();
578002c2d53SPeter Collingbourne AU.addRequired<TargetLibraryInfoWrapperPass>();
5791398a32eSPeter Collingbourne }
5801398a32eSPeter Collingbourne };
5811398a32eSPeter Collingbourne } // anonymous namespace
5821398a32eSPeter Collingbourne
5831398a32eSPeter Collingbourne char WriteThinLTOBitcode::ID = 0;
5841398a32eSPeter Collingbourne INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode",
5851398a32eSPeter Collingbourne "Write ThinLTO Bitcode", false, true)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)586002c2d53SPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
5871398a32eSPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass)
588002c2d53SPeter Collingbourne INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
5891398a32eSPeter Collingbourne INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode",
5901398a32eSPeter Collingbourne "Write ThinLTO Bitcode", false, true)
5911398a32eSPeter Collingbourne
5920c6a4ff8STeresa Johnson ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str,
5930c6a4ff8STeresa Johnson raw_ostream *ThinLinkOS) {
5940c6a4ff8STeresa Johnson return new WriteThinLTOBitcode(Str, ThinLinkOS);
5951398a32eSPeter Collingbourne }
5966b411418STim Shen
5976b411418STim Shen PreservedAnalyses
run(Module & M,ModuleAnalysisManager & AM)5986b411418STim Shen llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
5996b411418STim Shen FunctionAnalysisManager &FAM =
6006b411418STim Shen AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
6016b411418STim Shen writeThinLTOBitcode(OS, ThinLinkOS,
6026b411418STim Shen [&FAM](Function &F) -> AAResults & {
6036b411418STim Shen return FAM.getResult<AAManager>(F);
6046b411418STim Shen },
6056b411418STim Shen M, &AM.getResult<ModuleSummaryIndexAnalysis>(M));
6066b411418STim Shen return PreservedAnalyses::all();
6076b411418STim Shen }
608