1*0b57cec5SDimitry Andric //===- ThinLTOBitcodeWriter.cpp - Bitcode writing pass for ThinLTO --------===//
2*0b57cec5SDimitry Andric //
3*0b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4*0b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
5*0b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6*0b57cec5SDimitry Andric //
7*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
8*0b57cec5SDimitry Andric 
9*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
10*0b57cec5SDimitry Andric #include "llvm/Analysis/BasicAliasAnalysis.h"
11*0b57cec5SDimitry Andric #include "llvm/Analysis/ModuleSummaryAnalysis.h"
12*0b57cec5SDimitry Andric #include "llvm/Analysis/ProfileSummaryInfo.h"
13*0b57cec5SDimitry Andric #include "llvm/Analysis/TypeMetadataUtils.h"
14*0b57cec5SDimitry Andric #include "llvm/Bitcode/BitcodeWriter.h"
15*0b57cec5SDimitry Andric #include "llvm/IR/Constants.h"
16*0b57cec5SDimitry Andric #include "llvm/IR/DebugInfo.h"
17e8d8bef9SDimitry Andric #include "llvm/IR/Instructions.h"
18*0b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h"
19*0b57cec5SDimitry Andric #include "llvm/IR/Module.h"
20*0b57cec5SDimitry Andric #include "llvm/IR/PassManager.h"
21480093f4SDimitry Andric #include "llvm/InitializePasses.h"
22*0b57cec5SDimitry Andric #include "llvm/Object/ModuleSymbolTable.h"
23*0b57cec5SDimitry Andric #include "llvm/Pass.h"
24*0b57cec5SDimitry Andric #include "llvm/Support/ScopedPrinter.h"
25*0b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h"
26*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO.h"
27*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/FunctionAttrs.h"
28*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/FunctionImport.h"
298bcb0991SDimitry Andric #include "llvm/Transforms/IPO/LowerTypeTests.h"
30*0b57cec5SDimitry Andric #include "llvm/Transforms/Utils/Cloning.h"
31*0b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h"
32*0b57cec5SDimitry Andric using namespace llvm;
33*0b57cec5SDimitry Andric 
34*0b57cec5SDimitry Andric namespace {
35*0b57cec5SDimitry Andric 
3669ade1e0SDimitry Andric // Determine if a promotion alias should be created for a symbol name.
3769ade1e0SDimitry Andric static bool allowPromotionAlias(const std::string &Name) {
3869ade1e0SDimitry Andric   // Promotion aliases are used only in inline assembly. It's safe to
3969ade1e0SDimitry Andric   // simply skip unusual names. Subset of MCAsmInfo::isAcceptableChar()
4069ade1e0SDimitry Andric   // and MCAsmInfoXCOFF::isAcceptableChar().
4169ade1e0SDimitry Andric   for (const char &C : Name) {
4269ade1e0SDimitry Andric     if (isAlnum(C) || C == '_' || C == '.')
4369ade1e0SDimitry Andric       continue;
4469ade1e0SDimitry Andric     return false;
4569ade1e0SDimitry Andric   }
4669ade1e0SDimitry Andric   return true;
4769ade1e0SDimitry Andric }
4869ade1e0SDimitry Andric 
49*0b57cec5SDimitry Andric // Promote each local-linkage entity defined by ExportM and used by ImportM by
50*0b57cec5SDimitry Andric // changing visibility and appending the given ModuleId.
51*0b57cec5SDimitry Andric void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId,
52*0b57cec5SDimitry Andric                       SetVector<GlobalValue *> &PromoteExtra) {
53*0b57cec5SDimitry Andric   DenseMap<const Comdat *, Comdat *> RenamedComdats;
54*0b57cec5SDimitry Andric   for (auto &ExportGV : ExportM.global_values()) {
55*0b57cec5SDimitry Andric     if (!ExportGV.hasLocalLinkage())
56*0b57cec5SDimitry Andric       continue;
57*0b57cec5SDimitry Andric 
58*0b57cec5SDimitry Andric     auto Name = ExportGV.getName();
59*0b57cec5SDimitry Andric     GlobalValue *ImportGV = nullptr;
60*0b57cec5SDimitry Andric     if (!PromoteExtra.count(&ExportGV)) {
61*0b57cec5SDimitry Andric       ImportGV = ImportM.getNamedValue(Name);
62*0b57cec5SDimitry Andric       if (!ImportGV)
63*0b57cec5SDimitry Andric         continue;
64*0b57cec5SDimitry Andric       ImportGV->removeDeadConstantUsers();
65*0b57cec5SDimitry Andric       if (ImportGV->use_empty()) {
66*0b57cec5SDimitry Andric         ImportGV->eraseFromParent();
67*0b57cec5SDimitry Andric         continue;
68*0b57cec5SDimitry Andric       }
69*0b57cec5SDimitry Andric     }
70*0b57cec5SDimitry Andric 
7169ade1e0SDimitry Andric     std::string OldName = Name.str();
72*0b57cec5SDimitry Andric     std::string NewName = (Name + ModuleId).str();
73*0b57cec5SDimitry Andric 
74*0b57cec5SDimitry Andric     if (const auto *C = ExportGV.getComdat())
75*0b57cec5SDimitry Andric       if (C->getName() == Name)
76*0b57cec5SDimitry Andric         RenamedComdats.try_emplace(C, ExportM.getOrInsertComdat(NewName));
77*0b57cec5SDimitry Andric 
78*0b57cec5SDimitry Andric     ExportGV.setName(NewName);
79*0b57cec5SDimitry Andric     ExportGV.setLinkage(GlobalValue::ExternalLinkage);
80*0b57cec5SDimitry Andric     ExportGV.setVisibility(GlobalValue::HiddenVisibility);
81*0b57cec5SDimitry Andric 
82*0b57cec5SDimitry Andric     if (ImportGV) {
83*0b57cec5SDimitry Andric       ImportGV->setName(NewName);
84*0b57cec5SDimitry Andric       ImportGV->setVisibility(GlobalValue::HiddenVisibility);
85*0b57cec5SDimitry Andric     }
8669ade1e0SDimitry Andric 
8769ade1e0SDimitry Andric     if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) {
8869ade1e0SDimitry Andric       // Create a local alias with the original name to avoid breaking
8969ade1e0SDimitry Andric       // references from inline assembly.
900eae32dcSDimitry Andric       std::string Alias =
910eae32dcSDimitry Andric           ".lto_set_conditional " + OldName + "," + NewName + "\n";
9269ade1e0SDimitry Andric       ExportM.appendModuleInlineAsm(Alias);
9369ade1e0SDimitry Andric     }
94*0b57cec5SDimitry Andric   }
95*0b57cec5SDimitry Andric 
96*0b57cec5SDimitry Andric   if (!RenamedComdats.empty())
97*0b57cec5SDimitry Andric     for (auto &GO : ExportM.global_objects())
98*0b57cec5SDimitry Andric       if (auto *C = GO.getComdat()) {
99*0b57cec5SDimitry Andric         auto Replacement = RenamedComdats.find(C);
100*0b57cec5SDimitry Andric         if (Replacement != RenamedComdats.end())
101*0b57cec5SDimitry Andric           GO.setComdat(Replacement->second);
102*0b57cec5SDimitry Andric       }
103*0b57cec5SDimitry Andric }
104*0b57cec5SDimitry Andric 
105*0b57cec5SDimitry Andric // Promote all internal (i.e. distinct) type ids used by the module by replacing
106*0b57cec5SDimitry Andric // them with external type ids formed using the module id.
107*0b57cec5SDimitry Andric //
108*0b57cec5SDimitry Andric // Note that this needs to be done before we clone the module because each clone
109*0b57cec5SDimitry Andric // will receive its own set of distinct metadata nodes.
110*0b57cec5SDimitry Andric void promoteTypeIds(Module &M, StringRef ModuleId) {
111*0b57cec5SDimitry Andric   DenseMap<Metadata *, Metadata *> LocalToGlobal;
112*0b57cec5SDimitry Andric   auto ExternalizeTypeId = [&](CallInst *CI, unsigned ArgNo) {
113*0b57cec5SDimitry Andric     Metadata *MD =
114*0b57cec5SDimitry Andric         cast<MetadataAsValue>(CI->getArgOperand(ArgNo))->getMetadata();
115*0b57cec5SDimitry Andric 
116*0b57cec5SDimitry Andric     if (isa<MDNode>(MD) && cast<MDNode>(MD)->isDistinct()) {
117*0b57cec5SDimitry Andric       Metadata *&GlobalMD = LocalToGlobal[MD];
118*0b57cec5SDimitry Andric       if (!GlobalMD) {
119*0b57cec5SDimitry Andric         std::string NewName = (Twine(LocalToGlobal.size()) + ModuleId).str();
120*0b57cec5SDimitry Andric         GlobalMD = MDString::get(M.getContext(), NewName);
121*0b57cec5SDimitry Andric       }
122*0b57cec5SDimitry Andric 
123*0b57cec5SDimitry Andric       CI->setArgOperand(ArgNo,
124*0b57cec5SDimitry Andric                         MetadataAsValue::get(M.getContext(), GlobalMD));
125*0b57cec5SDimitry Andric     }
126*0b57cec5SDimitry Andric   };
127*0b57cec5SDimitry Andric 
128*0b57cec5SDimitry Andric   if (Function *TypeTestFunc =
129*0b57cec5SDimitry Andric           M.getFunction(Intrinsic::getName(Intrinsic::type_test))) {
130*0b57cec5SDimitry Andric     for (const Use &U : TypeTestFunc->uses()) {
131*0b57cec5SDimitry Andric       auto CI = cast<CallInst>(U.getUser());
132*0b57cec5SDimitry Andric       ExternalizeTypeId(CI, 1);
133*0b57cec5SDimitry Andric     }
134*0b57cec5SDimitry Andric   }
135*0b57cec5SDimitry Andric 
136*0b57cec5SDimitry Andric   if (Function *TypeCheckedLoadFunc =
137*0b57cec5SDimitry Andric           M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) {
138*0b57cec5SDimitry Andric     for (const Use &U : TypeCheckedLoadFunc->uses()) {
139*0b57cec5SDimitry Andric       auto CI = cast<CallInst>(U.getUser());
140*0b57cec5SDimitry Andric       ExternalizeTypeId(CI, 2);
141*0b57cec5SDimitry Andric     }
142*0b57cec5SDimitry Andric   }
143*0b57cec5SDimitry Andric 
144*0b57cec5SDimitry Andric   for (GlobalObject &GO : M.global_objects()) {
145*0b57cec5SDimitry Andric     SmallVector<MDNode *, 1> MDs;
146*0b57cec5SDimitry Andric     GO.getMetadata(LLVMContext::MD_type, MDs);
147*0b57cec5SDimitry Andric 
148*0b57cec5SDimitry Andric     GO.eraseMetadata(LLVMContext::MD_type);
149*0b57cec5SDimitry Andric     for (auto MD : MDs) {
150*0b57cec5SDimitry Andric       auto I = LocalToGlobal.find(MD->getOperand(1));
151*0b57cec5SDimitry Andric       if (I == LocalToGlobal.end()) {
152*0b57cec5SDimitry Andric         GO.addMetadata(LLVMContext::MD_type, *MD);
153*0b57cec5SDimitry Andric         continue;
154*0b57cec5SDimitry Andric       }
155*0b57cec5SDimitry Andric       GO.addMetadata(
156*0b57cec5SDimitry Andric           LLVMContext::MD_type,
157*0b57cec5SDimitry Andric           *MDNode::get(M.getContext(), {MD->getOperand(0), I->second}));
158*0b57cec5SDimitry Andric     }
159*0b57cec5SDimitry Andric   }
160*0b57cec5SDimitry Andric }
161*0b57cec5SDimitry Andric 
162*0b57cec5SDimitry Andric // Drop unused globals, and drop type information from function declarations.
163*0b57cec5SDimitry Andric // FIXME: If we made functions typeless then there would be no need to do this.
164*0b57cec5SDimitry Andric void simplifyExternals(Module &M) {
165*0b57cec5SDimitry Andric   FunctionType *EmptyFT =
166*0b57cec5SDimitry Andric       FunctionType::get(Type::getVoidTy(M.getContext()), false);
167*0b57cec5SDimitry Andric 
168349cc55cSDimitry Andric   for (Function &F : llvm::make_early_inc_range(M)) {
169*0b57cec5SDimitry Andric     if (F.isDeclaration() && F.use_empty()) {
170*0b57cec5SDimitry Andric       F.eraseFromParent();
171*0b57cec5SDimitry Andric       continue;
172*0b57cec5SDimitry Andric     }
173*0b57cec5SDimitry Andric 
174*0b57cec5SDimitry Andric     if (!F.isDeclaration() || F.getFunctionType() == EmptyFT ||
175*0b57cec5SDimitry Andric         // Changing the type of an intrinsic may invalidate the IR.
176*0b57cec5SDimitry Andric         F.getName().startswith("llvm."))
177*0b57cec5SDimitry Andric       continue;
178*0b57cec5SDimitry Andric 
179*0b57cec5SDimitry Andric     Function *NewF =
180*0b57cec5SDimitry Andric         Function::Create(EmptyFT, GlobalValue::ExternalLinkage,
181*0b57cec5SDimitry Andric                          F.getAddressSpace(), "", &M);
182fe6060f1SDimitry Andric     NewF->copyAttributesFrom(&F);
183fe6060f1SDimitry Andric     // Only copy function attribtues.
184349cc55cSDimitry Andric     NewF->setAttributes(AttributeList::get(M.getContext(),
185349cc55cSDimitry Andric                                            AttributeList::FunctionIndex,
186349cc55cSDimitry Andric                                            F.getAttributes().getFnAttrs()));
187*0b57cec5SDimitry Andric     NewF->takeName(&F);
188*0b57cec5SDimitry Andric     F.replaceAllUsesWith(ConstantExpr::getBitCast(NewF, F.getType()));
189*0b57cec5SDimitry Andric     F.eraseFromParent();
190*0b57cec5SDimitry Andric   }
191*0b57cec5SDimitry Andric 
192349cc55cSDimitry Andric   for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) {
193*0b57cec5SDimitry Andric     if (GV.isDeclaration() && GV.use_empty()) {
194*0b57cec5SDimitry Andric       GV.eraseFromParent();
195*0b57cec5SDimitry Andric       continue;
196*0b57cec5SDimitry Andric     }
197*0b57cec5SDimitry Andric   }
198*0b57cec5SDimitry Andric }
199*0b57cec5SDimitry Andric 
200*0b57cec5SDimitry Andric static void
201*0b57cec5SDimitry Andric filterModule(Module *M,
202*0b57cec5SDimitry Andric              function_ref<bool(const GlobalValue *)> ShouldKeepDefinition) {
203*0b57cec5SDimitry Andric   std::vector<GlobalValue *> V;
204*0b57cec5SDimitry Andric   for (GlobalValue &GV : M->global_values())
205*0b57cec5SDimitry Andric     if (!ShouldKeepDefinition(&GV))
206*0b57cec5SDimitry Andric       V.push_back(&GV);
207*0b57cec5SDimitry Andric 
208*0b57cec5SDimitry Andric   for (GlobalValue *GV : V)
209*0b57cec5SDimitry Andric     if (!convertToDeclaration(*GV))
210*0b57cec5SDimitry Andric       GV->eraseFromParent();
211*0b57cec5SDimitry Andric }
212*0b57cec5SDimitry Andric 
213*0b57cec5SDimitry Andric void forEachVirtualFunction(Constant *C, function_ref<void(Function *)> Fn) {
214*0b57cec5SDimitry Andric   if (auto *F = dyn_cast<Function>(C))
215*0b57cec5SDimitry Andric     return Fn(F);
216*0b57cec5SDimitry Andric   if (isa<GlobalValue>(C))
217*0b57cec5SDimitry Andric     return;
218*0b57cec5SDimitry Andric   for (Value *Op : C->operands())
219*0b57cec5SDimitry Andric     forEachVirtualFunction(cast<Constant>(Op), Fn);
220*0b57cec5SDimitry Andric }
221*0b57cec5SDimitry Andric 
222fe6060f1SDimitry Andric // Clone any @llvm[.compiler].used over to the new module and append
223fe6060f1SDimitry Andric // values whose defs were cloned into that module.
224fe6060f1SDimitry Andric static void cloneUsedGlobalVariables(const Module &SrcM, Module &DestM,
225fe6060f1SDimitry Andric                                      bool CompilerUsed) {
226fe6060f1SDimitry Andric   SmallVector<GlobalValue *, 4> Used, NewUsed;
227fe6060f1SDimitry Andric   // First collect those in the llvm[.compiler].used set.
228fe6060f1SDimitry Andric   collectUsedGlobalVariables(SrcM, Used, CompilerUsed);
229fe6060f1SDimitry Andric   // Next build a set of the equivalent values defined in DestM.
230fe6060f1SDimitry Andric   for (auto *V : Used) {
231fe6060f1SDimitry Andric     auto *GV = DestM.getNamedValue(V->getName());
232fe6060f1SDimitry Andric     if (GV && !GV->isDeclaration())
233fe6060f1SDimitry Andric       NewUsed.push_back(GV);
234fe6060f1SDimitry Andric   }
235fe6060f1SDimitry Andric   // Finally, add them to a llvm[.compiler].used variable in DestM.
236fe6060f1SDimitry Andric   if (CompilerUsed)
237fe6060f1SDimitry Andric     appendToCompilerUsed(DestM, NewUsed);
238fe6060f1SDimitry Andric   else
239fe6060f1SDimitry Andric     appendToUsed(DestM, NewUsed);
240fe6060f1SDimitry Andric }
241fe6060f1SDimitry Andric 
242*0b57cec5SDimitry Andric // If it's possible to split M into regular and thin LTO parts, do so and write
243*0b57cec5SDimitry Andric // a multi-module bitcode file with the two parts to OS. Otherwise, write only a
244*0b57cec5SDimitry Andric // regular LTO bitcode file to OS.
245*0b57cec5SDimitry Andric void splitAndWriteThinLTOBitcode(
246*0b57cec5SDimitry Andric     raw_ostream &OS, raw_ostream *ThinLinkOS,
247*0b57cec5SDimitry Andric     function_ref<AAResults &(Function &)> AARGetter, Module &M) {
248*0b57cec5SDimitry Andric   std::string ModuleId = getUniqueModuleId(&M);
249*0b57cec5SDimitry Andric   if (ModuleId.empty()) {
250*0b57cec5SDimitry Andric     // We couldn't generate a module ID for this module, write it out as a
251*0b57cec5SDimitry Andric     // regular LTO module with an index for summary-based dead stripping.
252*0b57cec5SDimitry Andric     ProfileSummaryInfo PSI(M);
253*0b57cec5SDimitry Andric     M.addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
254*0b57cec5SDimitry Andric     ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
255*0b57cec5SDimitry Andric     WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, &Index);
256*0b57cec5SDimitry Andric 
257*0b57cec5SDimitry Andric     if (ThinLinkOS)
258*0b57cec5SDimitry Andric       // We don't have a ThinLTO part, but still write the module to the
259*0b57cec5SDimitry Andric       // ThinLinkOS if requested so that the expected output file is produced.
260*0b57cec5SDimitry Andric       WriteBitcodeToFile(M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false,
261*0b57cec5SDimitry Andric                          &Index);
262*0b57cec5SDimitry Andric 
263*0b57cec5SDimitry Andric     return;
264*0b57cec5SDimitry Andric   }
265*0b57cec5SDimitry Andric 
266*0b57cec5SDimitry Andric   promoteTypeIds(M, ModuleId);
267*0b57cec5SDimitry Andric 
2688bcb0991SDimitry Andric   // Returns whether a global or its associated global has attached type
2698bcb0991SDimitry Andric   // metadata. The former may participate in CFI or whole-program
2708bcb0991SDimitry Andric   // devirtualization, so they need to appear in the merged module instead of
2718bcb0991SDimitry Andric   // the thin LTO module. Similarly, globals that are associated with globals
2728bcb0991SDimitry Andric   // with type metadata need to appear in the merged module because they will
2738bcb0991SDimitry Andric   // reference the global's section directly.
274*0b57cec5SDimitry Andric   auto HasTypeMetadata = [](const GlobalObject *GO) {
2758bcb0991SDimitry Andric     if (MDNode *MD = GO->getMetadata(LLVMContext::MD_associated))
2768bcb0991SDimitry Andric       if (auto *AssocVM = dyn_cast_or_null<ValueAsMetadata>(MD->getOperand(0)))
2778bcb0991SDimitry Andric         if (auto *AssocGO = dyn_cast<GlobalObject>(AssocVM->getValue()))
2788bcb0991SDimitry Andric           if (AssocGO->hasMetadata(LLVMContext::MD_type))
2798bcb0991SDimitry Andric             return true;
280*0b57cec5SDimitry Andric     return GO->hasMetadata(LLVMContext::MD_type);
281*0b57cec5SDimitry Andric   };
282*0b57cec5SDimitry Andric 
283*0b57cec5SDimitry Andric   // Collect the set of virtual functions that are eligible for virtual constant
284*0b57cec5SDimitry Andric   // propagation. Each eligible function must not access memory, must return
285*0b57cec5SDimitry Andric   // an integer of width <=64 bits, must take at least one argument, must not
286*0b57cec5SDimitry Andric   // use its first argument (assumed to be "this") and all arguments other than
287*0b57cec5SDimitry Andric   // the first one must be of <=64 bit integer type.
288*0b57cec5SDimitry Andric   //
289*0b57cec5SDimitry Andric   // Note that we test whether this copy of the function is readnone, rather
290*0b57cec5SDimitry Andric   // than testing function attributes, which must hold for any copy of the
291*0b57cec5SDimitry Andric   // function, even a less optimized version substituted at link time. This is
292*0b57cec5SDimitry Andric   // sound because the virtual constant propagation optimizations effectively
293*0b57cec5SDimitry Andric   // inline all implementations of the virtual function into each call site,
294*0b57cec5SDimitry Andric   // rather than using function attributes to perform local optimization.
295*0b57cec5SDimitry Andric   DenseSet<const Function *> EligibleVirtualFns;
296*0b57cec5SDimitry Andric   // If any member of a comdat lives in MergedM, put all members of that
297*0b57cec5SDimitry Andric   // comdat in MergedM to keep the comdat together.
298*0b57cec5SDimitry Andric   DenseSet<const Comdat *> MergedMComdats;
299*0b57cec5SDimitry Andric   for (GlobalVariable &GV : M.globals())
300*0b57cec5SDimitry Andric     if (HasTypeMetadata(&GV)) {
301*0b57cec5SDimitry Andric       if (const auto *C = GV.getComdat())
302*0b57cec5SDimitry Andric         MergedMComdats.insert(C);
303*0b57cec5SDimitry Andric       forEachVirtualFunction(GV.getInitializer(), [&](Function *F) {
304*0b57cec5SDimitry Andric         auto *RT = dyn_cast<IntegerType>(F->getReturnType());
305*0b57cec5SDimitry Andric         if (!RT || RT->getBitWidth() > 64 || F->arg_empty() ||
306*0b57cec5SDimitry Andric             !F->arg_begin()->use_empty())
307*0b57cec5SDimitry Andric           return;
308e8d8bef9SDimitry Andric         for (auto &Arg : drop_begin(F->args())) {
309*0b57cec5SDimitry Andric           auto *ArgT = dyn_cast<IntegerType>(Arg.getType());
310*0b57cec5SDimitry Andric           if (!ArgT || ArgT->getBitWidth() > 64)
311*0b57cec5SDimitry Andric             return;
312*0b57cec5SDimitry Andric         }
313*0b57cec5SDimitry Andric         if (!F->isDeclaration() &&
314*0b57cec5SDimitry Andric             computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == MAK_ReadNone)
315*0b57cec5SDimitry Andric           EligibleVirtualFns.insert(F);
316*0b57cec5SDimitry Andric       });
317*0b57cec5SDimitry Andric     }
318*0b57cec5SDimitry Andric 
319*0b57cec5SDimitry Andric   ValueToValueMapTy VMap;
320*0b57cec5SDimitry Andric   std::unique_ptr<Module> MergedM(
321*0b57cec5SDimitry Andric       CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool {
322*0b57cec5SDimitry Andric         if (const auto *C = GV->getComdat())
323*0b57cec5SDimitry Andric           if (MergedMComdats.count(C))
324*0b57cec5SDimitry Andric             return true;
325*0b57cec5SDimitry Andric         if (auto *F = dyn_cast<Function>(GV))
326*0b57cec5SDimitry Andric           return EligibleVirtualFns.count(F);
327349cc55cSDimitry Andric         if (auto *GVar =
328349cc55cSDimitry Andric                 dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
329*0b57cec5SDimitry Andric           return HasTypeMetadata(GVar);
330*0b57cec5SDimitry Andric         return false;
331*0b57cec5SDimitry Andric       }));
332*0b57cec5SDimitry Andric   StripDebugInfo(*MergedM);
333*0b57cec5SDimitry Andric   MergedM->setModuleInlineAsm("");
334*0b57cec5SDimitry Andric 
335fe6060f1SDimitry Andric   // Clone any llvm.*used globals to ensure the included values are
336fe6060f1SDimitry Andric   // not deleted.
337fe6060f1SDimitry Andric   cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ false);
338fe6060f1SDimitry Andric   cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true);
339fe6060f1SDimitry Andric 
340*0b57cec5SDimitry Andric   for (Function &F : *MergedM)
341*0b57cec5SDimitry Andric     if (!F.isDeclaration()) {
342*0b57cec5SDimitry Andric       // Reset the linkage of all functions eligible for virtual constant
343*0b57cec5SDimitry Andric       // propagation. The canonical definitions live in the thin LTO module so
344*0b57cec5SDimitry Andric       // that they can be imported.
345*0b57cec5SDimitry Andric       F.setLinkage(GlobalValue::AvailableExternallyLinkage);
346*0b57cec5SDimitry Andric       F.setComdat(nullptr);
347*0b57cec5SDimitry Andric     }
348*0b57cec5SDimitry Andric 
349*0b57cec5SDimitry Andric   SetVector<GlobalValue *> CfiFunctions;
350*0b57cec5SDimitry Andric   for (auto &F : M)
351*0b57cec5SDimitry Andric     if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F))
352*0b57cec5SDimitry Andric       CfiFunctions.insert(&F);
353*0b57cec5SDimitry Andric 
354*0b57cec5SDimitry Andric   // Remove all globals with type metadata, globals with comdats that live in
355*0b57cec5SDimitry Andric   // MergedM, and aliases pointing to such globals from the thin LTO module.
356*0b57cec5SDimitry Andric   filterModule(&M, [&](const GlobalValue *GV) {
357349cc55cSDimitry Andric     if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject()))
358*0b57cec5SDimitry Andric       if (HasTypeMetadata(GVar))
359*0b57cec5SDimitry Andric         return false;
360*0b57cec5SDimitry Andric     if (const auto *C = GV->getComdat())
361*0b57cec5SDimitry Andric       if (MergedMComdats.count(C))
362*0b57cec5SDimitry Andric         return false;
363*0b57cec5SDimitry Andric     return true;
364*0b57cec5SDimitry Andric   });
365*0b57cec5SDimitry Andric 
366*0b57cec5SDimitry Andric   promoteInternals(*MergedM, M, ModuleId, CfiFunctions);
367*0b57cec5SDimitry Andric   promoteInternals(M, *MergedM, ModuleId, CfiFunctions);
368*0b57cec5SDimitry Andric 
369*0b57cec5SDimitry Andric   auto &Ctx = MergedM->getContext();
370*0b57cec5SDimitry Andric   SmallVector<MDNode *, 8> CfiFunctionMDs;
371*0b57cec5SDimitry Andric   for (auto V : CfiFunctions) {
372*0b57cec5SDimitry Andric     Function &F = *cast<Function>(V);
373*0b57cec5SDimitry Andric     SmallVector<MDNode *, 2> Types;
374*0b57cec5SDimitry Andric     F.getMetadata(LLVMContext::MD_type, Types);
375*0b57cec5SDimitry Andric 
376*0b57cec5SDimitry Andric     SmallVector<Metadata *, 4> Elts;
377*0b57cec5SDimitry Andric     Elts.push_back(MDString::get(Ctx, F.getName()));
378*0b57cec5SDimitry Andric     CfiFunctionLinkage Linkage;
3798bcb0991SDimitry Andric     if (lowertypetests::isJumpTableCanonical(&F))
380*0b57cec5SDimitry Andric       Linkage = CFL_Definition;
3818bcb0991SDimitry Andric     else if (F.hasExternalWeakLinkage())
382*0b57cec5SDimitry Andric       Linkage = CFL_WeakDeclaration;
383*0b57cec5SDimitry Andric     else
384*0b57cec5SDimitry Andric       Linkage = CFL_Declaration;
385*0b57cec5SDimitry Andric     Elts.push_back(ConstantAsMetadata::get(
386*0b57cec5SDimitry Andric         llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage)));
387e8d8bef9SDimitry Andric     append_range(Elts, Types);
388*0b57cec5SDimitry Andric     CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts));
389*0b57cec5SDimitry Andric   }
390*0b57cec5SDimitry Andric 
391*0b57cec5SDimitry Andric   if(!CfiFunctionMDs.empty()) {
392*0b57cec5SDimitry Andric     NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions");
393*0b57cec5SDimitry Andric     for (auto MD : CfiFunctionMDs)
394*0b57cec5SDimitry Andric       NMD->addOperand(MD);
395*0b57cec5SDimitry Andric   }
396*0b57cec5SDimitry Andric 
397*0b57cec5SDimitry Andric   SmallVector<MDNode *, 8> FunctionAliases;
398*0b57cec5SDimitry Andric   for (auto &A : M.aliases()) {
399*0b57cec5SDimitry Andric     if (!isa<Function>(A.getAliasee()))
400*0b57cec5SDimitry Andric       continue;
401*0b57cec5SDimitry Andric 
402*0b57cec5SDimitry Andric     auto *F = cast<Function>(A.getAliasee());
403*0b57cec5SDimitry Andric 
404*0b57cec5SDimitry Andric     Metadata *Elts[] = {
405*0b57cec5SDimitry Andric         MDString::get(Ctx, A.getName()),
406*0b57cec5SDimitry Andric         MDString::get(Ctx, F->getName()),
407*0b57cec5SDimitry Andric         ConstantAsMetadata::get(
408*0b57cec5SDimitry Andric             ConstantInt::get(Type::getInt8Ty(Ctx), A.getVisibility())),
409*0b57cec5SDimitry Andric         ConstantAsMetadata::get(
410*0b57cec5SDimitry Andric             ConstantInt::get(Type::getInt8Ty(Ctx), A.isWeakForLinker())),
411*0b57cec5SDimitry Andric     };
412*0b57cec5SDimitry Andric 
413*0b57cec5SDimitry Andric     FunctionAliases.push_back(MDTuple::get(Ctx, Elts));
414*0b57cec5SDimitry Andric   }
415*0b57cec5SDimitry Andric 
416*0b57cec5SDimitry Andric   if (!FunctionAliases.empty()) {
417*0b57cec5SDimitry Andric     NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("aliases");
418*0b57cec5SDimitry Andric     for (auto MD : FunctionAliases)
419*0b57cec5SDimitry Andric       NMD->addOperand(MD);
420*0b57cec5SDimitry Andric   }
421*0b57cec5SDimitry Andric 
422*0b57cec5SDimitry Andric   SmallVector<MDNode *, 8> Symvers;
423*0b57cec5SDimitry Andric   ModuleSymbolTable::CollectAsmSymvers(M, [&](StringRef Name, StringRef Alias) {
424*0b57cec5SDimitry Andric     Function *F = M.getFunction(Name);
425*0b57cec5SDimitry Andric     if (!F || F->use_empty())
426*0b57cec5SDimitry Andric       return;
427*0b57cec5SDimitry Andric 
428*0b57cec5SDimitry Andric     Symvers.push_back(MDTuple::get(
429*0b57cec5SDimitry Andric         Ctx, {MDString::get(Ctx, Name), MDString::get(Ctx, Alias)}));
430*0b57cec5SDimitry Andric   });
431*0b57cec5SDimitry Andric 
432*0b57cec5SDimitry Andric   if (!Symvers.empty()) {
433*0b57cec5SDimitry Andric     NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("symvers");
434*0b57cec5SDimitry Andric     for (auto MD : Symvers)
435*0b57cec5SDimitry Andric       NMD->addOperand(MD);
436*0b57cec5SDimitry Andric   }
437*0b57cec5SDimitry Andric 
438*0b57cec5SDimitry Andric   simplifyExternals(*MergedM);
439*0b57cec5SDimitry Andric 
440*0b57cec5SDimitry Andric   // FIXME: Try to re-use BSI and PFI from the original module here.
441*0b57cec5SDimitry Andric   ProfileSummaryInfo PSI(M);
442*0b57cec5SDimitry Andric   ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI);
443*0b57cec5SDimitry Andric 
444*0b57cec5SDimitry Andric   // Mark the merged module as requiring full LTO. We still want an index for
445*0b57cec5SDimitry Andric   // it though, so that it can participate in summary-based dead stripping.
446*0b57cec5SDimitry Andric   MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0));
447*0b57cec5SDimitry Andric   ModuleSummaryIndex MergedMIndex =
448*0b57cec5SDimitry Andric       buildModuleSummaryIndex(*MergedM, nullptr, &PSI);
449*0b57cec5SDimitry Andric 
450*0b57cec5SDimitry Andric   SmallVector<char, 0> Buffer;
451*0b57cec5SDimitry Andric 
452*0b57cec5SDimitry Andric   BitcodeWriter W(Buffer);
453*0b57cec5SDimitry Andric   // Save the module hash produced for the full bitcode, which will
454*0b57cec5SDimitry Andric   // be used in the backends, and use that in the minimized bitcode
455*0b57cec5SDimitry Andric   // produced for the full link.
456*0b57cec5SDimitry Andric   ModuleHash ModHash = {{0}};
457*0b57cec5SDimitry Andric   W.writeModule(M, /*ShouldPreserveUseListOrder=*/false, &Index,
458*0b57cec5SDimitry Andric                 /*GenerateHash=*/true, &ModHash);
459*0b57cec5SDimitry Andric   W.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, &MergedMIndex);
460*0b57cec5SDimitry Andric   W.writeSymtab();
461*0b57cec5SDimitry Andric   W.writeStrtab();
462*0b57cec5SDimitry Andric   OS << Buffer;
463*0b57cec5SDimitry Andric 
464*0b57cec5SDimitry Andric   // If a minimized bitcode module was requested for the thin link, only
465*0b57cec5SDimitry Andric   // the information that is needed by thin link will be written in the
466*0b57cec5SDimitry Andric   // given OS (the merged module will be written as usual).
467*0b57cec5SDimitry Andric   if (ThinLinkOS) {
468*0b57cec5SDimitry Andric     Buffer.clear();
469*0b57cec5SDimitry Andric     BitcodeWriter W2(Buffer);
470*0b57cec5SDimitry Andric     StripDebugInfo(M);
471*0b57cec5SDimitry Andric     W2.writeThinLinkBitcode(M, Index, ModHash);
472*0b57cec5SDimitry Andric     W2.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false,
473*0b57cec5SDimitry Andric                    &MergedMIndex);
474*0b57cec5SDimitry Andric     W2.writeSymtab();
475*0b57cec5SDimitry Andric     W2.writeStrtab();
476*0b57cec5SDimitry Andric     *ThinLinkOS << Buffer;
477*0b57cec5SDimitry Andric   }
478*0b57cec5SDimitry Andric }
479*0b57cec5SDimitry Andric 
480*0b57cec5SDimitry Andric // Check if the LTO Unit splitting has been enabled.
481*0b57cec5SDimitry Andric bool enableSplitLTOUnit(Module &M) {
482*0b57cec5SDimitry Andric   bool EnableSplitLTOUnit = false;
483*0b57cec5SDimitry Andric   if (auto *MD = mdconst::extract_or_null<ConstantInt>(
484*0b57cec5SDimitry Andric           M.getModuleFlag("EnableSplitLTOUnit")))
485*0b57cec5SDimitry Andric     EnableSplitLTOUnit = MD->getZExtValue();
486*0b57cec5SDimitry Andric   return EnableSplitLTOUnit;
487*0b57cec5SDimitry Andric }
488*0b57cec5SDimitry Andric 
489*0b57cec5SDimitry Andric // Returns whether this module needs to be split because it uses type metadata.
490*0b57cec5SDimitry Andric bool hasTypeMetadata(Module &M) {
491*0b57cec5SDimitry Andric   for (auto &GO : M.global_objects()) {
492*0b57cec5SDimitry Andric     if (GO.hasMetadata(LLVMContext::MD_type))
493*0b57cec5SDimitry Andric       return true;
494*0b57cec5SDimitry Andric   }
495*0b57cec5SDimitry Andric   return false;
496*0b57cec5SDimitry Andric }
497*0b57cec5SDimitry Andric 
498*0b57cec5SDimitry Andric void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
499*0b57cec5SDimitry Andric                          function_ref<AAResults &(Function &)> AARGetter,
500*0b57cec5SDimitry Andric                          Module &M, const ModuleSummaryIndex *Index) {
501*0b57cec5SDimitry Andric   std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr;
502*0b57cec5SDimitry Andric   // See if this module has any type metadata. If so, we try to split it
503*0b57cec5SDimitry Andric   // or at least promote type ids to enable WPD.
504*0b57cec5SDimitry Andric   if (hasTypeMetadata(M)) {
505*0b57cec5SDimitry Andric     if (enableSplitLTOUnit(M))
506*0b57cec5SDimitry Andric       return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
507*0b57cec5SDimitry Andric     // Promote type ids as needed for index-based WPD.
508*0b57cec5SDimitry Andric     std::string ModuleId = getUniqueModuleId(&M);
509*0b57cec5SDimitry Andric     if (!ModuleId.empty()) {
510*0b57cec5SDimitry Andric       promoteTypeIds(M, ModuleId);
511*0b57cec5SDimitry Andric       // Need to rebuild the index so that it contains type metadata
512*0b57cec5SDimitry Andric       // for the newly promoted type ids.
513*0b57cec5SDimitry Andric       // FIXME: Probably should not bother building the index at all
514*0b57cec5SDimitry Andric       // in the caller of writeThinLTOBitcode (which does so via the
515*0b57cec5SDimitry Andric       // ModuleSummaryIndexAnalysis pass), since we have to rebuild it
516*0b57cec5SDimitry Andric       // anyway whenever there is type metadata (here or in
517*0b57cec5SDimitry Andric       // splitAndWriteThinLTOBitcode). Just always build it once via the
518*0b57cec5SDimitry Andric       // buildModuleSummaryIndex when Module(s) are ready.
519*0b57cec5SDimitry Andric       ProfileSummaryInfo PSI(M);
5208bcb0991SDimitry Andric       NewIndex = std::make_unique<ModuleSummaryIndex>(
521*0b57cec5SDimitry Andric           buildModuleSummaryIndex(M, nullptr, &PSI));
522*0b57cec5SDimitry Andric       Index = NewIndex.get();
523*0b57cec5SDimitry Andric     }
524*0b57cec5SDimitry Andric   }
525*0b57cec5SDimitry Andric 
526*0b57cec5SDimitry Andric   // Write it out as an unsplit ThinLTO module.
527*0b57cec5SDimitry Andric 
528*0b57cec5SDimitry Andric   // Save the module hash produced for the full bitcode, which will
529*0b57cec5SDimitry Andric   // be used in the backends, and use that in the minimized bitcode
530*0b57cec5SDimitry Andric   // produced for the full link.
531*0b57cec5SDimitry Andric   ModuleHash ModHash = {{0}};
532*0b57cec5SDimitry Andric   WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, Index,
533*0b57cec5SDimitry Andric                      /*GenerateHash=*/true, &ModHash);
534*0b57cec5SDimitry Andric   // If a minimized bitcode module was requested for the thin link, only
535*0b57cec5SDimitry Andric   // the information that is needed by thin link will be written in the
536*0b57cec5SDimitry Andric   // given OS.
537*0b57cec5SDimitry Andric   if (ThinLinkOS && Index)
538*0b57cec5SDimitry Andric     WriteThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash);
539*0b57cec5SDimitry Andric }
540*0b57cec5SDimitry Andric 
541*0b57cec5SDimitry Andric class WriteThinLTOBitcode : public ModulePass {
542*0b57cec5SDimitry Andric   raw_ostream &OS; // raw_ostream to print on
543*0b57cec5SDimitry Andric   // The output stream on which to emit a minimized module for use
544*0b57cec5SDimitry Andric   // just in the thin link, if requested.
545*0b57cec5SDimitry Andric   raw_ostream *ThinLinkOS;
546*0b57cec5SDimitry Andric 
547*0b57cec5SDimitry Andric public:
548*0b57cec5SDimitry Andric   static char ID; // Pass identification, replacement for typeid
549*0b57cec5SDimitry Andric   WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()), ThinLinkOS(nullptr) {
550*0b57cec5SDimitry Andric     initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
551*0b57cec5SDimitry Andric   }
552*0b57cec5SDimitry Andric 
553*0b57cec5SDimitry Andric   explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS)
554*0b57cec5SDimitry Andric       : ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) {
555*0b57cec5SDimitry Andric     initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry());
556*0b57cec5SDimitry Andric   }
557*0b57cec5SDimitry Andric 
558*0b57cec5SDimitry Andric   StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; }
559*0b57cec5SDimitry Andric 
560*0b57cec5SDimitry Andric   bool runOnModule(Module &M) override {
561*0b57cec5SDimitry Andric     const ModuleSummaryIndex *Index =
562*0b57cec5SDimitry Andric         &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex());
563*0b57cec5SDimitry Andric     writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index);
564*0b57cec5SDimitry Andric     return true;
565*0b57cec5SDimitry Andric   }
566*0b57cec5SDimitry Andric   void getAnalysisUsage(AnalysisUsage &AU) const override {
567*0b57cec5SDimitry Andric     AU.setPreservesAll();
568*0b57cec5SDimitry Andric     AU.addRequired<AssumptionCacheTracker>();
569*0b57cec5SDimitry Andric     AU.addRequired<ModuleSummaryIndexWrapperPass>();
570*0b57cec5SDimitry Andric     AU.addRequired<TargetLibraryInfoWrapperPass>();
571*0b57cec5SDimitry Andric   }
572*0b57cec5SDimitry Andric };
573*0b57cec5SDimitry Andric } // anonymous namespace
574*0b57cec5SDimitry Andric 
575*0b57cec5SDimitry Andric char WriteThinLTOBitcode::ID = 0;
576*0b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode",
577*0b57cec5SDimitry Andric                       "Write ThinLTO Bitcode", false, true)
578*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
579*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass)
580*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
581*0b57cec5SDimitry Andric INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode",
582*0b57cec5SDimitry Andric                     "Write ThinLTO Bitcode", false, true)
583*0b57cec5SDimitry Andric 
584*0b57cec5SDimitry Andric ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str,
585*0b57cec5SDimitry Andric                                                 raw_ostream *ThinLinkOS) {
586*0b57cec5SDimitry Andric   return new WriteThinLTOBitcode(Str, ThinLinkOS);
587*0b57cec5SDimitry Andric }
588*0b57cec5SDimitry Andric 
589*0b57cec5SDimitry Andric PreservedAnalyses
590*0b57cec5SDimitry Andric llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) {
591*0b57cec5SDimitry Andric   FunctionAnalysisManager &FAM =
592*0b57cec5SDimitry Andric       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
593*0b57cec5SDimitry Andric   writeThinLTOBitcode(OS, ThinLinkOS,
594*0b57cec5SDimitry Andric                       [&FAM](Function &F) -> AAResults & {
595*0b57cec5SDimitry Andric                         return FAM.getResult<AAManager>(F);
596*0b57cec5SDimitry Andric                       },
597*0b57cec5SDimitry Andric                       M, &AM.getResult<ModuleSummaryIndexAnalysis>(M));
598*0b57cec5SDimitry Andric   return PreservedAnalyses::all();
599*0b57cec5SDimitry Andric }
600