1*0b57cec5SDimitry Andric //===- ThinLTOBitcodeWriter.cpp - Bitcode writing pass for ThinLTO --------===// 2*0b57cec5SDimitry Andric // 3*0b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*0b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5*0b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*0b57cec5SDimitry Andric // 7*0b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 8*0b57cec5SDimitry Andric 9*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" 10*0b57cec5SDimitry Andric #include "llvm/Analysis/BasicAliasAnalysis.h" 11*0b57cec5SDimitry Andric #include "llvm/Analysis/ModuleSummaryAnalysis.h" 12*0b57cec5SDimitry Andric #include "llvm/Analysis/ProfileSummaryInfo.h" 13*0b57cec5SDimitry Andric #include "llvm/Analysis/TypeMetadataUtils.h" 14*0b57cec5SDimitry Andric #include "llvm/Bitcode/BitcodeWriter.h" 15*0b57cec5SDimitry Andric #include "llvm/IR/Constants.h" 16*0b57cec5SDimitry Andric #include "llvm/IR/DebugInfo.h" 17e8d8bef9SDimitry Andric #include "llvm/IR/Instructions.h" 18*0b57cec5SDimitry Andric #include "llvm/IR/Intrinsics.h" 19*0b57cec5SDimitry Andric #include "llvm/IR/Module.h" 20*0b57cec5SDimitry Andric #include "llvm/IR/PassManager.h" 21480093f4SDimitry Andric #include "llvm/InitializePasses.h" 22*0b57cec5SDimitry Andric #include "llvm/Object/ModuleSymbolTable.h" 23*0b57cec5SDimitry Andric #include "llvm/Pass.h" 24*0b57cec5SDimitry Andric #include "llvm/Support/raw_ostream.h" 25*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO.h" 26*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/FunctionAttrs.h" 27*0b57cec5SDimitry Andric #include "llvm/Transforms/IPO/FunctionImport.h" 288bcb0991SDimitry Andric #include "llvm/Transforms/IPO/LowerTypeTests.h" 29*0b57cec5SDimitry Andric #include "llvm/Transforms/Utils/Cloning.h" 30*0b57cec5SDimitry Andric #include "llvm/Transforms/Utils/ModuleUtils.h" 31*0b57cec5SDimitry Andric using namespace llvm; 32*0b57cec5SDimitry Andric 33*0b57cec5SDimitry Andric namespace { 34*0b57cec5SDimitry Andric 3569ade1e0SDimitry Andric // Determine if a promotion alias should be created for a symbol name. 3669ade1e0SDimitry Andric static bool allowPromotionAlias(const std::string &Name) { 3769ade1e0SDimitry Andric // Promotion aliases are used only in inline assembly. It's safe to 3869ade1e0SDimitry Andric // simply skip unusual names. Subset of MCAsmInfo::isAcceptableChar() 3969ade1e0SDimitry Andric // and MCAsmInfoXCOFF::isAcceptableChar(). 4069ade1e0SDimitry Andric for (const char &C : Name) { 4169ade1e0SDimitry Andric if (isAlnum(C) || C == '_' || C == '.') 4269ade1e0SDimitry Andric continue; 4369ade1e0SDimitry Andric return false; 4469ade1e0SDimitry Andric } 4569ade1e0SDimitry Andric return true; 4669ade1e0SDimitry Andric } 4769ade1e0SDimitry Andric 48*0b57cec5SDimitry Andric // Promote each local-linkage entity defined by ExportM and used by ImportM by 49*0b57cec5SDimitry Andric // changing visibility and appending the given ModuleId. 50*0b57cec5SDimitry Andric void promoteInternals(Module &ExportM, Module &ImportM, StringRef ModuleId, 51*0b57cec5SDimitry Andric SetVector<GlobalValue *> &PromoteExtra) { 52*0b57cec5SDimitry Andric DenseMap<const Comdat *, Comdat *> RenamedComdats; 53*0b57cec5SDimitry Andric for (auto &ExportGV : ExportM.global_values()) { 54*0b57cec5SDimitry Andric if (!ExportGV.hasLocalLinkage()) 55*0b57cec5SDimitry Andric continue; 56*0b57cec5SDimitry Andric 57*0b57cec5SDimitry Andric auto Name = ExportGV.getName(); 58*0b57cec5SDimitry Andric GlobalValue *ImportGV = nullptr; 59*0b57cec5SDimitry Andric if (!PromoteExtra.count(&ExportGV)) { 60*0b57cec5SDimitry Andric ImportGV = ImportM.getNamedValue(Name); 61*0b57cec5SDimitry Andric if (!ImportGV) 62*0b57cec5SDimitry Andric continue; 63*0b57cec5SDimitry Andric ImportGV->removeDeadConstantUsers(); 64*0b57cec5SDimitry Andric if (ImportGV->use_empty()) { 65*0b57cec5SDimitry Andric ImportGV->eraseFromParent(); 66*0b57cec5SDimitry Andric continue; 67*0b57cec5SDimitry Andric } 68*0b57cec5SDimitry Andric } 69*0b57cec5SDimitry Andric 7069ade1e0SDimitry Andric std::string OldName = Name.str(); 71*0b57cec5SDimitry Andric std::string NewName = (Name + ModuleId).str(); 72*0b57cec5SDimitry Andric 73*0b57cec5SDimitry Andric if (const auto *C = ExportGV.getComdat()) 74*0b57cec5SDimitry Andric if (C->getName() == Name) 75*0b57cec5SDimitry Andric RenamedComdats.try_emplace(C, ExportM.getOrInsertComdat(NewName)); 76*0b57cec5SDimitry Andric 77*0b57cec5SDimitry Andric ExportGV.setName(NewName); 78*0b57cec5SDimitry Andric ExportGV.setLinkage(GlobalValue::ExternalLinkage); 79*0b57cec5SDimitry Andric ExportGV.setVisibility(GlobalValue::HiddenVisibility); 80*0b57cec5SDimitry Andric 81*0b57cec5SDimitry Andric if (ImportGV) { 82*0b57cec5SDimitry Andric ImportGV->setName(NewName); 83*0b57cec5SDimitry Andric ImportGV->setVisibility(GlobalValue::HiddenVisibility); 84*0b57cec5SDimitry Andric } 8569ade1e0SDimitry Andric 8669ade1e0SDimitry Andric if (isa<Function>(&ExportGV) && allowPromotionAlias(OldName)) { 8769ade1e0SDimitry Andric // Create a local alias with the original name to avoid breaking 8869ade1e0SDimitry Andric // references from inline assembly. 890eae32dcSDimitry Andric std::string Alias = 900eae32dcSDimitry Andric ".lto_set_conditional " + OldName + "," + NewName + "\n"; 9169ade1e0SDimitry Andric ExportM.appendModuleInlineAsm(Alias); 9269ade1e0SDimitry Andric } 93*0b57cec5SDimitry Andric } 94*0b57cec5SDimitry Andric 95*0b57cec5SDimitry Andric if (!RenamedComdats.empty()) 96*0b57cec5SDimitry Andric for (auto &GO : ExportM.global_objects()) 97*0b57cec5SDimitry Andric if (auto *C = GO.getComdat()) { 98*0b57cec5SDimitry Andric auto Replacement = RenamedComdats.find(C); 99*0b57cec5SDimitry Andric if (Replacement != RenamedComdats.end()) 100*0b57cec5SDimitry Andric GO.setComdat(Replacement->second); 101*0b57cec5SDimitry Andric } 102*0b57cec5SDimitry Andric } 103*0b57cec5SDimitry Andric 104*0b57cec5SDimitry Andric // Promote all internal (i.e. distinct) type ids used by the module by replacing 105*0b57cec5SDimitry Andric // them with external type ids formed using the module id. 106*0b57cec5SDimitry Andric // 107*0b57cec5SDimitry Andric // Note that this needs to be done before we clone the module because each clone 108*0b57cec5SDimitry Andric // will receive its own set of distinct metadata nodes. 109*0b57cec5SDimitry Andric void promoteTypeIds(Module &M, StringRef ModuleId) { 110*0b57cec5SDimitry Andric DenseMap<Metadata *, Metadata *> LocalToGlobal; 111*0b57cec5SDimitry Andric auto ExternalizeTypeId = [&](CallInst *CI, unsigned ArgNo) { 112*0b57cec5SDimitry Andric Metadata *MD = 113*0b57cec5SDimitry Andric cast<MetadataAsValue>(CI->getArgOperand(ArgNo))->getMetadata(); 114*0b57cec5SDimitry Andric 115*0b57cec5SDimitry Andric if (isa<MDNode>(MD) && cast<MDNode>(MD)->isDistinct()) { 116*0b57cec5SDimitry Andric Metadata *&GlobalMD = LocalToGlobal[MD]; 117*0b57cec5SDimitry Andric if (!GlobalMD) { 118*0b57cec5SDimitry Andric std::string NewName = (Twine(LocalToGlobal.size()) + ModuleId).str(); 119*0b57cec5SDimitry Andric GlobalMD = MDString::get(M.getContext(), NewName); 120*0b57cec5SDimitry Andric } 121*0b57cec5SDimitry Andric 122*0b57cec5SDimitry Andric CI->setArgOperand(ArgNo, 123*0b57cec5SDimitry Andric MetadataAsValue::get(M.getContext(), GlobalMD)); 124*0b57cec5SDimitry Andric } 125*0b57cec5SDimitry Andric }; 126*0b57cec5SDimitry Andric 127*0b57cec5SDimitry Andric if (Function *TypeTestFunc = 128*0b57cec5SDimitry Andric M.getFunction(Intrinsic::getName(Intrinsic::type_test))) { 129*0b57cec5SDimitry Andric for (const Use &U : TypeTestFunc->uses()) { 130*0b57cec5SDimitry Andric auto CI = cast<CallInst>(U.getUser()); 131*0b57cec5SDimitry Andric ExternalizeTypeId(CI, 1); 132*0b57cec5SDimitry Andric } 133*0b57cec5SDimitry Andric } 134*0b57cec5SDimitry Andric 135*0b57cec5SDimitry Andric if (Function *TypeCheckedLoadFunc = 136*0b57cec5SDimitry Andric M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load))) { 137*0b57cec5SDimitry Andric for (const Use &U : TypeCheckedLoadFunc->uses()) { 138*0b57cec5SDimitry Andric auto CI = cast<CallInst>(U.getUser()); 139*0b57cec5SDimitry Andric ExternalizeTypeId(CI, 2); 140*0b57cec5SDimitry Andric } 141*0b57cec5SDimitry Andric } 142*0b57cec5SDimitry Andric 143*0b57cec5SDimitry Andric for (GlobalObject &GO : M.global_objects()) { 144*0b57cec5SDimitry Andric SmallVector<MDNode *, 1> MDs; 145*0b57cec5SDimitry Andric GO.getMetadata(LLVMContext::MD_type, MDs); 146*0b57cec5SDimitry Andric 147*0b57cec5SDimitry Andric GO.eraseMetadata(LLVMContext::MD_type); 148*0b57cec5SDimitry Andric for (auto MD : MDs) { 149*0b57cec5SDimitry Andric auto I = LocalToGlobal.find(MD->getOperand(1)); 150*0b57cec5SDimitry Andric if (I == LocalToGlobal.end()) { 151*0b57cec5SDimitry Andric GO.addMetadata(LLVMContext::MD_type, *MD); 152*0b57cec5SDimitry Andric continue; 153*0b57cec5SDimitry Andric } 154*0b57cec5SDimitry Andric GO.addMetadata( 155*0b57cec5SDimitry Andric LLVMContext::MD_type, 156*0b57cec5SDimitry Andric *MDNode::get(M.getContext(), {MD->getOperand(0), I->second})); 157*0b57cec5SDimitry Andric } 158*0b57cec5SDimitry Andric } 159*0b57cec5SDimitry Andric } 160*0b57cec5SDimitry Andric 161*0b57cec5SDimitry Andric // Drop unused globals, and drop type information from function declarations. 162*0b57cec5SDimitry Andric // FIXME: If we made functions typeless then there would be no need to do this. 163*0b57cec5SDimitry Andric void simplifyExternals(Module &M) { 164*0b57cec5SDimitry Andric FunctionType *EmptyFT = 165*0b57cec5SDimitry Andric FunctionType::get(Type::getVoidTy(M.getContext()), false); 166*0b57cec5SDimitry Andric 167349cc55cSDimitry Andric for (Function &F : llvm::make_early_inc_range(M)) { 168*0b57cec5SDimitry Andric if (F.isDeclaration() && F.use_empty()) { 169*0b57cec5SDimitry Andric F.eraseFromParent(); 170*0b57cec5SDimitry Andric continue; 171*0b57cec5SDimitry Andric } 172*0b57cec5SDimitry Andric 173*0b57cec5SDimitry Andric if (!F.isDeclaration() || F.getFunctionType() == EmptyFT || 174*0b57cec5SDimitry Andric // Changing the type of an intrinsic may invalidate the IR. 175*0b57cec5SDimitry Andric F.getName().startswith("llvm.")) 176*0b57cec5SDimitry Andric continue; 177*0b57cec5SDimitry Andric 178*0b57cec5SDimitry Andric Function *NewF = 179*0b57cec5SDimitry Andric Function::Create(EmptyFT, GlobalValue::ExternalLinkage, 180*0b57cec5SDimitry Andric F.getAddressSpace(), "", &M); 181fe6060f1SDimitry Andric NewF->copyAttributesFrom(&F); 182fe6060f1SDimitry Andric // Only copy function attribtues. 183349cc55cSDimitry Andric NewF->setAttributes(AttributeList::get(M.getContext(), 184349cc55cSDimitry Andric AttributeList::FunctionIndex, 185349cc55cSDimitry Andric F.getAttributes().getFnAttrs())); 186*0b57cec5SDimitry Andric NewF->takeName(&F); 187*0b57cec5SDimitry Andric F.replaceAllUsesWith(ConstantExpr::getBitCast(NewF, F.getType())); 188*0b57cec5SDimitry Andric F.eraseFromParent(); 189*0b57cec5SDimitry Andric } 190*0b57cec5SDimitry Andric 191349cc55cSDimitry Andric for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) { 192*0b57cec5SDimitry Andric if (GV.isDeclaration() && GV.use_empty()) { 193*0b57cec5SDimitry Andric GV.eraseFromParent(); 194*0b57cec5SDimitry Andric continue; 195*0b57cec5SDimitry Andric } 196*0b57cec5SDimitry Andric } 197*0b57cec5SDimitry Andric } 198*0b57cec5SDimitry Andric 199*0b57cec5SDimitry Andric static void 200*0b57cec5SDimitry Andric filterModule(Module *M, 201*0b57cec5SDimitry Andric function_ref<bool(const GlobalValue *)> ShouldKeepDefinition) { 202*0b57cec5SDimitry Andric std::vector<GlobalValue *> V; 203*0b57cec5SDimitry Andric for (GlobalValue &GV : M->global_values()) 204*0b57cec5SDimitry Andric if (!ShouldKeepDefinition(&GV)) 205*0b57cec5SDimitry Andric V.push_back(&GV); 206*0b57cec5SDimitry Andric 207*0b57cec5SDimitry Andric for (GlobalValue *GV : V) 208*0b57cec5SDimitry Andric if (!convertToDeclaration(*GV)) 209*0b57cec5SDimitry Andric GV->eraseFromParent(); 210*0b57cec5SDimitry Andric } 211*0b57cec5SDimitry Andric 212*0b57cec5SDimitry Andric void forEachVirtualFunction(Constant *C, function_ref<void(Function *)> Fn) { 213*0b57cec5SDimitry Andric if (auto *F = dyn_cast<Function>(C)) 214*0b57cec5SDimitry Andric return Fn(F); 215*0b57cec5SDimitry Andric if (isa<GlobalValue>(C)) 216*0b57cec5SDimitry Andric return; 217*0b57cec5SDimitry Andric for (Value *Op : C->operands()) 218*0b57cec5SDimitry Andric forEachVirtualFunction(cast<Constant>(Op), Fn); 219*0b57cec5SDimitry Andric } 220*0b57cec5SDimitry Andric 221fe6060f1SDimitry Andric // Clone any @llvm[.compiler].used over to the new module and append 222fe6060f1SDimitry Andric // values whose defs were cloned into that module. 223fe6060f1SDimitry Andric static void cloneUsedGlobalVariables(const Module &SrcM, Module &DestM, 224fe6060f1SDimitry Andric bool CompilerUsed) { 225fe6060f1SDimitry Andric SmallVector<GlobalValue *, 4> Used, NewUsed; 226fe6060f1SDimitry Andric // First collect those in the llvm[.compiler].used set. 227fe6060f1SDimitry Andric collectUsedGlobalVariables(SrcM, Used, CompilerUsed); 228fe6060f1SDimitry Andric // Next build a set of the equivalent values defined in DestM. 229fe6060f1SDimitry Andric for (auto *V : Used) { 230fe6060f1SDimitry Andric auto *GV = DestM.getNamedValue(V->getName()); 231fe6060f1SDimitry Andric if (GV && !GV->isDeclaration()) 232fe6060f1SDimitry Andric NewUsed.push_back(GV); 233fe6060f1SDimitry Andric } 234fe6060f1SDimitry Andric // Finally, add them to a llvm[.compiler].used variable in DestM. 235fe6060f1SDimitry Andric if (CompilerUsed) 236fe6060f1SDimitry Andric appendToCompilerUsed(DestM, NewUsed); 237fe6060f1SDimitry Andric else 238fe6060f1SDimitry Andric appendToUsed(DestM, NewUsed); 239fe6060f1SDimitry Andric } 240fe6060f1SDimitry Andric 241*0b57cec5SDimitry Andric // If it's possible to split M into regular and thin LTO parts, do so and write 242*0b57cec5SDimitry Andric // a multi-module bitcode file with the two parts to OS. Otherwise, write only a 243*0b57cec5SDimitry Andric // regular LTO bitcode file to OS. 244*0b57cec5SDimitry Andric void splitAndWriteThinLTOBitcode( 245*0b57cec5SDimitry Andric raw_ostream &OS, raw_ostream *ThinLinkOS, 246*0b57cec5SDimitry Andric function_ref<AAResults &(Function &)> AARGetter, Module &M) { 247*0b57cec5SDimitry Andric std::string ModuleId = getUniqueModuleId(&M); 248*0b57cec5SDimitry Andric if (ModuleId.empty()) { 249*0b57cec5SDimitry Andric // We couldn't generate a module ID for this module, write it out as a 250*0b57cec5SDimitry Andric // regular LTO module with an index for summary-based dead stripping. 251*0b57cec5SDimitry Andric ProfileSummaryInfo PSI(M); 252*0b57cec5SDimitry Andric M.addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); 253*0b57cec5SDimitry Andric ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI); 254*0b57cec5SDimitry Andric WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, &Index); 255*0b57cec5SDimitry Andric 256*0b57cec5SDimitry Andric if (ThinLinkOS) 257*0b57cec5SDimitry Andric // We don't have a ThinLTO part, but still write the module to the 258*0b57cec5SDimitry Andric // ThinLinkOS if requested so that the expected output file is produced. 259*0b57cec5SDimitry Andric WriteBitcodeToFile(M, *ThinLinkOS, /*ShouldPreserveUseListOrder=*/false, 260*0b57cec5SDimitry Andric &Index); 261*0b57cec5SDimitry Andric 262*0b57cec5SDimitry Andric return; 263*0b57cec5SDimitry Andric } 264*0b57cec5SDimitry Andric 265*0b57cec5SDimitry Andric promoteTypeIds(M, ModuleId); 266*0b57cec5SDimitry Andric 2678bcb0991SDimitry Andric // Returns whether a global or its associated global has attached type 2688bcb0991SDimitry Andric // metadata. The former may participate in CFI or whole-program 2698bcb0991SDimitry Andric // devirtualization, so they need to appear in the merged module instead of 2708bcb0991SDimitry Andric // the thin LTO module. Similarly, globals that are associated with globals 2718bcb0991SDimitry Andric // with type metadata need to appear in the merged module because they will 2728bcb0991SDimitry Andric // reference the global's section directly. 273*0b57cec5SDimitry Andric auto HasTypeMetadata = [](const GlobalObject *GO) { 2748bcb0991SDimitry Andric if (MDNode *MD = GO->getMetadata(LLVMContext::MD_associated)) 2758bcb0991SDimitry Andric if (auto *AssocVM = dyn_cast_or_null<ValueAsMetadata>(MD->getOperand(0))) 2768bcb0991SDimitry Andric if (auto *AssocGO = dyn_cast<GlobalObject>(AssocVM->getValue())) 2778bcb0991SDimitry Andric if (AssocGO->hasMetadata(LLVMContext::MD_type)) 2788bcb0991SDimitry Andric return true; 279*0b57cec5SDimitry Andric return GO->hasMetadata(LLVMContext::MD_type); 280*0b57cec5SDimitry Andric }; 281*0b57cec5SDimitry Andric 282*0b57cec5SDimitry Andric // Collect the set of virtual functions that are eligible for virtual constant 283*0b57cec5SDimitry Andric // propagation. Each eligible function must not access memory, must return 284*0b57cec5SDimitry Andric // an integer of width <=64 bits, must take at least one argument, must not 285*0b57cec5SDimitry Andric // use its first argument (assumed to be "this") and all arguments other than 286*0b57cec5SDimitry Andric // the first one must be of <=64 bit integer type. 287*0b57cec5SDimitry Andric // 288*0b57cec5SDimitry Andric // Note that we test whether this copy of the function is readnone, rather 289*0b57cec5SDimitry Andric // than testing function attributes, which must hold for any copy of the 290*0b57cec5SDimitry Andric // function, even a less optimized version substituted at link time. This is 291*0b57cec5SDimitry Andric // sound because the virtual constant propagation optimizations effectively 292*0b57cec5SDimitry Andric // inline all implementations of the virtual function into each call site, 293*0b57cec5SDimitry Andric // rather than using function attributes to perform local optimization. 294*0b57cec5SDimitry Andric DenseSet<const Function *> EligibleVirtualFns; 295*0b57cec5SDimitry Andric // If any member of a comdat lives in MergedM, put all members of that 296*0b57cec5SDimitry Andric // comdat in MergedM to keep the comdat together. 297*0b57cec5SDimitry Andric DenseSet<const Comdat *> MergedMComdats; 298*0b57cec5SDimitry Andric for (GlobalVariable &GV : M.globals()) 299*0b57cec5SDimitry Andric if (HasTypeMetadata(&GV)) { 300*0b57cec5SDimitry Andric if (const auto *C = GV.getComdat()) 301*0b57cec5SDimitry Andric MergedMComdats.insert(C); 302*0b57cec5SDimitry Andric forEachVirtualFunction(GV.getInitializer(), [&](Function *F) { 303*0b57cec5SDimitry Andric auto *RT = dyn_cast<IntegerType>(F->getReturnType()); 304*0b57cec5SDimitry Andric if (!RT || RT->getBitWidth() > 64 || F->arg_empty() || 305*0b57cec5SDimitry Andric !F->arg_begin()->use_empty()) 306*0b57cec5SDimitry Andric return; 307e8d8bef9SDimitry Andric for (auto &Arg : drop_begin(F->args())) { 308*0b57cec5SDimitry Andric auto *ArgT = dyn_cast<IntegerType>(Arg.getType()); 309*0b57cec5SDimitry Andric if (!ArgT || ArgT->getBitWidth() > 64) 310*0b57cec5SDimitry Andric return; 311*0b57cec5SDimitry Andric } 312*0b57cec5SDimitry Andric if (!F->isDeclaration() && 31381ad6265SDimitry Andric computeFunctionBodyMemoryAccess(*F, AARGetter(*F)) == 31481ad6265SDimitry Andric FMRB_DoesNotAccessMemory) 315*0b57cec5SDimitry Andric EligibleVirtualFns.insert(F); 316*0b57cec5SDimitry Andric }); 317*0b57cec5SDimitry Andric } 318*0b57cec5SDimitry Andric 319*0b57cec5SDimitry Andric ValueToValueMapTy VMap; 320*0b57cec5SDimitry Andric std::unique_ptr<Module> MergedM( 321*0b57cec5SDimitry Andric CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool { 322*0b57cec5SDimitry Andric if (const auto *C = GV->getComdat()) 323*0b57cec5SDimitry Andric if (MergedMComdats.count(C)) 324*0b57cec5SDimitry Andric return true; 325*0b57cec5SDimitry Andric if (auto *F = dyn_cast<Function>(GV)) 326*0b57cec5SDimitry Andric return EligibleVirtualFns.count(F); 327349cc55cSDimitry Andric if (auto *GVar = 328349cc55cSDimitry Andric dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject())) 329*0b57cec5SDimitry Andric return HasTypeMetadata(GVar); 330*0b57cec5SDimitry Andric return false; 331*0b57cec5SDimitry Andric })); 332*0b57cec5SDimitry Andric StripDebugInfo(*MergedM); 333*0b57cec5SDimitry Andric MergedM->setModuleInlineAsm(""); 334*0b57cec5SDimitry Andric 335fe6060f1SDimitry Andric // Clone any llvm.*used globals to ensure the included values are 336fe6060f1SDimitry Andric // not deleted. 337fe6060f1SDimitry Andric cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ false); 338fe6060f1SDimitry Andric cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true); 339fe6060f1SDimitry Andric 340*0b57cec5SDimitry Andric for (Function &F : *MergedM) 341*0b57cec5SDimitry Andric if (!F.isDeclaration()) { 342*0b57cec5SDimitry Andric // Reset the linkage of all functions eligible for virtual constant 343*0b57cec5SDimitry Andric // propagation. The canonical definitions live in the thin LTO module so 344*0b57cec5SDimitry Andric // that they can be imported. 345*0b57cec5SDimitry Andric F.setLinkage(GlobalValue::AvailableExternallyLinkage); 346*0b57cec5SDimitry Andric F.setComdat(nullptr); 347*0b57cec5SDimitry Andric } 348*0b57cec5SDimitry Andric 349*0b57cec5SDimitry Andric SetVector<GlobalValue *> CfiFunctions; 350*0b57cec5SDimitry Andric for (auto &F : M) 351*0b57cec5SDimitry Andric if ((!F.hasLocalLinkage() || F.hasAddressTaken()) && HasTypeMetadata(&F)) 352*0b57cec5SDimitry Andric CfiFunctions.insert(&F); 353*0b57cec5SDimitry Andric 354*0b57cec5SDimitry Andric // Remove all globals with type metadata, globals with comdats that live in 355*0b57cec5SDimitry Andric // MergedM, and aliases pointing to such globals from the thin LTO module. 356*0b57cec5SDimitry Andric filterModule(&M, [&](const GlobalValue *GV) { 357349cc55cSDimitry Andric if (auto *GVar = dyn_cast_or_null<GlobalVariable>(GV->getAliaseeObject())) 358*0b57cec5SDimitry Andric if (HasTypeMetadata(GVar)) 359*0b57cec5SDimitry Andric return false; 360*0b57cec5SDimitry Andric if (const auto *C = GV->getComdat()) 361*0b57cec5SDimitry Andric if (MergedMComdats.count(C)) 362*0b57cec5SDimitry Andric return false; 363*0b57cec5SDimitry Andric return true; 364*0b57cec5SDimitry Andric }); 365*0b57cec5SDimitry Andric 366*0b57cec5SDimitry Andric promoteInternals(*MergedM, M, ModuleId, CfiFunctions); 367*0b57cec5SDimitry Andric promoteInternals(M, *MergedM, ModuleId, CfiFunctions); 368*0b57cec5SDimitry Andric 369*0b57cec5SDimitry Andric auto &Ctx = MergedM->getContext(); 370*0b57cec5SDimitry Andric SmallVector<MDNode *, 8> CfiFunctionMDs; 371*0b57cec5SDimitry Andric for (auto V : CfiFunctions) { 372*0b57cec5SDimitry Andric Function &F = *cast<Function>(V); 373*0b57cec5SDimitry Andric SmallVector<MDNode *, 2> Types; 374*0b57cec5SDimitry Andric F.getMetadata(LLVMContext::MD_type, Types); 375*0b57cec5SDimitry Andric 376*0b57cec5SDimitry Andric SmallVector<Metadata *, 4> Elts; 377*0b57cec5SDimitry Andric Elts.push_back(MDString::get(Ctx, F.getName())); 378*0b57cec5SDimitry Andric CfiFunctionLinkage Linkage; 3798bcb0991SDimitry Andric if (lowertypetests::isJumpTableCanonical(&F)) 380*0b57cec5SDimitry Andric Linkage = CFL_Definition; 3818bcb0991SDimitry Andric else if (F.hasExternalWeakLinkage()) 382*0b57cec5SDimitry Andric Linkage = CFL_WeakDeclaration; 383*0b57cec5SDimitry Andric else 384*0b57cec5SDimitry Andric Linkage = CFL_Declaration; 385*0b57cec5SDimitry Andric Elts.push_back(ConstantAsMetadata::get( 386*0b57cec5SDimitry Andric llvm::ConstantInt::get(Type::getInt8Ty(Ctx), Linkage))); 387e8d8bef9SDimitry Andric append_range(Elts, Types); 388*0b57cec5SDimitry Andric CfiFunctionMDs.push_back(MDTuple::get(Ctx, Elts)); 389*0b57cec5SDimitry Andric } 390*0b57cec5SDimitry Andric 391*0b57cec5SDimitry Andric if(!CfiFunctionMDs.empty()) { 392*0b57cec5SDimitry Andric NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("cfi.functions"); 393*0b57cec5SDimitry Andric for (auto MD : CfiFunctionMDs) 394*0b57cec5SDimitry Andric NMD->addOperand(MD); 395*0b57cec5SDimitry Andric } 396*0b57cec5SDimitry Andric 397*0b57cec5SDimitry Andric SmallVector<MDNode *, 8> FunctionAliases; 398*0b57cec5SDimitry Andric for (auto &A : M.aliases()) { 399*0b57cec5SDimitry Andric if (!isa<Function>(A.getAliasee())) 400*0b57cec5SDimitry Andric continue; 401*0b57cec5SDimitry Andric 402*0b57cec5SDimitry Andric auto *F = cast<Function>(A.getAliasee()); 403*0b57cec5SDimitry Andric 404*0b57cec5SDimitry Andric Metadata *Elts[] = { 405*0b57cec5SDimitry Andric MDString::get(Ctx, A.getName()), 406*0b57cec5SDimitry Andric MDString::get(Ctx, F->getName()), 407*0b57cec5SDimitry Andric ConstantAsMetadata::get( 408*0b57cec5SDimitry Andric ConstantInt::get(Type::getInt8Ty(Ctx), A.getVisibility())), 409*0b57cec5SDimitry Andric ConstantAsMetadata::get( 410*0b57cec5SDimitry Andric ConstantInt::get(Type::getInt8Ty(Ctx), A.isWeakForLinker())), 411*0b57cec5SDimitry Andric }; 412*0b57cec5SDimitry Andric 413*0b57cec5SDimitry Andric FunctionAliases.push_back(MDTuple::get(Ctx, Elts)); 414*0b57cec5SDimitry Andric } 415*0b57cec5SDimitry Andric 416*0b57cec5SDimitry Andric if (!FunctionAliases.empty()) { 417*0b57cec5SDimitry Andric NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("aliases"); 418*0b57cec5SDimitry Andric for (auto MD : FunctionAliases) 419*0b57cec5SDimitry Andric NMD->addOperand(MD); 420*0b57cec5SDimitry Andric } 421*0b57cec5SDimitry Andric 422*0b57cec5SDimitry Andric SmallVector<MDNode *, 8> Symvers; 423*0b57cec5SDimitry Andric ModuleSymbolTable::CollectAsmSymvers(M, [&](StringRef Name, StringRef Alias) { 424*0b57cec5SDimitry Andric Function *F = M.getFunction(Name); 425*0b57cec5SDimitry Andric if (!F || F->use_empty()) 426*0b57cec5SDimitry Andric return; 427*0b57cec5SDimitry Andric 428*0b57cec5SDimitry Andric Symvers.push_back(MDTuple::get( 429*0b57cec5SDimitry Andric Ctx, {MDString::get(Ctx, Name), MDString::get(Ctx, Alias)})); 430*0b57cec5SDimitry Andric }); 431*0b57cec5SDimitry Andric 432*0b57cec5SDimitry Andric if (!Symvers.empty()) { 433*0b57cec5SDimitry Andric NamedMDNode *NMD = MergedM->getOrInsertNamedMetadata("symvers"); 434*0b57cec5SDimitry Andric for (auto MD : Symvers) 435*0b57cec5SDimitry Andric NMD->addOperand(MD); 436*0b57cec5SDimitry Andric } 437*0b57cec5SDimitry Andric 438*0b57cec5SDimitry Andric simplifyExternals(*MergedM); 439*0b57cec5SDimitry Andric 440*0b57cec5SDimitry Andric // FIXME: Try to re-use BSI and PFI from the original module here. 441*0b57cec5SDimitry Andric ProfileSummaryInfo PSI(M); 442*0b57cec5SDimitry Andric ModuleSummaryIndex Index = buildModuleSummaryIndex(M, nullptr, &PSI); 443*0b57cec5SDimitry Andric 444*0b57cec5SDimitry Andric // Mark the merged module as requiring full LTO. We still want an index for 445*0b57cec5SDimitry Andric // it though, so that it can participate in summary-based dead stripping. 446*0b57cec5SDimitry Andric MergedM->addModuleFlag(Module::Error, "ThinLTO", uint32_t(0)); 447*0b57cec5SDimitry Andric ModuleSummaryIndex MergedMIndex = 448*0b57cec5SDimitry Andric buildModuleSummaryIndex(*MergedM, nullptr, &PSI); 449*0b57cec5SDimitry Andric 450*0b57cec5SDimitry Andric SmallVector<char, 0> Buffer; 451*0b57cec5SDimitry Andric 452*0b57cec5SDimitry Andric BitcodeWriter W(Buffer); 453*0b57cec5SDimitry Andric // Save the module hash produced for the full bitcode, which will 454*0b57cec5SDimitry Andric // be used in the backends, and use that in the minimized bitcode 455*0b57cec5SDimitry Andric // produced for the full link. 456*0b57cec5SDimitry Andric ModuleHash ModHash = {{0}}; 457*0b57cec5SDimitry Andric W.writeModule(M, /*ShouldPreserveUseListOrder=*/false, &Index, 458*0b57cec5SDimitry Andric /*GenerateHash=*/true, &ModHash); 459*0b57cec5SDimitry Andric W.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, &MergedMIndex); 460*0b57cec5SDimitry Andric W.writeSymtab(); 461*0b57cec5SDimitry Andric W.writeStrtab(); 462*0b57cec5SDimitry Andric OS << Buffer; 463*0b57cec5SDimitry Andric 464*0b57cec5SDimitry Andric // If a minimized bitcode module was requested for the thin link, only 465*0b57cec5SDimitry Andric // the information that is needed by thin link will be written in the 466*0b57cec5SDimitry Andric // given OS (the merged module will be written as usual). 467*0b57cec5SDimitry Andric if (ThinLinkOS) { 468*0b57cec5SDimitry Andric Buffer.clear(); 469*0b57cec5SDimitry Andric BitcodeWriter W2(Buffer); 470*0b57cec5SDimitry Andric StripDebugInfo(M); 471*0b57cec5SDimitry Andric W2.writeThinLinkBitcode(M, Index, ModHash); 472*0b57cec5SDimitry Andric W2.writeModule(*MergedM, /*ShouldPreserveUseListOrder=*/false, 473*0b57cec5SDimitry Andric &MergedMIndex); 474*0b57cec5SDimitry Andric W2.writeSymtab(); 475*0b57cec5SDimitry Andric W2.writeStrtab(); 476*0b57cec5SDimitry Andric *ThinLinkOS << Buffer; 477*0b57cec5SDimitry Andric } 478*0b57cec5SDimitry Andric } 479*0b57cec5SDimitry Andric 480*0b57cec5SDimitry Andric // Check if the LTO Unit splitting has been enabled. 481*0b57cec5SDimitry Andric bool enableSplitLTOUnit(Module &M) { 482*0b57cec5SDimitry Andric bool EnableSplitLTOUnit = false; 483*0b57cec5SDimitry Andric if (auto *MD = mdconst::extract_or_null<ConstantInt>( 484*0b57cec5SDimitry Andric M.getModuleFlag("EnableSplitLTOUnit"))) 485*0b57cec5SDimitry Andric EnableSplitLTOUnit = MD->getZExtValue(); 486*0b57cec5SDimitry Andric return EnableSplitLTOUnit; 487*0b57cec5SDimitry Andric } 488*0b57cec5SDimitry Andric 489*0b57cec5SDimitry Andric // Returns whether this module needs to be split because it uses type metadata. 490*0b57cec5SDimitry Andric bool hasTypeMetadata(Module &M) { 491*0b57cec5SDimitry Andric for (auto &GO : M.global_objects()) { 492*0b57cec5SDimitry Andric if (GO.hasMetadata(LLVMContext::MD_type)) 493*0b57cec5SDimitry Andric return true; 494*0b57cec5SDimitry Andric } 495*0b57cec5SDimitry Andric return false; 496*0b57cec5SDimitry Andric } 497*0b57cec5SDimitry Andric 498*0b57cec5SDimitry Andric void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS, 499*0b57cec5SDimitry Andric function_ref<AAResults &(Function &)> AARGetter, 500*0b57cec5SDimitry Andric Module &M, const ModuleSummaryIndex *Index) { 501*0b57cec5SDimitry Andric std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr; 502*0b57cec5SDimitry Andric // See if this module has any type metadata. If so, we try to split it 503*0b57cec5SDimitry Andric // or at least promote type ids to enable WPD. 504*0b57cec5SDimitry Andric if (hasTypeMetadata(M)) { 505*0b57cec5SDimitry Andric if (enableSplitLTOUnit(M)) 506*0b57cec5SDimitry Andric return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M); 507*0b57cec5SDimitry Andric // Promote type ids as needed for index-based WPD. 508*0b57cec5SDimitry Andric std::string ModuleId = getUniqueModuleId(&M); 509*0b57cec5SDimitry Andric if (!ModuleId.empty()) { 510*0b57cec5SDimitry Andric promoteTypeIds(M, ModuleId); 511*0b57cec5SDimitry Andric // Need to rebuild the index so that it contains type metadata 512*0b57cec5SDimitry Andric // for the newly promoted type ids. 513*0b57cec5SDimitry Andric // FIXME: Probably should not bother building the index at all 514*0b57cec5SDimitry Andric // in the caller of writeThinLTOBitcode (which does so via the 515*0b57cec5SDimitry Andric // ModuleSummaryIndexAnalysis pass), since we have to rebuild it 516*0b57cec5SDimitry Andric // anyway whenever there is type metadata (here or in 517*0b57cec5SDimitry Andric // splitAndWriteThinLTOBitcode). Just always build it once via the 518*0b57cec5SDimitry Andric // buildModuleSummaryIndex when Module(s) are ready. 519*0b57cec5SDimitry Andric ProfileSummaryInfo PSI(M); 5208bcb0991SDimitry Andric NewIndex = std::make_unique<ModuleSummaryIndex>( 521*0b57cec5SDimitry Andric buildModuleSummaryIndex(M, nullptr, &PSI)); 522*0b57cec5SDimitry Andric Index = NewIndex.get(); 523*0b57cec5SDimitry Andric } 524*0b57cec5SDimitry Andric } 525*0b57cec5SDimitry Andric 526*0b57cec5SDimitry Andric // Write it out as an unsplit ThinLTO module. 527*0b57cec5SDimitry Andric 528*0b57cec5SDimitry Andric // Save the module hash produced for the full bitcode, which will 529*0b57cec5SDimitry Andric // be used in the backends, and use that in the minimized bitcode 530*0b57cec5SDimitry Andric // produced for the full link. 531*0b57cec5SDimitry Andric ModuleHash ModHash = {{0}}; 532*0b57cec5SDimitry Andric WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false, Index, 533*0b57cec5SDimitry Andric /*GenerateHash=*/true, &ModHash); 534*0b57cec5SDimitry Andric // If a minimized bitcode module was requested for the thin link, only 535*0b57cec5SDimitry Andric // the information that is needed by thin link will be written in the 536*0b57cec5SDimitry Andric // given OS. 537*0b57cec5SDimitry Andric if (ThinLinkOS && Index) 5381fd87a68SDimitry Andric writeThinLinkBitcodeToFile(M, *ThinLinkOS, *Index, ModHash); 539*0b57cec5SDimitry Andric } 540*0b57cec5SDimitry Andric 541*0b57cec5SDimitry Andric class WriteThinLTOBitcode : public ModulePass { 542*0b57cec5SDimitry Andric raw_ostream &OS; // raw_ostream to print on 543*0b57cec5SDimitry Andric // The output stream on which to emit a minimized module for use 544*0b57cec5SDimitry Andric // just in the thin link, if requested. 54581ad6265SDimitry Andric raw_ostream *ThinLinkOS = nullptr; 546*0b57cec5SDimitry Andric 547*0b57cec5SDimitry Andric public: 548*0b57cec5SDimitry Andric static char ID; // Pass identification, replacement for typeid 54981ad6265SDimitry Andric WriteThinLTOBitcode() : ModulePass(ID), OS(dbgs()) { 550*0b57cec5SDimitry Andric initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); 551*0b57cec5SDimitry Andric } 552*0b57cec5SDimitry Andric 553*0b57cec5SDimitry Andric explicit WriteThinLTOBitcode(raw_ostream &o, raw_ostream *ThinLinkOS) 554*0b57cec5SDimitry Andric : ModulePass(ID), OS(o), ThinLinkOS(ThinLinkOS) { 555*0b57cec5SDimitry Andric initializeWriteThinLTOBitcodePass(*PassRegistry::getPassRegistry()); 556*0b57cec5SDimitry Andric } 557*0b57cec5SDimitry Andric 558*0b57cec5SDimitry Andric StringRef getPassName() const override { return "ThinLTO Bitcode Writer"; } 559*0b57cec5SDimitry Andric 560*0b57cec5SDimitry Andric bool runOnModule(Module &M) override { 561*0b57cec5SDimitry Andric const ModuleSummaryIndex *Index = 562*0b57cec5SDimitry Andric &(getAnalysis<ModuleSummaryIndexWrapperPass>().getIndex()); 563*0b57cec5SDimitry Andric writeThinLTOBitcode(OS, ThinLinkOS, LegacyAARGetter(*this), M, Index); 564*0b57cec5SDimitry Andric return true; 565*0b57cec5SDimitry Andric } 566*0b57cec5SDimitry Andric void getAnalysisUsage(AnalysisUsage &AU) const override { 567*0b57cec5SDimitry Andric AU.setPreservesAll(); 568*0b57cec5SDimitry Andric AU.addRequired<AssumptionCacheTracker>(); 569*0b57cec5SDimitry Andric AU.addRequired<ModuleSummaryIndexWrapperPass>(); 570*0b57cec5SDimitry Andric AU.addRequired<TargetLibraryInfoWrapperPass>(); 571*0b57cec5SDimitry Andric } 572*0b57cec5SDimitry Andric }; 573*0b57cec5SDimitry Andric } // anonymous namespace 574*0b57cec5SDimitry Andric 575*0b57cec5SDimitry Andric char WriteThinLTOBitcode::ID = 0; 576*0b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(WriteThinLTOBitcode, "write-thinlto-bitcode", 577*0b57cec5SDimitry Andric "Write ThinLTO Bitcode", false, true) 578*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 579*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ModuleSummaryIndexWrapperPass) 580*0b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 581*0b57cec5SDimitry Andric INITIALIZE_PASS_END(WriteThinLTOBitcode, "write-thinlto-bitcode", 582*0b57cec5SDimitry Andric "Write ThinLTO Bitcode", false, true) 583*0b57cec5SDimitry Andric 584*0b57cec5SDimitry Andric ModulePass *llvm::createWriteThinLTOBitcodePass(raw_ostream &Str, 585*0b57cec5SDimitry Andric raw_ostream *ThinLinkOS) { 586*0b57cec5SDimitry Andric return new WriteThinLTOBitcode(Str, ThinLinkOS); 587*0b57cec5SDimitry Andric } 588*0b57cec5SDimitry Andric 589*0b57cec5SDimitry Andric PreservedAnalyses 590*0b57cec5SDimitry Andric llvm::ThinLTOBitcodeWriterPass::run(Module &M, ModuleAnalysisManager &AM) { 591*0b57cec5SDimitry Andric FunctionAnalysisManager &FAM = 592*0b57cec5SDimitry Andric AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 593*0b57cec5SDimitry Andric writeThinLTOBitcode(OS, ThinLinkOS, 594*0b57cec5SDimitry Andric [&FAM](Function &F) -> AAResults & { 595*0b57cec5SDimitry Andric return FAM.getResult<AAManager>(F); 596*0b57cec5SDimitry Andric }, 597*0b57cec5SDimitry Andric M, &AM.getResult<ModuleSummaryIndexAnalysis>(M)); 598*0b57cec5SDimitry Andric return PreservedAnalyses::all(); 599*0b57cec5SDimitry Andric } 600