12f09f445SMaksim Panchenko //===- bolt/Rewrite/MachORewriteInstance.cpp - MachO rewriter -------------===//
2a34c753fSRafael Auler //
3a34c753fSRafael Auler // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4a34c753fSRafael Auler // See https://llvm.org/LICENSE.txt for license information.
5a34c753fSRafael Auler // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6a34c753fSRafael Auler //
7a34c753fSRafael Auler //===----------------------------------------------------------------------===//
8a34c753fSRafael Auler 
9a34c753fSRafael Auler #include "bolt/Rewrite/MachORewriteInstance.h"
10a34c753fSRafael Auler #include "bolt/Core/BinaryContext.h"
11a34c753fSRafael Auler #include "bolt/Core/BinaryEmitter.h"
12a34c753fSRafael Auler #include "bolt/Core/BinaryFunction.h"
13a34c753fSRafael Auler #include "bolt/Core/JumpTable.h"
14a34c753fSRafael Auler #include "bolt/Core/MCPlusBuilder.h"
15a34c753fSRafael Auler #include "bolt/Passes/Instrumentation.h"
16a34c753fSRafael Auler #include "bolt/Passes/PatchEntries.h"
17a34c753fSRafael Auler #include "bolt/Profile/DataReader.h"
18a34c753fSRafael Auler #include "bolt/Rewrite/BinaryPassManager.h"
19a34c753fSRafael Auler #include "bolt/Rewrite/ExecutableFileMemoryManager.h"
20a34c753fSRafael Auler #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
21a34c753fSRafael Auler #include "bolt/Utils/Utils.h"
22a34c753fSRafael Auler #include "llvm/MC/MCAsmLayout.h"
23a34c753fSRafael Auler #include "llvm/MC/MCObjectStreamer.h"
24290e4823Sserge-sans-paille #include "llvm/Support/Errc.h"
25a34c753fSRafael Auler #include "llvm/Support/FileSystem.h"
26a34c753fSRafael Auler #include "llvm/Support/ToolOutputFile.h"
2732d2473aSAmir Ayupov #include <memory>
28a34c753fSRafael Auler 
29a34c753fSRafael Auler namespace opts {
30a34c753fSRafael Auler 
31a34c753fSRafael Auler using namespace llvm;
32a34c753fSRafael Auler extern cl::opt<unsigned> AlignText;
33a34c753fSRafael Auler //FIXME! Upstream change
34a34c753fSRafael Auler //extern cl::opt<bool> CheckOverlappingElements;
35a34c753fSRafael Auler extern cl::opt<bool> ForcePatch;
36a34c753fSRafael Auler extern cl::opt<bool> Instrument;
37a34c753fSRafael Auler extern cl::opt<bool> InstrumentCalls;
38a34c753fSRafael Auler extern cl::opt<bolt::JumpTableSupportLevel> JumpTables;
39a34c753fSRafael Auler extern cl::opt<bool> KeepTmp;
40a34c753fSRafael Auler extern cl::opt<bool> NeverPrint;
41a34c753fSRafael Auler extern cl::opt<std::string> OutputFilename;
42a34c753fSRafael Auler extern cl::opt<bool> PrintAfterBranchFixup;
43a34c753fSRafael Auler extern cl::opt<bool> PrintFinalized;
44ccb99dd1SMaksim Panchenko extern cl::opt<bool> PrintNormalized;
45a34c753fSRafael Auler extern cl::opt<bool> PrintReordered;
46a34c753fSRafael Auler extern cl::opt<bool> PrintSections;
47a34c753fSRafael Auler extern cl::opt<bool> PrintDisasm;
48a34c753fSRafael Auler extern cl::opt<bool> PrintCFG;
49a34c753fSRafael Auler extern cl::opt<std::string> RuntimeInstrumentationLib;
50a34c753fSRafael Auler extern cl::opt<unsigned> Verbosity;
51a34c753fSRafael Auler } // namespace opts
52a34c753fSRafael Auler 
53a34c753fSRafael Auler namespace llvm {
54a34c753fSRafael Auler namespace bolt {
55a34c753fSRafael Auler 
56a34c753fSRafael Auler extern MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *,
57a34c753fSRafael Auler                                              const MCInstrInfo *,
58a34c753fSRafael Auler                                              const MCRegisterInfo *);
59a34c753fSRafael Auler extern MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *,
60a34c753fSRafael Auler                                                  const MCInstrInfo *,
61a34c753fSRafael Auler                                                  const MCRegisterInfo *);
62a34c753fSRafael Auler 
63a34c753fSRafael Auler namespace {
64a34c753fSRafael Auler 
createMCPlusBuilder(const Triple::ArchType Arch,const MCInstrAnalysis * Analysis,const MCInstrInfo * Info,const MCRegisterInfo * RegInfo)65a34c753fSRafael Auler MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
66a34c753fSRafael Auler                                    const MCInstrAnalysis *Analysis,
67a34c753fSRafael Auler                                    const MCInstrInfo *Info,
68a34c753fSRafael Auler                                    const MCRegisterInfo *RegInfo) {
69a34c753fSRafael Auler #ifdef X86_AVAILABLE
70a34c753fSRafael Auler   if (Arch == Triple::x86_64)
71a34c753fSRafael Auler     return createX86MCPlusBuilder(Analysis, Info, RegInfo);
72a34c753fSRafael Auler #endif
73a34c753fSRafael Auler 
74a34c753fSRafael Auler #ifdef AARCH64_AVAILABLE
75a34c753fSRafael Auler   if (Arch == Triple::aarch64)
76a34c753fSRafael Auler     return createAArch64MCPlusBuilder(Analysis, Info, RegInfo);
77a34c753fSRafael Auler #endif
78a34c753fSRafael Auler 
79a34c753fSRafael Auler   llvm_unreachable("architecture unsupported by MCPlusBuilder");
80a34c753fSRafael Auler }
81a34c753fSRafael Auler 
82a34c753fSRafael Auler } // anonymous namespace
83a34c753fSRafael Auler 
84a34c753fSRafael Auler #define DEBUG_TYPE "bolt"
85a34c753fSRafael Auler 
8632d2473aSAmir Ayupov Expected<std::unique_ptr<MachORewriteInstance>>
createMachORewriteInstance(object::MachOObjectFile * InputFile,StringRef ToolPath)8732d2473aSAmir Ayupov MachORewriteInstance::createMachORewriteInstance(
8832d2473aSAmir Ayupov     object::MachOObjectFile *InputFile, StringRef ToolPath) {
8932d2473aSAmir Ayupov   Error Err = Error::success();
9032d2473aSAmir Ayupov   auto MachORI =
9132d2473aSAmir Ayupov       std::make_unique<MachORewriteInstance>(InputFile, ToolPath, Err);
9232d2473aSAmir Ayupov   if (Err)
9332d2473aSAmir Ayupov     return std::move(Err);
9463686af1SVladislav Khmelevsky   return std::move(MachORI);
9532d2473aSAmir Ayupov }
9632d2473aSAmir Ayupov 
MachORewriteInstance(object::MachOObjectFile * InputFile,StringRef ToolPath,Error & Err)97a34c753fSRafael Auler MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile,
9832d2473aSAmir Ayupov                                            StringRef ToolPath, Error &Err)
9932d2473aSAmir Ayupov     : InputFile(InputFile), ToolPath(ToolPath) {
10032d2473aSAmir Ayupov   ErrorAsOutParameter EAO(&Err);
10132d2473aSAmir Ayupov   auto BCOrErr = BinaryContext::createBinaryContext(
10232d2473aSAmir Ayupov       InputFile, /* IsPIC */ true, DWARFContext::create(*InputFile));
10332d2473aSAmir Ayupov   if (Error E = BCOrErr.takeError()) {
10432d2473aSAmir Ayupov     Err = std::move(E);
10532d2473aSAmir Ayupov     return;
10632d2473aSAmir Ayupov   }
10732d2473aSAmir Ayupov   BC = std::move(BCOrErr.get());
108a34c753fSRafael Auler   BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(createMCPlusBuilder(
109a34c753fSRafael Auler       BC->TheTriple->getArch(), BC->MIA.get(), BC->MII.get(), BC->MRI.get())));
110a34c753fSRafael Auler   if (opts::Instrument)
111a34c753fSRafael Auler     BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
112a34c753fSRafael Auler }
113a34c753fSRafael Auler 
setProfile(StringRef Filename)114a34c753fSRafael Auler Error MachORewriteInstance::setProfile(StringRef Filename) {
115a34c753fSRafael Auler   if (!sys::fs::exists(Filename))
116a34c753fSRafael Auler     return errorCodeToError(make_error_code(errc::no_such_file_or_directory));
117a34c753fSRafael Auler 
118a34c753fSRafael Auler   if (ProfileReader) {
119a34c753fSRafael Auler     // Already exists
120a34c753fSRafael Auler     return make_error<StringError>(
121a34c753fSRafael Auler         Twine("multiple profiles specified: ") + ProfileReader->getFilename() +
122a34c753fSRafael Auler         " and " + Filename, inconvertibleErrorCode());
123a34c753fSRafael Auler   }
124a34c753fSRafael Auler 
125a34c753fSRafael Auler   ProfileReader = std::make_unique<DataReader>(Filename);
126a34c753fSRafael Auler   return Error::success();
127a34c753fSRafael Auler }
128a34c753fSRafael Auler 
preprocessProfileData()129a34c753fSRafael Auler void MachORewriteInstance::preprocessProfileData() {
130a34c753fSRafael Auler   if (!ProfileReader)
131a34c753fSRafael Auler     return;
132a34c753fSRafael Auler   if (Error E = ProfileReader->preprocessProfile(*BC.get()))
133a34c753fSRafael Auler     report_error("cannot pre-process profile", std::move(E));
134a34c753fSRafael Auler }
135a34c753fSRafael Auler 
processProfileDataPreCFG()136a34c753fSRafael Auler void MachORewriteInstance::processProfileDataPreCFG() {
137a34c753fSRafael Auler   if (!ProfileReader)
138a34c753fSRafael Auler     return;
139a34c753fSRafael Auler   if (Error E = ProfileReader->readProfilePreCFG(*BC.get()))
140a34c753fSRafael Auler     report_error("cannot read profile pre-CFG", std::move(E));
141a34c753fSRafael Auler }
142a34c753fSRafael Auler 
processProfileData()143a34c753fSRafael Auler void MachORewriteInstance::processProfileData() {
144a34c753fSRafael Auler   if (!ProfileReader)
145a34c753fSRafael Auler     return;
146a34c753fSRafael Auler   if (Error E = ProfileReader->readProfile(*BC.get()))
147a34c753fSRafael Auler     report_error("cannot read profile", std::move(E));
148a34c753fSRafael Auler }
149a34c753fSRafael Auler 
readSpecialSections()150a34c753fSRafael Auler void MachORewriteInstance::readSpecialSections() {
151a34c753fSRafael Auler   for (const object::SectionRef &Section : InputFile->sections()) {
152a34c753fSRafael Auler     Expected<StringRef> SectionName = Section.getName();;
153a34c753fSRafael Auler     check_error(SectionName.takeError(), "cannot get section name");
154a34c753fSRafael Auler     // Only register sections with names.
155a34c753fSRafael Auler     if (!SectionName->empty()) {
156a34c753fSRafael Auler       BC->registerSection(Section);
157a34c753fSRafael Auler       LLVM_DEBUG(
158a34c753fSRafael Auler           dbgs() << "BOLT-DEBUG: registering section " << *SectionName
159a34c753fSRafael Auler                  << " @ 0x" << Twine::utohexstr(Section.getAddress()) << ":0x"
160a34c753fSRafael Auler                  << Twine::utohexstr(Section.getAddress() + Section.getSize())
161a34c753fSRafael Auler                  << "\n");
162a34c753fSRafael Auler     }
163a34c753fSRafael Auler   }
164a34c753fSRafael Auler 
165a34c753fSRafael Auler   if (opts::PrintSections) {
166a34c753fSRafael Auler     outs() << "BOLT-INFO: Sections from original binary:\n";
167a34c753fSRafael Auler     BC->printSections(outs());
168a34c753fSRafael Auler   }
169a34c753fSRafael Auler }
170a34c753fSRafael Auler 
171a34c753fSRafael Auler namespace {
172a34c753fSRafael Auler 
173a34c753fSRafael Auler struct DataInCodeRegion {
DataInCodeRegionllvm::bolt::__anonf2314c260211::DataInCodeRegion174a34c753fSRafael Auler   explicit DataInCodeRegion(DiceRef D) {
175a34c753fSRafael Auler     D.getOffset(Offset);
176a34c753fSRafael Auler     D.getLength(Length);
177a34c753fSRafael Auler     D.getKind(Kind);
178a34c753fSRafael Auler   }
179a34c753fSRafael Auler 
180a34c753fSRafael Auler   uint32_t Offset;
181a34c753fSRafael Auler   uint16_t Length;
182a34c753fSRafael Auler   uint16_t Kind;
183a34c753fSRafael Auler };
184a34c753fSRafael Auler 
readDataInCode(const MachOObjectFile & O)185a34c753fSRafael Auler std::vector<DataInCodeRegion> readDataInCode(const MachOObjectFile &O) {
186a34c753fSRafael Auler   const MachO::linkedit_data_command DataInCodeLC =
187a34c753fSRafael Auler       O.getDataInCodeLoadCommand();
188a34c753fSRafael Auler   const uint32_t NumberOfEntries =
189a34c753fSRafael Auler       DataInCodeLC.datasize / sizeof(MachO::data_in_code_entry);
190a34c753fSRafael Auler   std::vector<DataInCodeRegion> DataInCode;
191a34c753fSRafael Auler   DataInCode.reserve(NumberOfEntries);
192a34c753fSRafael Auler   for (auto I = O.begin_dices(), E = O.end_dices(); I != E; ++I)
193a34c753fSRafael Auler     DataInCode.emplace_back(*I);
194*d2c87699SAmir Ayupov   llvm::stable_sort(DataInCode, [](DataInCodeRegion LHS, DataInCodeRegion RHS) {
195a34c753fSRafael Auler     return LHS.Offset < RHS.Offset;
196a34c753fSRafael Auler   });
197a34c753fSRafael Auler   return DataInCode;
198a34c753fSRafael Auler }
199a34c753fSRafael Auler 
readStartAddress(const MachOObjectFile & O)200a34c753fSRafael Auler Optional<uint64_t> readStartAddress(const MachOObjectFile &O) {
201a34c753fSRafael Auler   Optional<uint64_t> StartOffset;
202a34c753fSRafael Auler   Optional<uint64_t> TextVMAddr;
203a34c753fSRafael Auler   for (const object::MachOObjectFile::LoadCommandInfo &LC : O.load_commands()) {
204a34c753fSRafael Auler     switch (LC.C.cmd) {
205a34c753fSRafael Auler     case MachO::LC_MAIN: {
206a34c753fSRafael Auler       MachO::entry_point_command LCMain = O.getEntryPointCommand(LC);
207a34c753fSRafael Auler       StartOffset = LCMain.entryoff;
208a34c753fSRafael Auler       break;
209a34c753fSRafael Auler     }
210a34c753fSRafael Auler     case MachO::LC_SEGMENT: {
211a34c753fSRafael Auler       MachO::segment_command LCSeg = O.getSegmentLoadCommand(LC);
212a34c753fSRafael Auler       StringRef SegmentName(LCSeg.segname,
213a34c753fSRafael Auler                             strnlen(LCSeg.segname, sizeof(LCSeg.segname)));
214a34c753fSRafael Auler       if (SegmentName == "__TEXT")
215a34c753fSRafael Auler         TextVMAddr = LCSeg.vmaddr;
216a34c753fSRafael Auler       break;
217a34c753fSRafael Auler     }
218a34c753fSRafael Auler     case MachO::LC_SEGMENT_64: {
219a34c753fSRafael Auler       MachO::segment_command_64 LCSeg = O.getSegment64LoadCommand(LC);
220a34c753fSRafael Auler       StringRef SegmentName(LCSeg.segname,
221a34c753fSRafael Auler                             strnlen(LCSeg.segname, sizeof(LCSeg.segname)));
222a34c753fSRafael Auler       if (SegmentName == "__TEXT")
223a34c753fSRafael Auler         TextVMAddr = LCSeg.vmaddr;
224a34c753fSRafael Auler       break;
225a34c753fSRafael Auler     }
226a34c753fSRafael Auler     default:
227a34c753fSRafael Auler       continue;
228a34c753fSRafael Auler     }
229a34c753fSRafael Auler   }
230a34c753fSRafael Auler   return (TextVMAddr && StartOffset)
231a34c753fSRafael Auler              ? Optional<uint64_t>(*TextVMAddr + *StartOffset)
232a34c753fSRafael Auler              : llvm::None;
233a34c753fSRafael Auler }
234a34c753fSRafael Auler 
235a34c753fSRafael Auler } // anonymous namespace
236a34c753fSRafael Auler 
discoverFileObjects()237a34c753fSRafael Auler void MachORewriteInstance::discoverFileObjects() {
238a34c753fSRafael Auler   std::vector<SymbolRef> FunctionSymbols;
239a34c753fSRafael Auler   for (const SymbolRef &S : InputFile->symbols()) {
240a34c753fSRafael Auler     SymbolRef::Type Type = cantFail(S.getType(), "cannot get symbol type");
241a34c753fSRafael Auler     if (Type == SymbolRef::ST_Function)
242a34c753fSRafael Auler       FunctionSymbols.push_back(S);
243a34c753fSRafael Auler   }
244a34c753fSRafael Auler   if (FunctionSymbols.empty())
245a34c753fSRafael Auler     return;
246*d2c87699SAmir Ayupov   llvm::stable_sort(
247*d2c87699SAmir Ayupov       FunctionSymbols, [](const SymbolRef &LHS, const SymbolRef &RHS) {
248a34c753fSRafael Auler         return cantFail(LHS.getValue()) < cantFail(RHS.getValue());
249a34c753fSRafael Auler       });
250a34c753fSRafael Auler   for (size_t Index = 0; Index < FunctionSymbols.size(); ++Index) {
251a34c753fSRafael Auler     const uint64_t Address = cantFail(FunctionSymbols[Index].getValue());
252a34c753fSRafael Auler     ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
253a34c753fSRafael Auler     // TODO: It happens for some symbols (e.g. __mh_execute_header).
254a34c753fSRafael Auler     // Add proper logic to handle them correctly.
255a34c753fSRafael Auler     if (!Section) {
256a34c753fSRafael Auler       errs() << "BOLT-WARNING: no section found for address " << Address
257a34c753fSRafael Auler              << "\n";
258a34c753fSRafael Auler       continue;
259a34c753fSRafael Auler     }
260a34c753fSRafael Auler 
261a34c753fSRafael Auler     std::string SymbolName =
262a34c753fSRafael Auler         cantFail(FunctionSymbols[Index].getName(), "cannot get symbol name")
263a34c753fSRafael Auler             .str();
264a34c753fSRafael Auler     // Uniquify names of local symbols.
265a34c753fSRafael Auler     if (!(cantFail(FunctionSymbols[Index].getFlags()) & SymbolRef::SF_Global))
266a34c753fSRafael Auler       SymbolName = NR.uniquify(SymbolName);
267a34c753fSRafael Auler 
268a34c753fSRafael Auler     section_iterator S = cantFail(FunctionSymbols[Index].getSection());
269a34c753fSRafael Auler     uint64_t EndAddress = S->getAddress() + S->getSize();
270a34c753fSRafael Auler 
271a34c753fSRafael Auler     size_t NFIndex = Index + 1;
272a34c753fSRafael Auler     // Skip aliases.
273a34c753fSRafael Auler     while (NFIndex < FunctionSymbols.size() &&
274a34c753fSRafael Auler            cantFail(FunctionSymbols[NFIndex].getValue()) == Address)
275a34c753fSRafael Auler       ++NFIndex;
276a34c753fSRafael Auler     if (NFIndex < FunctionSymbols.size() &&
277a34c753fSRafael Auler         S == cantFail(FunctionSymbols[NFIndex].getSection()))
278a34c753fSRafael Auler       EndAddress = cantFail(FunctionSymbols[NFIndex].getValue());
279a34c753fSRafael Auler 
280a34c753fSRafael Auler     const uint64_t SymbolSize = EndAddress - Address;
281a34c753fSRafael Auler     const auto It = BC->getBinaryFunctions().find(Address);
282a34c753fSRafael Auler     if (It == BC->getBinaryFunctions().end()) {
283a34c753fSRafael Auler       BinaryFunction *Function = BC->createBinaryFunction(
284a34c753fSRafael Auler           std::move(SymbolName), *Section, Address, SymbolSize);
285a34c753fSRafael Auler       if (!opts::Instrument)
286a34c753fSRafael Auler         Function->setOutputAddress(Function->getAddress());
287a34c753fSRafael Auler 
288a34c753fSRafael Auler     } else {
289a34c753fSRafael Auler       It->second.addAlternativeName(std::move(SymbolName));
290a34c753fSRafael Auler     }
291a34c753fSRafael Auler   }
292a34c753fSRafael Auler 
293a34c753fSRafael Auler   const std::vector<DataInCodeRegion> DataInCode = readDataInCode(*InputFile);
294a34c753fSRafael Auler 
295a34c753fSRafael Auler   for (auto &BFI : BC->getBinaryFunctions()) {
296a34c753fSRafael Auler     BinaryFunction &Function = BFI.second;
297a34c753fSRafael Auler     Function.setMaxSize(Function.getSize());
298a34c753fSRafael Auler 
299a34c753fSRafael Auler     ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
300a34c753fSRafael Auler     if (!FunctionData) {
301a34c753fSRafael Auler       errs() << "BOLT-ERROR: corresponding section is non-executable or "
302a34c753fSRafael Auler              << "empty for function " << Function << '\n';
303a34c753fSRafael Auler       continue;
304a34c753fSRafael Auler     }
305a34c753fSRafael Auler 
306a34c753fSRafael Auler     // Treat zero-sized functions as non-simple ones.
307a34c753fSRafael Auler     if (Function.getSize() == 0) {
308a34c753fSRafael Auler       Function.setSimple(false);
309a34c753fSRafael Auler       continue;
310a34c753fSRafael Auler     }
311a34c753fSRafael Auler 
312a34c753fSRafael Auler     // Offset of the function in the file.
313a34c753fSRafael Auler     const auto *FileBegin =
314a34c753fSRafael Auler         reinterpret_cast<const uint8_t *>(InputFile->getData().data());
315a34c753fSRafael Auler     Function.setFileOffset(FunctionData->begin() - FileBegin);
316a34c753fSRafael Auler 
317a34c753fSRafael Auler     // Treat functions which contain data in code as non-simple ones.
318a34c753fSRafael Auler     const auto It = std::lower_bound(
319a34c753fSRafael Auler         DataInCode.cbegin(), DataInCode.cend(), Function.getFileOffset(),
320a34c753fSRafael Auler         [](DataInCodeRegion D, uint64_t Offset) { return D.Offset < Offset; });
321a34c753fSRafael Auler     if (It != DataInCode.cend() &&
322a34c753fSRafael Auler         It->Offset + It->Length <=
323a34c753fSRafael Auler             Function.getFileOffset() + Function.getMaxSize())
324a34c753fSRafael Auler       Function.setSimple(false);
325a34c753fSRafael Auler   }
326a34c753fSRafael Auler 
327a34c753fSRafael Auler   BC->StartFunctionAddress = readStartAddress(*InputFile);
328a34c753fSRafael Auler }
329a34c753fSRafael Auler 
disassembleFunctions()330a34c753fSRafael Auler void MachORewriteInstance::disassembleFunctions() {
331a34c753fSRafael Auler   for (auto &BFI : BC->getBinaryFunctions()) {
332a34c753fSRafael Auler     BinaryFunction &Function = BFI.second;
333a34c753fSRafael Auler     if (!Function.isSimple())
334a34c753fSRafael Auler       continue;
335a34c753fSRafael Auler     Function.disassemble();
336a34c753fSRafael Auler     if (opts::PrintDisasm)
337a34c753fSRafael Auler       Function.print(outs(), "after disassembly", true);
338a34c753fSRafael Auler   }
339a34c753fSRafael Auler }
340a34c753fSRafael Auler 
buildFunctionsCFG()341a34c753fSRafael Auler void MachORewriteInstance::buildFunctionsCFG() {
342a34c753fSRafael Auler   for (auto &BFI : BC->getBinaryFunctions()) {
343a34c753fSRafael Auler     BinaryFunction &Function = BFI.second;
344a34c753fSRafael Auler     if (!Function.isSimple())
345a34c753fSRafael Auler       continue;
346a34c753fSRafael Auler     if (!Function.buildCFG(/*AllocId*/ 0)) {
347a34c753fSRafael Auler       errs() << "BOLT-WARNING: failed to build CFG for the function "
348a34c753fSRafael Auler              << Function << "\n";
349a34c753fSRafael Auler     }
350a34c753fSRafael Auler   }
351a34c753fSRafael Auler }
352a34c753fSRafael Auler 
postProcessFunctions()353a34c753fSRafael Auler void MachORewriteInstance::postProcessFunctions() {
354a34c753fSRafael Auler   for (auto &BFI : BC->getBinaryFunctions()) {
355a34c753fSRafael Auler     BinaryFunction &Function = BFI.second;
356a34c753fSRafael Auler     if (Function.empty())
357a34c753fSRafael Auler       continue;
358a34c753fSRafael Auler     Function.postProcessCFG();
359a34c753fSRafael Auler     if (opts::PrintCFG)
360a34c753fSRafael Auler       Function.print(outs(), "after building cfg", true);
361a34c753fSRafael Auler   }
362a34c753fSRafael Auler }
363a34c753fSRafael Auler 
runOptimizationPasses()364a34c753fSRafael Auler void MachORewriteInstance::runOptimizationPasses() {
365a34c753fSRafael Auler   BinaryFunctionPassManager Manager(*BC);
366a34c753fSRafael Auler   if (opts::Instrument) {
367a34c753fSRafael Auler     Manager.registerPass(std::make_unique<PatchEntries>());
368a34c753fSRafael Auler     Manager.registerPass(std::make_unique<Instrumentation>(opts::NeverPrint));
369a34c753fSRafael Auler   }
370ccb99dd1SMaksim Panchenko 
371ccb99dd1SMaksim Panchenko   Manager.registerPass(std::make_unique<ShortenInstructions>(opts::NeverPrint));
372ccb99dd1SMaksim Panchenko 
373ccb99dd1SMaksim Panchenko   Manager.registerPass(std::make_unique<RemoveNops>(opts::NeverPrint));
374ccb99dd1SMaksim Panchenko 
375ccb99dd1SMaksim Panchenko   Manager.registerPass(std::make_unique<NormalizeCFG>(opts::PrintNormalized));
376ccb99dd1SMaksim Panchenko 
377a34c753fSRafael Auler   Manager.registerPass(
378a34c753fSRafael Auler       std::make_unique<ReorderBasicBlocks>(opts::PrintReordered));
379a34c753fSRafael Auler   Manager.registerPass(
380a34c753fSRafael Auler       std::make_unique<FixupBranches>(opts::PrintAfterBranchFixup));
381a34c753fSRafael Auler   // This pass should always run last.*
382a34c753fSRafael Auler   Manager.registerPass(
383a34c753fSRafael Auler       std::make_unique<FinalizeFunctions>(opts::PrintFinalized));
384a34c753fSRafael Auler 
385a34c753fSRafael Auler   Manager.runPasses();
386a34c753fSRafael Auler }
387a34c753fSRafael Auler 
mapInstrumentationSection(StringRef SectionName)388a34c753fSRafael Auler void MachORewriteInstance::mapInstrumentationSection(StringRef SectionName) {
389a34c753fSRafael Auler   if (!opts::Instrument)
390a34c753fSRafael Auler     return;
391a34c753fSRafael Auler   ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
392a34c753fSRafael Auler   if (!Section) {
393a34c753fSRafael Auler     llvm::errs() << "Cannot find " + SectionName + " section\n";
394a34c753fSRafael Auler     exit(1);
395a34c753fSRafael Auler   }
396a34c753fSRafael Auler   if (!Section->hasValidSectionID())
397a34c753fSRafael Auler     return;
398a34c753fSRafael Auler   RTDyld->reassignSectionAddress(Section->getSectionID(),
399a34c753fSRafael Auler                                  Section->getAddress());
400a34c753fSRafael Auler }
401a34c753fSRafael Auler 
mapCodeSections()402a34c753fSRafael Auler void MachORewriteInstance::mapCodeSections() {
403a34c753fSRafael Auler   for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
404a34c753fSRafael Auler     if (!Function->isEmitted())
405a34c753fSRafael Auler       continue;
406a34c753fSRafael Auler     if (Function->getOutputAddress() == 0)
407a34c753fSRafael Auler       continue;
408a34c753fSRafael Auler     ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();
409a34c753fSRafael Auler     if (!FuncSection)
410a34c753fSRafael Auler       report_error(
411a34c753fSRafael Auler           (Twine("Cannot find section for function ") + Function->getOneName())
412a34c753fSRafael Auler               .str(),
413a34c753fSRafael Auler           FuncSection.getError());
414a34c753fSRafael Auler 
415a34c753fSRafael Auler     FuncSection->setOutputAddress(Function->getOutputAddress());
416a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
417a34c753fSRafael Auler                  << Twine::utohexstr(FuncSection->getAllocAddress()) << " to 0x"
418a34c753fSRafael Auler                  << Twine::utohexstr(Function->getOutputAddress()) << '\n');
419a34c753fSRafael Auler     RTDyld->reassignSectionAddress(FuncSection->getSectionID(),
420a34c753fSRafael Auler                                    Function->getOutputAddress());
421a34c753fSRafael Auler     Function->setImageAddress(FuncSection->getAllocAddress());
422a34c753fSRafael Auler     Function->setImageSize(FuncSection->getOutputSize());
423a34c753fSRafael Auler   }
424a34c753fSRafael Auler 
425a34c753fSRafael Auler   if (opts::Instrument) {
426a34c753fSRafael Auler     ErrorOr<BinarySection &> BOLT = BC->getUniqueSectionByName("__bolt");
427a34c753fSRafael Auler     if (!BOLT) {
428a34c753fSRafael Auler       llvm::errs() << "Cannot find __bolt section\n";
429a34c753fSRafael Auler       exit(1);
430a34c753fSRafael Auler     }
431a34c753fSRafael Auler     uint64_t Addr = BOLT->getAddress();
432a34c753fSRafael Auler     for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
433a34c753fSRafael Auler       if (!Function->isEmitted())
434a34c753fSRafael Auler         continue;
435a34c753fSRafael Auler       if (Function->getOutputAddress() != 0)
436a34c753fSRafael Auler         continue;
437a34c753fSRafael Auler       ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();
438a34c753fSRafael Auler       assert(FuncSection && "cannot find section for function");
439a34c753fSRafael Auler       Addr = llvm::alignTo(Addr, 4);
440a34c753fSRafael Auler       FuncSection->setOutputAddress(Addr);
441a34c753fSRafael Auler       RTDyld->reassignSectionAddress(FuncSection->getSectionID(), Addr);
442a34c753fSRafael Auler       Function->setFileOffset(Addr - BOLT->getAddress() +
443a34c753fSRafael Auler                               BOLT->getInputFileOffset());
444a34c753fSRafael Auler       Function->setImageAddress(FuncSection->getAllocAddress());
445a34c753fSRafael Auler       Function->setImageSize(FuncSection->getOutputSize());
446a34c753fSRafael Auler       BC->registerNameAtAddress(Function->getOneName(), Addr, 0, 0);
447a34c753fSRafael Auler       Addr += FuncSection->getOutputSize();
448a34c753fSRafael Auler     }
449a34c753fSRafael Auler   }
450a34c753fSRafael Auler }
451a34c753fSRafael Auler 
452a34c753fSRafael Auler namespace {
453a34c753fSRafael Auler 
454a34c753fSRafael Auler class BOLTSymbolResolver : public LegacyJITSymbolResolver {
455a34c753fSRafael Auler   BinaryContext &BC;
456a34c753fSRafael Auler public:
BOLTSymbolResolver(BinaryContext & BC)457a34c753fSRafael Auler   BOLTSymbolResolver(BinaryContext &BC) : BC(BC) {}
458a34c753fSRafael Auler 
findSymbolInLogicalDylib(const std::string & Name)459a34c753fSRafael Auler   JITSymbol findSymbolInLogicalDylib(const std::string &Name) override {
460a34c753fSRafael Auler     return JITSymbol(nullptr);
461a34c753fSRafael Auler   }
462a34c753fSRafael Auler 
findSymbol(const std::string & Name)463a34c753fSRafael Auler   JITSymbol findSymbol(const std::string &Name) override {
464a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << "BOLT: looking for " << Name << "\n");
465a34c753fSRafael Auler     if (BinaryData *I = BC.getBinaryDataByName(Name)) {
466a34c753fSRafael Auler       const uint64_t Address = I->isMoved() && !I->isJumpTable()
467a34c753fSRafael Auler                                    ? I->getOutputAddress()
468a34c753fSRafael Auler                                    : I->getAddress();
469a34c753fSRafael Auler       LLVM_DEBUG(dbgs() << "Resolved to address 0x" << Twine::utohexstr(Address)
470a34c753fSRafael Auler                         << "\n");
471a34c753fSRafael Auler       return JITSymbol(Address, JITSymbolFlags());
472a34c753fSRafael Auler     }
473a34c753fSRafael Auler     LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n");
474a34c753fSRafael Auler     return JITSymbol(nullptr);
475a34c753fSRafael Auler   }
476a34c753fSRafael Auler };
477a34c753fSRafael Auler 
478a34c753fSRafael Auler } // end anonymous namespace
479a34c753fSRafael Auler 
emitAndLink()480a34c753fSRafael Auler void MachORewriteInstance::emitAndLink() {
481a34c753fSRafael Auler   std::error_code EC;
482a34c753fSRafael Auler   std::unique_ptr<::llvm::ToolOutputFile> TempOut =
483a34c753fSRafael Auler       std::make_unique<::llvm::ToolOutputFile>(
484a34c753fSRafael Auler           opts::OutputFilename + ".bolt.o", EC, sys::fs::OF_None);
485a34c753fSRafael Auler   check_error(EC, "cannot create output object file");
486a34c753fSRafael Auler 
487a34c753fSRafael Auler   if (opts::KeepTmp)
488a34c753fSRafael Auler     TempOut->keep();
489a34c753fSRafael Auler 
490a34c753fSRafael Auler   std::unique_ptr<buffer_ostream> BOS =
491a34c753fSRafael Auler       std::make_unique<buffer_ostream>(TempOut->os());
492a34c753fSRafael Auler   raw_pwrite_stream *OS = BOS.get();
493a34c753fSRafael Auler   auto Streamer = BC->createStreamer(*OS);
494a34c753fSRafael Auler 
495a34c753fSRafael Auler   emitBinaryContext(*Streamer, *BC, getOrgSecPrefix());
49615d82c62SFangrui Song   Streamer->finish();
497a34c753fSRafael Auler 
498a34c753fSRafael Auler   std::unique_ptr<MemoryBuffer> ObjectMemBuffer =
499a34c753fSRafael Auler       MemoryBuffer::getMemBuffer(BOS->str(), "in-memory object file", false);
500a34c753fSRafael Auler   std::unique_ptr<object::ObjectFile> Obj = cantFail(
501a34c753fSRafael Auler       object::ObjectFile::createObjectFile(ObjectMemBuffer->getMemBufferRef()),
502a34c753fSRafael Auler       "error creating in-memory object");
503a34c753fSRafael Auler   assert(Obj && "createObjectFile cannot return nullptr");
504a34c753fSRafael Auler 
505a34c753fSRafael Auler   BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC);
506a34c753fSRafael Auler 
507a34c753fSRafael Auler   MCAsmLayout FinalLayout(
508a34c753fSRafael Auler       static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler());
509a34c753fSRafael Auler 
510a34c753fSRafael Auler   BC->EFMM.reset(new ExecutableFileMemoryManager(*BC, /*AllowStubs*/ false));
511a34c753fSRafael Auler 
512a34c753fSRafael Auler   RTDyld.reset(new decltype(RTDyld)::element_type(*BC->EFMM, Resolver));
513a34c753fSRafael Auler   RTDyld->setProcessAllSections(true);
514a34c753fSRafael Auler   RTDyld->loadObject(*Obj);
515a34c753fSRafael Auler   if (RTDyld->hasError()) {
516a34c753fSRafael Auler     outs() << "BOLT-ERROR: RTDyld failed.\n";
517a34c753fSRafael Auler     exit(1);
518a34c753fSRafael Auler   }
519a34c753fSRafael Auler 
520a34c753fSRafael Auler   // Assign addresses to all sections. If key corresponds to the object
521a34c753fSRafael Auler   // created by ourselves, call our regular mapping function. If we are
522a34c753fSRafael Auler   // loading additional objects as part of runtime libraries for
523a34c753fSRafael Auler   // instrumentation, treat them as extra sections.
524a34c753fSRafael Auler   mapCodeSections();
525a34c753fSRafael Auler   mapInstrumentationSection("__counters");
526a34c753fSRafael Auler   mapInstrumentationSection("__tables");
527a34c753fSRafael Auler 
528a34c753fSRafael Auler           // TODO: Refactor addRuntimeLibSections to work properly on Mach-O
529a34c753fSRafael Auler           // and use it here.
530a34c753fSRafael Auler   //FIXME! Put this in RtLibrary->link
531a34c753fSRafael Auler //          mapInstrumentationSection("I__setup");
532a34c753fSRafael Auler //          mapInstrumentationSection("I__fini");
533a34c753fSRafael Auler //          mapInstrumentationSection("I__data");
534a34c753fSRafael Auler //          mapInstrumentationSection("I__text");
535a34c753fSRafael Auler //          mapInstrumentationSection("I__cstring");
536a34c753fSRafael Auler //          mapInstrumentationSection("I__literal16");
537a34c753fSRafael Auler 
538a34c753fSRafael Auler //  if (auto *RtLibrary = BC->getRuntimeLibrary()) {
539a34c753fSRafael Auler //    RtLibrary->link(*BC, ToolPath, *ES, *OLT);
540a34c753fSRafael Auler //  }
541a34c753fSRafael Auler }
542a34c753fSRafael Auler 
writeInstrumentationSection(StringRef SectionName,raw_pwrite_stream & OS)543a34c753fSRafael Auler void MachORewriteInstance::writeInstrumentationSection(StringRef SectionName,
544a34c753fSRafael Auler                                                        raw_pwrite_stream &OS) {
545a34c753fSRafael Auler   if (!opts::Instrument)
546a34c753fSRafael Auler     return;
547a34c753fSRafael Auler   ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
548a34c753fSRafael Auler   if (!Section) {
549a34c753fSRafael Auler     llvm::errs() << "Cannot find " + SectionName + " section\n";
550a34c753fSRafael Auler     exit(1);
551a34c753fSRafael Auler   }
552a34c753fSRafael Auler   if (!Section->hasValidSectionID())
553a34c753fSRafael Auler     return;
554a34c753fSRafael Auler   assert(Section->getInputFileOffset() &&
555a34c753fSRafael Auler          "Section input offset cannot be zero");
556a34c753fSRafael Auler   assert(Section->getAllocAddress() && "Section alloc address cannot be zero");
557a34c753fSRafael Auler   assert(Section->getOutputSize() && "Section output size cannot be zero");
558a34c753fSRafael Auler   OS.pwrite(reinterpret_cast<char *>(Section->getAllocAddress()),
559a34c753fSRafael Auler             Section->getOutputSize(), Section->getInputFileOffset());
560a34c753fSRafael Auler }
561a34c753fSRafael Auler 
rewriteFile()562a34c753fSRafael Auler void MachORewriteInstance::rewriteFile() {
563a34c753fSRafael Auler   std::error_code EC;
564a34c753fSRafael Auler   Out = std::make_unique<ToolOutputFile>(opts::OutputFilename, EC,
565a34c753fSRafael Auler                                          sys::fs::OF_None);
566a34c753fSRafael Auler   check_error(EC, "cannot create output executable file");
567a34c753fSRafael Auler   raw_fd_ostream &OS = Out->os();
568a34c753fSRafael Auler   OS << InputFile->getData();
569a34c753fSRafael Auler 
570a34c753fSRafael Auler   for (auto &BFI : BC->getBinaryFunctions()) {
571a34c753fSRafael Auler     BinaryFunction &Function = BFI.second;
572a34c753fSRafael Auler     if (!Function.isSimple())
573a34c753fSRafael Auler       continue;
574a34c753fSRafael Auler     assert(Function.isEmitted() && "Simple function has not been emitted");
575a34c753fSRafael Auler     if (!opts::Instrument && (Function.getImageSize() > Function.getMaxSize()))
576a34c753fSRafael Auler       continue;
577a34c753fSRafael Auler     if (opts::Verbosity >= 2)
578a34c753fSRafael Auler       outs() << "BOLT: rewriting function \"" << Function << "\"\n";
579a34c753fSRafael Auler     OS.pwrite(reinterpret_cast<char *>(Function.getImageAddress()),
580a34c753fSRafael Auler               Function.getImageSize(), Function.getFileOffset());
581a34c753fSRafael Auler   }
582a34c753fSRafael Auler 
583a34c753fSRafael Auler   for (const BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {
584a34c753fSRafael Auler     OS.pwrite(reinterpret_cast<char *>(Function->getImageAddress()),
585a34c753fSRafael Auler               Function->getImageSize(), Function->getFileOffset());
586a34c753fSRafael Auler   }
587a34c753fSRafael Auler 
588a34c753fSRafael Auler   writeInstrumentationSection("__counters", OS);
589a34c753fSRafael Auler   writeInstrumentationSection("__tables", OS);
590a34c753fSRafael Auler 
591a34c753fSRafael Auler   // TODO: Refactor addRuntimeLibSections to work properly on Mach-O and
592a34c753fSRafael Auler   // use it here.
593a34c753fSRafael Auler   writeInstrumentationSection("I__setup", OS);
594a34c753fSRafael Auler   writeInstrumentationSection("I__fini", OS);
595a34c753fSRafael Auler   writeInstrumentationSection("I__data", OS);
596a34c753fSRafael Auler   writeInstrumentationSection("I__text", OS);
597a34c753fSRafael Auler   writeInstrumentationSection("I__cstring", OS);
598a34c753fSRafael Auler   writeInstrumentationSection("I__literal16", OS);
599a34c753fSRafael Auler 
600a34c753fSRafael Auler   Out->keep();
601a34c753fSRafael Auler   EC = sys::fs::setPermissions(opts::OutputFilename,
602a34c753fSRafael Auler                                sys::fs::perms::all_all);
603a34c753fSRafael Auler   check_error(EC, "cannot set permissions of output file");
604a34c753fSRafael Auler }
605a34c753fSRafael Auler 
adjustCommandLineOptions()606a34c753fSRafael Auler void MachORewriteInstance::adjustCommandLineOptions() {
607a34c753fSRafael Auler //FIXME! Upstream change
608a34c753fSRafael Auler //  opts::CheckOverlappingElements = false;
609a34c753fSRafael Auler   if (!opts::AlignText.getNumOccurrences())
610a34c753fSRafael Auler     opts::AlignText = BC->PageAlign;
611a34c753fSRafael Auler   if (opts::Instrument.getNumOccurrences())
612a34c753fSRafael Auler     opts::ForcePatch = true;
613a34c753fSRafael Auler   opts::JumpTables = JTS_MOVE;
614a34c753fSRafael Auler   opts::InstrumentCalls = false;
615a34c753fSRafael Auler   opts::RuntimeInstrumentationLib = "libbolt_rt_instr_osx.a";
616a34c753fSRafael Auler }
617a34c753fSRafael Auler 
run()618a34c753fSRafael Auler void MachORewriteInstance::run() {
619a34c753fSRafael Auler   adjustCommandLineOptions();
620a34c753fSRafael Auler 
621a34c753fSRafael Auler   readSpecialSections();
622a34c753fSRafael Auler 
623a34c753fSRafael Auler   discoverFileObjects();
624a34c753fSRafael Auler 
625a34c753fSRafael Auler   preprocessProfileData();
626a34c753fSRafael Auler 
627a34c753fSRafael Auler   disassembleFunctions();
628a34c753fSRafael Auler 
629a34c753fSRafael Auler   processProfileDataPreCFG();
630a34c753fSRafael Auler 
631a34c753fSRafael Auler   buildFunctionsCFG();
632a34c753fSRafael Auler 
633a34c753fSRafael Auler   processProfileData();
634a34c753fSRafael Auler 
635a34c753fSRafael Auler   postProcessFunctions();
636a34c753fSRafael Auler 
637a34c753fSRafael Auler   runOptimizationPasses();
638a34c753fSRafael Auler 
639a34c753fSRafael Auler   emitAndLink();
640a34c753fSRafael Auler 
641a34c753fSRafael Auler   rewriteFile();
642a34c753fSRafael Auler }
643a34c753fSRafael Auler 
~MachORewriteInstance()644a34c753fSRafael Auler MachORewriteInstance::~MachORewriteInstance() {}
645a34c753fSRafael Auler 
646a34c753fSRafael Auler } // namespace bolt
647a34c753fSRafael Auler } // namespace llvm
648