12f09f445SMaksim Panchenko //===- bolt/Rewrite/MachORewriteInstance.cpp - MachO rewriter -------------===//
2a34c753fSRafael Auler //
3a34c753fSRafael Auler // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4a34c753fSRafael Auler // See https://llvm.org/LICENSE.txt for license information.
5a34c753fSRafael Auler // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6a34c753fSRafael Auler //
7a34c753fSRafael Auler //===----------------------------------------------------------------------===//
8a34c753fSRafael Auler
9a34c753fSRafael Auler #include "bolt/Rewrite/MachORewriteInstance.h"
10a34c753fSRafael Auler #include "bolt/Core/BinaryContext.h"
11a34c753fSRafael Auler #include "bolt/Core/BinaryEmitter.h"
12a34c753fSRafael Auler #include "bolt/Core/BinaryFunction.h"
13a34c753fSRafael Auler #include "bolt/Core/JumpTable.h"
14a34c753fSRafael Auler #include "bolt/Core/MCPlusBuilder.h"
15a34c753fSRafael Auler #include "bolt/Passes/Instrumentation.h"
16a34c753fSRafael Auler #include "bolt/Passes/PatchEntries.h"
17a34c753fSRafael Auler #include "bolt/Profile/DataReader.h"
18a34c753fSRafael Auler #include "bolt/Rewrite/BinaryPassManager.h"
19a34c753fSRafael Auler #include "bolt/Rewrite/ExecutableFileMemoryManager.h"
20a34c753fSRafael Auler #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
21a34c753fSRafael Auler #include "bolt/Utils/Utils.h"
22a34c753fSRafael Auler #include "llvm/MC/MCAsmLayout.h"
23a34c753fSRafael Auler #include "llvm/MC/MCObjectStreamer.h"
24290e4823Sserge-sans-paille #include "llvm/Support/Errc.h"
25a34c753fSRafael Auler #include "llvm/Support/FileSystem.h"
26a34c753fSRafael Auler #include "llvm/Support/ToolOutputFile.h"
2732d2473aSAmir Ayupov #include <memory>
28a34c753fSRafael Auler
29a34c753fSRafael Auler namespace opts {
30a34c753fSRafael Auler
31a34c753fSRafael Auler using namespace llvm;
32a34c753fSRafael Auler extern cl::opt<unsigned> AlignText;
33a34c753fSRafael Auler //FIXME! Upstream change
34a34c753fSRafael Auler //extern cl::opt<bool> CheckOverlappingElements;
35a34c753fSRafael Auler extern cl::opt<bool> ForcePatch;
36a34c753fSRafael Auler extern cl::opt<bool> Instrument;
37a34c753fSRafael Auler extern cl::opt<bool> InstrumentCalls;
38a34c753fSRafael Auler extern cl::opt<bolt::JumpTableSupportLevel> JumpTables;
39a34c753fSRafael Auler extern cl::opt<bool> KeepTmp;
40a34c753fSRafael Auler extern cl::opt<bool> NeverPrint;
41a34c753fSRafael Auler extern cl::opt<std::string> OutputFilename;
42a34c753fSRafael Auler extern cl::opt<bool> PrintAfterBranchFixup;
43a34c753fSRafael Auler extern cl::opt<bool> PrintFinalized;
44ccb99dd1SMaksim Panchenko extern cl::opt<bool> PrintNormalized;
45a34c753fSRafael Auler extern cl::opt<bool> PrintReordered;
46a34c753fSRafael Auler extern cl::opt<bool> PrintSections;
47a34c753fSRafael Auler extern cl::opt<bool> PrintDisasm;
48a34c753fSRafael Auler extern cl::opt<bool> PrintCFG;
49a34c753fSRafael Auler extern cl::opt<std::string> RuntimeInstrumentationLib;
50a34c753fSRafael Auler extern cl::opt<unsigned> Verbosity;
51a34c753fSRafael Auler } // namespace opts
52a34c753fSRafael Auler
53a34c753fSRafael Auler namespace llvm {
54a34c753fSRafael Auler namespace bolt {
55a34c753fSRafael Auler
56a34c753fSRafael Auler extern MCPlusBuilder *createX86MCPlusBuilder(const MCInstrAnalysis *,
57a34c753fSRafael Auler const MCInstrInfo *,
58a34c753fSRafael Auler const MCRegisterInfo *);
59a34c753fSRafael Auler extern MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *,
60a34c753fSRafael Auler const MCInstrInfo *,
61a34c753fSRafael Auler const MCRegisterInfo *);
62a34c753fSRafael Auler
63a34c753fSRafael Auler namespace {
64a34c753fSRafael Auler
createMCPlusBuilder(const Triple::ArchType Arch,const MCInstrAnalysis * Analysis,const MCInstrInfo * Info,const MCRegisterInfo * RegInfo)65a34c753fSRafael Auler MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,
66a34c753fSRafael Auler const MCInstrAnalysis *Analysis,
67a34c753fSRafael Auler const MCInstrInfo *Info,
68a34c753fSRafael Auler const MCRegisterInfo *RegInfo) {
69a34c753fSRafael Auler #ifdef X86_AVAILABLE
70a34c753fSRafael Auler if (Arch == Triple::x86_64)
71a34c753fSRafael Auler return createX86MCPlusBuilder(Analysis, Info, RegInfo);
72a34c753fSRafael Auler #endif
73a34c753fSRafael Auler
74a34c753fSRafael Auler #ifdef AARCH64_AVAILABLE
75a34c753fSRafael Auler if (Arch == Triple::aarch64)
76a34c753fSRafael Auler return createAArch64MCPlusBuilder(Analysis, Info, RegInfo);
77a34c753fSRafael Auler #endif
78a34c753fSRafael Auler
79a34c753fSRafael Auler llvm_unreachable("architecture unsupported by MCPlusBuilder");
80a34c753fSRafael Auler }
81a34c753fSRafael Auler
82a34c753fSRafael Auler } // anonymous namespace
83a34c753fSRafael Auler
84a34c753fSRafael Auler #define DEBUG_TYPE "bolt"
85a34c753fSRafael Auler
8632d2473aSAmir Ayupov Expected<std::unique_ptr<MachORewriteInstance>>
createMachORewriteInstance(object::MachOObjectFile * InputFile,StringRef ToolPath)8732d2473aSAmir Ayupov MachORewriteInstance::createMachORewriteInstance(
8832d2473aSAmir Ayupov object::MachOObjectFile *InputFile, StringRef ToolPath) {
8932d2473aSAmir Ayupov Error Err = Error::success();
9032d2473aSAmir Ayupov auto MachORI =
9132d2473aSAmir Ayupov std::make_unique<MachORewriteInstance>(InputFile, ToolPath, Err);
9232d2473aSAmir Ayupov if (Err)
9332d2473aSAmir Ayupov return std::move(Err);
9463686af1SVladislav Khmelevsky return std::move(MachORI);
9532d2473aSAmir Ayupov }
9632d2473aSAmir Ayupov
MachORewriteInstance(object::MachOObjectFile * InputFile,StringRef ToolPath,Error & Err)97a34c753fSRafael Auler MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile,
9832d2473aSAmir Ayupov StringRef ToolPath, Error &Err)
9932d2473aSAmir Ayupov : InputFile(InputFile), ToolPath(ToolPath) {
10032d2473aSAmir Ayupov ErrorAsOutParameter EAO(&Err);
10132d2473aSAmir Ayupov auto BCOrErr = BinaryContext::createBinaryContext(
10232d2473aSAmir Ayupov InputFile, /* IsPIC */ true, DWARFContext::create(*InputFile));
10332d2473aSAmir Ayupov if (Error E = BCOrErr.takeError()) {
10432d2473aSAmir Ayupov Err = std::move(E);
10532d2473aSAmir Ayupov return;
10632d2473aSAmir Ayupov }
10732d2473aSAmir Ayupov BC = std::move(BCOrErr.get());
108a34c753fSRafael Auler BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(createMCPlusBuilder(
109a34c753fSRafael Auler BC->TheTriple->getArch(), BC->MIA.get(), BC->MII.get(), BC->MRI.get())));
110a34c753fSRafael Auler if (opts::Instrument)
111a34c753fSRafael Auler BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());
112a34c753fSRafael Auler }
113a34c753fSRafael Auler
setProfile(StringRef Filename)114a34c753fSRafael Auler Error MachORewriteInstance::setProfile(StringRef Filename) {
115a34c753fSRafael Auler if (!sys::fs::exists(Filename))
116a34c753fSRafael Auler return errorCodeToError(make_error_code(errc::no_such_file_or_directory));
117a34c753fSRafael Auler
118a34c753fSRafael Auler if (ProfileReader) {
119a34c753fSRafael Auler // Already exists
120a34c753fSRafael Auler return make_error<StringError>(
121a34c753fSRafael Auler Twine("multiple profiles specified: ") + ProfileReader->getFilename() +
122a34c753fSRafael Auler " and " + Filename, inconvertibleErrorCode());
123a34c753fSRafael Auler }
124a34c753fSRafael Auler
125a34c753fSRafael Auler ProfileReader = std::make_unique<DataReader>(Filename);
126a34c753fSRafael Auler return Error::success();
127a34c753fSRafael Auler }
128a34c753fSRafael Auler
preprocessProfileData()129a34c753fSRafael Auler void MachORewriteInstance::preprocessProfileData() {
130a34c753fSRafael Auler if (!ProfileReader)
131a34c753fSRafael Auler return;
132a34c753fSRafael Auler if (Error E = ProfileReader->preprocessProfile(*BC.get()))
133a34c753fSRafael Auler report_error("cannot pre-process profile", std::move(E));
134a34c753fSRafael Auler }
135a34c753fSRafael Auler
processProfileDataPreCFG()136a34c753fSRafael Auler void MachORewriteInstance::processProfileDataPreCFG() {
137a34c753fSRafael Auler if (!ProfileReader)
138a34c753fSRafael Auler return;
139a34c753fSRafael Auler if (Error E = ProfileReader->readProfilePreCFG(*BC.get()))
140a34c753fSRafael Auler report_error("cannot read profile pre-CFG", std::move(E));
141a34c753fSRafael Auler }
142a34c753fSRafael Auler
processProfileData()143a34c753fSRafael Auler void MachORewriteInstance::processProfileData() {
144a34c753fSRafael Auler if (!ProfileReader)
145a34c753fSRafael Auler return;
146a34c753fSRafael Auler if (Error E = ProfileReader->readProfile(*BC.get()))
147a34c753fSRafael Auler report_error("cannot read profile", std::move(E));
148a34c753fSRafael Auler }
149a34c753fSRafael Auler
readSpecialSections()150a34c753fSRafael Auler void MachORewriteInstance::readSpecialSections() {
151a34c753fSRafael Auler for (const object::SectionRef &Section : InputFile->sections()) {
152a34c753fSRafael Auler Expected<StringRef> SectionName = Section.getName();;
153a34c753fSRafael Auler check_error(SectionName.takeError(), "cannot get section name");
154a34c753fSRafael Auler // Only register sections with names.
155a34c753fSRafael Auler if (!SectionName->empty()) {
156a34c753fSRafael Auler BC->registerSection(Section);
157a34c753fSRafael Auler LLVM_DEBUG(
158a34c753fSRafael Auler dbgs() << "BOLT-DEBUG: registering section " << *SectionName
159a34c753fSRafael Auler << " @ 0x" << Twine::utohexstr(Section.getAddress()) << ":0x"
160a34c753fSRafael Auler << Twine::utohexstr(Section.getAddress() + Section.getSize())
161a34c753fSRafael Auler << "\n");
162a34c753fSRafael Auler }
163a34c753fSRafael Auler }
164a34c753fSRafael Auler
165a34c753fSRafael Auler if (opts::PrintSections) {
166a34c753fSRafael Auler outs() << "BOLT-INFO: Sections from original binary:\n";
167a34c753fSRafael Auler BC->printSections(outs());
168a34c753fSRafael Auler }
169a34c753fSRafael Auler }
170a34c753fSRafael Auler
171a34c753fSRafael Auler namespace {
172a34c753fSRafael Auler
173a34c753fSRafael Auler struct DataInCodeRegion {
DataInCodeRegionllvm::bolt::__anonf2314c260211::DataInCodeRegion174a34c753fSRafael Auler explicit DataInCodeRegion(DiceRef D) {
175a34c753fSRafael Auler D.getOffset(Offset);
176a34c753fSRafael Auler D.getLength(Length);
177a34c753fSRafael Auler D.getKind(Kind);
178a34c753fSRafael Auler }
179a34c753fSRafael Auler
180a34c753fSRafael Auler uint32_t Offset;
181a34c753fSRafael Auler uint16_t Length;
182a34c753fSRafael Auler uint16_t Kind;
183a34c753fSRafael Auler };
184a34c753fSRafael Auler
readDataInCode(const MachOObjectFile & O)185a34c753fSRafael Auler std::vector<DataInCodeRegion> readDataInCode(const MachOObjectFile &O) {
186a34c753fSRafael Auler const MachO::linkedit_data_command DataInCodeLC =
187a34c753fSRafael Auler O.getDataInCodeLoadCommand();
188a34c753fSRafael Auler const uint32_t NumberOfEntries =
189a34c753fSRafael Auler DataInCodeLC.datasize / sizeof(MachO::data_in_code_entry);
190a34c753fSRafael Auler std::vector<DataInCodeRegion> DataInCode;
191a34c753fSRafael Auler DataInCode.reserve(NumberOfEntries);
192a34c753fSRafael Auler for (auto I = O.begin_dices(), E = O.end_dices(); I != E; ++I)
193a34c753fSRafael Auler DataInCode.emplace_back(*I);
194*d2c87699SAmir Ayupov llvm::stable_sort(DataInCode, [](DataInCodeRegion LHS, DataInCodeRegion RHS) {
195a34c753fSRafael Auler return LHS.Offset < RHS.Offset;
196a34c753fSRafael Auler });
197a34c753fSRafael Auler return DataInCode;
198a34c753fSRafael Auler }
199a34c753fSRafael Auler
readStartAddress(const MachOObjectFile & O)200a34c753fSRafael Auler Optional<uint64_t> readStartAddress(const MachOObjectFile &O) {
201a34c753fSRafael Auler Optional<uint64_t> StartOffset;
202a34c753fSRafael Auler Optional<uint64_t> TextVMAddr;
203a34c753fSRafael Auler for (const object::MachOObjectFile::LoadCommandInfo &LC : O.load_commands()) {
204a34c753fSRafael Auler switch (LC.C.cmd) {
205a34c753fSRafael Auler case MachO::LC_MAIN: {
206a34c753fSRafael Auler MachO::entry_point_command LCMain = O.getEntryPointCommand(LC);
207a34c753fSRafael Auler StartOffset = LCMain.entryoff;
208a34c753fSRafael Auler break;
209a34c753fSRafael Auler }
210a34c753fSRafael Auler case MachO::LC_SEGMENT: {
211a34c753fSRafael Auler MachO::segment_command LCSeg = O.getSegmentLoadCommand(LC);
212a34c753fSRafael Auler StringRef SegmentName(LCSeg.segname,
213a34c753fSRafael Auler strnlen(LCSeg.segname, sizeof(LCSeg.segname)));
214a34c753fSRafael Auler if (SegmentName == "__TEXT")
215a34c753fSRafael Auler TextVMAddr = LCSeg.vmaddr;
216a34c753fSRafael Auler break;
217a34c753fSRafael Auler }
218a34c753fSRafael Auler case MachO::LC_SEGMENT_64: {
219a34c753fSRafael Auler MachO::segment_command_64 LCSeg = O.getSegment64LoadCommand(LC);
220a34c753fSRafael Auler StringRef SegmentName(LCSeg.segname,
221a34c753fSRafael Auler strnlen(LCSeg.segname, sizeof(LCSeg.segname)));
222a34c753fSRafael Auler if (SegmentName == "__TEXT")
223a34c753fSRafael Auler TextVMAddr = LCSeg.vmaddr;
224a34c753fSRafael Auler break;
225a34c753fSRafael Auler }
226a34c753fSRafael Auler default:
227a34c753fSRafael Auler continue;
228a34c753fSRafael Auler }
229a34c753fSRafael Auler }
230a34c753fSRafael Auler return (TextVMAddr && StartOffset)
231a34c753fSRafael Auler ? Optional<uint64_t>(*TextVMAddr + *StartOffset)
232a34c753fSRafael Auler : llvm::None;
233a34c753fSRafael Auler }
234a34c753fSRafael Auler
235a34c753fSRafael Auler } // anonymous namespace
236a34c753fSRafael Auler
discoverFileObjects()237a34c753fSRafael Auler void MachORewriteInstance::discoverFileObjects() {
238a34c753fSRafael Auler std::vector<SymbolRef> FunctionSymbols;
239a34c753fSRafael Auler for (const SymbolRef &S : InputFile->symbols()) {
240a34c753fSRafael Auler SymbolRef::Type Type = cantFail(S.getType(), "cannot get symbol type");
241a34c753fSRafael Auler if (Type == SymbolRef::ST_Function)
242a34c753fSRafael Auler FunctionSymbols.push_back(S);
243a34c753fSRafael Auler }
244a34c753fSRafael Auler if (FunctionSymbols.empty())
245a34c753fSRafael Auler return;
246*d2c87699SAmir Ayupov llvm::stable_sort(
247*d2c87699SAmir Ayupov FunctionSymbols, [](const SymbolRef &LHS, const SymbolRef &RHS) {
248a34c753fSRafael Auler return cantFail(LHS.getValue()) < cantFail(RHS.getValue());
249a34c753fSRafael Auler });
250a34c753fSRafael Auler for (size_t Index = 0; Index < FunctionSymbols.size(); ++Index) {
251a34c753fSRafael Auler const uint64_t Address = cantFail(FunctionSymbols[Index].getValue());
252a34c753fSRafael Auler ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);
253a34c753fSRafael Auler // TODO: It happens for some symbols (e.g. __mh_execute_header).
254a34c753fSRafael Auler // Add proper logic to handle them correctly.
255a34c753fSRafael Auler if (!Section) {
256a34c753fSRafael Auler errs() << "BOLT-WARNING: no section found for address " << Address
257a34c753fSRafael Auler << "\n";
258a34c753fSRafael Auler continue;
259a34c753fSRafael Auler }
260a34c753fSRafael Auler
261a34c753fSRafael Auler std::string SymbolName =
262a34c753fSRafael Auler cantFail(FunctionSymbols[Index].getName(), "cannot get symbol name")
263a34c753fSRafael Auler .str();
264a34c753fSRafael Auler // Uniquify names of local symbols.
265a34c753fSRafael Auler if (!(cantFail(FunctionSymbols[Index].getFlags()) & SymbolRef::SF_Global))
266a34c753fSRafael Auler SymbolName = NR.uniquify(SymbolName);
267a34c753fSRafael Auler
268a34c753fSRafael Auler section_iterator S = cantFail(FunctionSymbols[Index].getSection());
269a34c753fSRafael Auler uint64_t EndAddress = S->getAddress() + S->getSize();
270a34c753fSRafael Auler
271a34c753fSRafael Auler size_t NFIndex = Index + 1;
272a34c753fSRafael Auler // Skip aliases.
273a34c753fSRafael Auler while (NFIndex < FunctionSymbols.size() &&
274a34c753fSRafael Auler cantFail(FunctionSymbols[NFIndex].getValue()) == Address)
275a34c753fSRafael Auler ++NFIndex;
276a34c753fSRafael Auler if (NFIndex < FunctionSymbols.size() &&
277a34c753fSRafael Auler S == cantFail(FunctionSymbols[NFIndex].getSection()))
278a34c753fSRafael Auler EndAddress = cantFail(FunctionSymbols[NFIndex].getValue());
279a34c753fSRafael Auler
280a34c753fSRafael Auler const uint64_t SymbolSize = EndAddress - Address;
281a34c753fSRafael Auler const auto It = BC->getBinaryFunctions().find(Address);
282a34c753fSRafael Auler if (It == BC->getBinaryFunctions().end()) {
283a34c753fSRafael Auler BinaryFunction *Function = BC->createBinaryFunction(
284a34c753fSRafael Auler std::move(SymbolName), *Section, Address, SymbolSize);
285a34c753fSRafael Auler if (!opts::Instrument)
286a34c753fSRafael Auler Function->setOutputAddress(Function->getAddress());
287a34c753fSRafael Auler
288a34c753fSRafael Auler } else {
289a34c753fSRafael Auler It->second.addAlternativeName(std::move(SymbolName));
290a34c753fSRafael Auler }
291a34c753fSRafael Auler }
292a34c753fSRafael Auler
293a34c753fSRafael Auler const std::vector<DataInCodeRegion> DataInCode = readDataInCode(*InputFile);
294a34c753fSRafael Auler
295a34c753fSRafael Auler for (auto &BFI : BC->getBinaryFunctions()) {
296a34c753fSRafael Auler BinaryFunction &Function = BFI.second;
297a34c753fSRafael Auler Function.setMaxSize(Function.getSize());
298a34c753fSRafael Auler
299a34c753fSRafael Auler ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();
300a34c753fSRafael Auler if (!FunctionData) {
301a34c753fSRafael Auler errs() << "BOLT-ERROR: corresponding section is non-executable or "
302a34c753fSRafael Auler << "empty for function " << Function << '\n';
303a34c753fSRafael Auler continue;
304a34c753fSRafael Auler }
305a34c753fSRafael Auler
306a34c753fSRafael Auler // Treat zero-sized functions as non-simple ones.
307a34c753fSRafael Auler if (Function.getSize() == 0) {
308a34c753fSRafael Auler Function.setSimple(false);
309a34c753fSRafael Auler continue;
310a34c753fSRafael Auler }
311a34c753fSRafael Auler
312a34c753fSRafael Auler // Offset of the function in the file.
313a34c753fSRafael Auler const auto *FileBegin =
314a34c753fSRafael Auler reinterpret_cast<const uint8_t *>(InputFile->getData().data());
315a34c753fSRafael Auler Function.setFileOffset(FunctionData->begin() - FileBegin);
316a34c753fSRafael Auler
317a34c753fSRafael Auler // Treat functions which contain data in code as non-simple ones.
318a34c753fSRafael Auler const auto It = std::lower_bound(
319a34c753fSRafael Auler DataInCode.cbegin(), DataInCode.cend(), Function.getFileOffset(),
320a34c753fSRafael Auler [](DataInCodeRegion D, uint64_t Offset) { return D.Offset < Offset; });
321a34c753fSRafael Auler if (It != DataInCode.cend() &&
322a34c753fSRafael Auler It->Offset + It->Length <=
323a34c753fSRafael Auler Function.getFileOffset() + Function.getMaxSize())
324a34c753fSRafael Auler Function.setSimple(false);
325a34c753fSRafael Auler }
326a34c753fSRafael Auler
327a34c753fSRafael Auler BC->StartFunctionAddress = readStartAddress(*InputFile);
328a34c753fSRafael Auler }
329a34c753fSRafael Auler
disassembleFunctions()330a34c753fSRafael Auler void MachORewriteInstance::disassembleFunctions() {
331a34c753fSRafael Auler for (auto &BFI : BC->getBinaryFunctions()) {
332a34c753fSRafael Auler BinaryFunction &Function = BFI.second;
333a34c753fSRafael Auler if (!Function.isSimple())
334a34c753fSRafael Auler continue;
335a34c753fSRafael Auler Function.disassemble();
336a34c753fSRafael Auler if (opts::PrintDisasm)
337a34c753fSRafael Auler Function.print(outs(), "after disassembly", true);
338a34c753fSRafael Auler }
339a34c753fSRafael Auler }
340a34c753fSRafael Auler
buildFunctionsCFG()341a34c753fSRafael Auler void MachORewriteInstance::buildFunctionsCFG() {
342a34c753fSRafael Auler for (auto &BFI : BC->getBinaryFunctions()) {
343a34c753fSRafael Auler BinaryFunction &Function = BFI.second;
344a34c753fSRafael Auler if (!Function.isSimple())
345a34c753fSRafael Auler continue;
346a34c753fSRafael Auler if (!Function.buildCFG(/*AllocId*/ 0)) {
347a34c753fSRafael Auler errs() << "BOLT-WARNING: failed to build CFG for the function "
348a34c753fSRafael Auler << Function << "\n";
349a34c753fSRafael Auler }
350a34c753fSRafael Auler }
351a34c753fSRafael Auler }
352a34c753fSRafael Auler
postProcessFunctions()353a34c753fSRafael Auler void MachORewriteInstance::postProcessFunctions() {
354a34c753fSRafael Auler for (auto &BFI : BC->getBinaryFunctions()) {
355a34c753fSRafael Auler BinaryFunction &Function = BFI.second;
356a34c753fSRafael Auler if (Function.empty())
357a34c753fSRafael Auler continue;
358a34c753fSRafael Auler Function.postProcessCFG();
359a34c753fSRafael Auler if (opts::PrintCFG)
360a34c753fSRafael Auler Function.print(outs(), "after building cfg", true);
361a34c753fSRafael Auler }
362a34c753fSRafael Auler }
363a34c753fSRafael Auler
runOptimizationPasses()364a34c753fSRafael Auler void MachORewriteInstance::runOptimizationPasses() {
365a34c753fSRafael Auler BinaryFunctionPassManager Manager(*BC);
366a34c753fSRafael Auler if (opts::Instrument) {
367a34c753fSRafael Auler Manager.registerPass(std::make_unique<PatchEntries>());
368a34c753fSRafael Auler Manager.registerPass(std::make_unique<Instrumentation>(opts::NeverPrint));
369a34c753fSRafael Auler }
370ccb99dd1SMaksim Panchenko
371ccb99dd1SMaksim Panchenko Manager.registerPass(std::make_unique<ShortenInstructions>(opts::NeverPrint));
372ccb99dd1SMaksim Panchenko
373ccb99dd1SMaksim Panchenko Manager.registerPass(std::make_unique<RemoveNops>(opts::NeverPrint));
374ccb99dd1SMaksim Panchenko
375ccb99dd1SMaksim Panchenko Manager.registerPass(std::make_unique<NormalizeCFG>(opts::PrintNormalized));
376ccb99dd1SMaksim Panchenko
377a34c753fSRafael Auler Manager.registerPass(
378a34c753fSRafael Auler std::make_unique<ReorderBasicBlocks>(opts::PrintReordered));
379a34c753fSRafael Auler Manager.registerPass(
380a34c753fSRafael Auler std::make_unique<FixupBranches>(opts::PrintAfterBranchFixup));
381a34c753fSRafael Auler // This pass should always run last.*
382a34c753fSRafael Auler Manager.registerPass(
383a34c753fSRafael Auler std::make_unique<FinalizeFunctions>(opts::PrintFinalized));
384a34c753fSRafael Auler
385a34c753fSRafael Auler Manager.runPasses();
386a34c753fSRafael Auler }
387a34c753fSRafael Auler
mapInstrumentationSection(StringRef SectionName)388a34c753fSRafael Auler void MachORewriteInstance::mapInstrumentationSection(StringRef SectionName) {
389a34c753fSRafael Auler if (!opts::Instrument)
390a34c753fSRafael Auler return;
391a34c753fSRafael Auler ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
392a34c753fSRafael Auler if (!Section) {
393a34c753fSRafael Auler llvm::errs() << "Cannot find " + SectionName + " section\n";
394a34c753fSRafael Auler exit(1);
395a34c753fSRafael Auler }
396a34c753fSRafael Auler if (!Section->hasValidSectionID())
397a34c753fSRafael Auler return;
398a34c753fSRafael Auler RTDyld->reassignSectionAddress(Section->getSectionID(),
399a34c753fSRafael Auler Section->getAddress());
400a34c753fSRafael Auler }
401a34c753fSRafael Auler
mapCodeSections()402a34c753fSRafael Auler void MachORewriteInstance::mapCodeSections() {
403a34c753fSRafael Auler for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
404a34c753fSRafael Auler if (!Function->isEmitted())
405a34c753fSRafael Auler continue;
406a34c753fSRafael Auler if (Function->getOutputAddress() == 0)
407a34c753fSRafael Auler continue;
408a34c753fSRafael Auler ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();
409a34c753fSRafael Auler if (!FuncSection)
410a34c753fSRafael Auler report_error(
411a34c753fSRafael Auler (Twine("Cannot find section for function ") + Function->getOneName())
412a34c753fSRafael Auler .str(),
413a34c753fSRafael Auler FuncSection.getError());
414a34c753fSRafael Auler
415a34c753fSRafael Auler FuncSection->setOutputAddress(Function->getOutputAddress());
416a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"
417a34c753fSRafael Auler << Twine::utohexstr(FuncSection->getAllocAddress()) << " to 0x"
418a34c753fSRafael Auler << Twine::utohexstr(Function->getOutputAddress()) << '\n');
419a34c753fSRafael Auler RTDyld->reassignSectionAddress(FuncSection->getSectionID(),
420a34c753fSRafael Auler Function->getOutputAddress());
421a34c753fSRafael Auler Function->setImageAddress(FuncSection->getAllocAddress());
422a34c753fSRafael Auler Function->setImageSize(FuncSection->getOutputSize());
423a34c753fSRafael Auler }
424a34c753fSRafael Auler
425a34c753fSRafael Auler if (opts::Instrument) {
426a34c753fSRafael Auler ErrorOr<BinarySection &> BOLT = BC->getUniqueSectionByName("__bolt");
427a34c753fSRafael Auler if (!BOLT) {
428a34c753fSRafael Auler llvm::errs() << "Cannot find __bolt section\n";
429a34c753fSRafael Auler exit(1);
430a34c753fSRafael Auler }
431a34c753fSRafael Auler uint64_t Addr = BOLT->getAddress();
432a34c753fSRafael Auler for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {
433a34c753fSRafael Auler if (!Function->isEmitted())
434a34c753fSRafael Auler continue;
435a34c753fSRafael Auler if (Function->getOutputAddress() != 0)
436a34c753fSRafael Auler continue;
437a34c753fSRafael Auler ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();
438a34c753fSRafael Auler assert(FuncSection && "cannot find section for function");
439a34c753fSRafael Auler Addr = llvm::alignTo(Addr, 4);
440a34c753fSRafael Auler FuncSection->setOutputAddress(Addr);
441a34c753fSRafael Auler RTDyld->reassignSectionAddress(FuncSection->getSectionID(), Addr);
442a34c753fSRafael Auler Function->setFileOffset(Addr - BOLT->getAddress() +
443a34c753fSRafael Auler BOLT->getInputFileOffset());
444a34c753fSRafael Auler Function->setImageAddress(FuncSection->getAllocAddress());
445a34c753fSRafael Auler Function->setImageSize(FuncSection->getOutputSize());
446a34c753fSRafael Auler BC->registerNameAtAddress(Function->getOneName(), Addr, 0, 0);
447a34c753fSRafael Auler Addr += FuncSection->getOutputSize();
448a34c753fSRafael Auler }
449a34c753fSRafael Auler }
450a34c753fSRafael Auler }
451a34c753fSRafael Auler
452a34c753fSRafael Auler namespace {
453a34c753fSRafael Auler
454a34c753fSRafael Auler class BOLTSymbolResolver : public LegacyJITSymbolResolver {
455a34c753fSRafael Auler BinaryContext &BC;
456a34c753fSRafael Auler public:
BOLTSymbolResolver(BinaryContext & BC)457a34c753fSRafael Auler BOLTSymbolResolver(BinaryContext &BC) : BC(BC) {}
458a34c753fSRafael Auler
findSymbolInLogicalDylib(const std::string & Name)459a34c753fSRafael Auler JITSymbol findSymbolInLogicalDylib(const std::string &Name) override {
460a34c753fSRafael Auler return JITSymbol(nullptr);
461a34c753fSRafael Auler }
462a34c753fSRafael Auler
findSymbol(const std::string & Name)463a34c753fSRafael Auler JITSymbol findSymbol(const std::string &Name) override {
464a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "BOLT: looking for " << Name << "\n");
465a34c753fSRafael Auler if (BinaryData *I = BC.getBinaryDataByName(Name)) {
466a34c753fSRafael Auler const uint64_t Address = I->isMoved() && !I->isJumpTable()
467a34c753fSRafael Auler ? I->getOutputAddress()
468a34c753fSRafael Auler : I->getAddress();
469a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "Resolved to address 0x" << Twine::utohexstr(Address)
470a34c753fSRafael Auler << "\n");
471a34c753fSRafael Auler return JITSymbol(Address, JITSymbolFlags());
472a34c753fSRafael Auler }
473a34c753fSRafael Auler LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n");
474a34c753fSRafael Auler return JITSymbol(nullptr);
475a34c753fSRafael Auler }
476a34c753fSRafael Auler };
477a34c753fSRafael Auler
478a34c753fSRafael Auler } // end anonymous namespace
479a34c753fSRafael Auler
emitAndLink()480a34c753fSRafael Auler void MachORewriteInstance::emitAndLink() {
481a34c753fSRafael Auler std::error_code EC;
482a34c753fSRafael Auler std::unique_ptr<::llvm::ToolOutputFile> TempOut =
483a34c753fSRafael Auler std::make_unique<::llvm::ToolOutputFile>(
484a34c753fSRafael Auler opts::OutputFilename + ".bolt.o", EC, sys::fs::OF_None);
485a34c753fSRafael Auler check_error(EC, "cannot create output object file");
486a34c753fSRafael Auler
487a34c753fSRafael Auler if (opts::KeepTmp)
488a34c753fSRafael Auler TempOut->keep();
489a34c753fSRafael Auler
490a34c753fSRafael Auler std::unique_ptr<buffer_ostream> BOS =
491a34c753fSRafael Auler std::make_unique<buffer_ostream>(TempOut->os());
492a34c753fSRafael Auler raw_pwrite_stream *OS = BOS.get();
493a34c753fSRafael Auler auto Streamer = BC->createStreamer(*OS);
494a34c753fSRafael Auler
495a34c753fSRafael Auler emitBinaryContext(*Streamer, *BC, getOrgSecPrefix());
49615d82c62SFangrui Song Streamer->finish();
497a34c753fSRafael Auler
498a34c753fSRafael Auler std::unique_ptr<MemoryBuffer> ObjectMemBuffer =
499a34c753fSRafael Auler MemoryBuffer::getMemBuffer(BOS->str(), "in-memory object file", false);
500a34c753fSRafael Auler std::unique_ptr<object::ObjectFile> Obj = cantFail(
501a34c753fSRafael Auler object::ObjectFile::createObjectFile(ObjectMemBuffer->getMemBufferRef()),
502a34c753fSRafael Auler "error creating in-memory object");
503a34c753fSRafael Auler assert(Obj && "createObjectFile cannot return nullptr");
504a34c753fSRafael Auler
505a34c753fSRafael Auler BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC);
506a34c753fSRafael Auler
507a34c753fSRafael Auler MCAsmLayout FinalLayout(
508a34c753fSRafael Auler static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler());
509a34c753fSRafael Auler
510a34c753fSRafael Auler BC->EFMM.reset(new ExecutableFileMemoryManager(*BC, /*AllowStubs*/ false));
511a34c753fSRafael Auler
512a34c753fSRafael Auler RTDyld.reset(new decltype(RTDyld)::element_type(*BC->EFMM, Resolver));
513a34c753fSRafael Auler RTDyld->setProcessAllSections(true);
514a34c753fSRafael Auler RTDyld->loadObject(*Obj);
515a34c753fSRafael Auler if (RTDyld->hasError()) {
516a34c753fSRafael Auler outs() << "BOLT-ERROR: RTDyld failed.\n";
517a34c753fSRafael Auler exit(1);
518a34c753fSRafael Auler }
519a34c753fSRafael Auler
520a34c753fSRafael Auler // Assign addresses to all sections. If key corresponds to the object
521a34c753fSRafael Auler // created by ourselves, call our regular mapping function. If we are
522a34c753fSRafael Auler // loading additional objects as part of runtime libraries for
523a34c753fSRafael Auler // instrumentation, treat them as extra sections.
524a34c753fSRafael Auler mapCodeSections();
525a34c753fSRafael Auler mapInstrumentationSection("__counters");
526a34c753fSRafael Auler mapInstrumentationSection("__tables");
527a34c753fSRafael Auler
528a34c753fSRafael Auler // TODO: Refactor addRuntimeLibSections to work properly on Mach-O
529a34c753fSRafael Auler // and use it here.
530a34c753fSRafael Auler //FIXME! Put this in RtLibrary->link
531a34c753fSRafael Auler // mapInstrumentationSection("I__setup");
532a34c753fSRafael Auler // mapInstrumentationSection("I__fini");
533a34c753fSRafael Auler // mapInstrumentationSection("I__data");
534a34c753fSRafael Auler // mapInstrumentationSection("I__text");
535a34c753fSRafael Auler // mapInstrumentationSection("I__cstring");
536a34c753fSRafael Auler // mapInstrumentationSection("I__literal16");
537a34c753fSRafael Auler
538a34c753fSRafael Auler // if (auto *RtLibrary = BC->getRuntimeLibrary()) {
539a34c753fSRafael Auler // RtLibrary->link(*BC, ToolPath, *ES, *OLT);
540a34c753fSRafael Auler // }
541a34c753fSRafael Auler }
542a34c753fSRafael Auler
writeInstrumentationSection(StringRef SectionName,raw_pwrite_stream & OS)543a34c753fSRafael Auler void MachORewriteInstance::writeInstrumentationSection(StringRef SectionName,
544a34c753fSRafael Auler raw_pwrite_stream &OS) {
545a34c753fSRafael Auler if (!opts::Instrument)
546a34c753fSRafael Auler return;
547a34c753fSRafael Auler ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);
548a34c753fSRafael Auler if (!Section) {
549a34c753fSRafael Auler llvm::errs() << "Cannot find " + SectionName + " section\n";
550a34c753fSRafael Auler exit(1);
551a34c753fSRafael Auler }
552a34c753fSRafael Auler if (!Section->hasValidSectionID())
553a34c753fSRafael Auler return;
554a34c753fSRafael Auler assert(Section->getInputFileOffset() &&
555a34c753fSRafael Auler "Section input offset cannot be zero");
556a34c753fSRafael Auler assert(Section->getAllocAddress() && "Section alloc address cannot be zero");
557a34c753fSRafael Auler assert(Section->getOutputSize() && "Section output size cannot be zero");
558a34c753fSRafael Auler OS.pwrite(reinterpret_cast<char *>(Section->getAllocAddress()),
559a34c753fSRafael Auler Section->getOutputSize(), Section->getInputFileOffset());
560a34c753fSRafael Auler }
561a34c753fSRafael Auler
rewriteFile()562a34c753fSRafael Auler void MachORewriteInstance::rewriteFile() {
563a34c753fSRafael Auler std::error_code EC;
564a34c753fSRafael Auler Out = std::make_unique<ToolOutputFile>(opts::OutputFilename, EC,
565a34c753fSRafael Auler sys::fs::OF_None);
566a34c753fSRafael Auler check_error(EC, "cannot create output executable file");
567a34c753fSRafael Auler raw_fd_ostream &OS = Out->os();
568a34c753fSRafael Auler OS << InputFile->getData();
569a34c753fSRafael Auler
570a34c753fSRafael Auler for (auto &BFI : BC->getBinaryFunctions()) {
571a34c753fSRafael Auler BinaryFunction &Function = BFI.second;
572a34c753fSRafael Auler if (!Function.isSimple())
573a34c753fSRafael Auler continue;
574a34c753fSRafael Auler assert(Function.isEmitted() && "Simple function has not been emitted");
575a34c753fSRafael Auler if (!opts::Instrument && (Function.getImageSize() > Function.getMaxSize()))
576a34c753fSRafael Auler continue;
577a34c753fSRafael Auler if (opts::Verbosity >= 2)
578a34c753fSRafael Auler outs() << "BOLT: rewriting function \"" << Function << "\"\n";
579a34c753fSRafael Auler OS.pwrite(reinterpret_cast<char *>(Function.getImageAddress()),
580a34c753fSRafael Auler Function.getImageSize(), Function.getFileOffset());
581a34c753fSRafael Auler }
582a34c753fSRafael Auler
583a34c753fSRafael Auler for (const BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {
584a34c753fSRafael Auler OS.pwrite(reinterpret_cast<char *>(Function->getImageAddress()),
585a34c753fSRafael Auler Function->getImageSize(), Function->getFileOffset());
586a34c753fSRafael Auler }
587a34c753fSRafael Auler
588a34c753fSRafael Auler writeInstrumentationSection("__counters", OS);
589a34c753fSRafael Auler writeInstrumentationSection("__tables", OS);
590a34c753fSRafael Auler
591a34c753fSRafael Auler // TODO: Refactor addRuntimeLibSections to work properly on Mach-O and
592a34c753fSRafael Auler // use it here.
593a34c753fSRafael Auler writeInstrumentationSection("I__setup", OS);
594a34c753fSRafael Auler writeInstrumentationSection("I__fini", OS);
595a34c753fSRafael Auler writeInstrumentationSection("I__data", OS);
596a34c753fSRafael Auler writeInstrumentationSection("I__text", OS);
597a34c753fSRafael Auler writeInstrumentationSection("I__cstring", OS);
598a34c753fSRafael Auler writeInstrumentationSection("I__literal16", OS);
599a34c753fSRafael Auler
600a34c753fSRafael Auler Out->keep();
601a34c753fSRafael Auler EC = sys::fs::setPermissions(opts::OutputFilename,
602a34c753fSRafael Auler sys::fs::perms::all_all);
603a34c753fSRafael Auler check_error(EC, "cannot set permissions of output file");
604a34c753fSRafael Auler }
605a34c753fSRafael Auler
adjustCommandLineOptions()606a34c753fSRafael Auler void MachORewriteInstance::adjustCommandLineOptions() {
607a34c753fSRafael Auler //FIXME! Upstream change
608a34c753fSRafael Auler // opts::CheckOverlappingElements = false;
609a34c753fSRafael Auler if (!opts::AlignText.getNumOccurrences())
610a34c753fSRafael Auler opts::AlignText = BC->PageAlign;
611a34c753fSRafael Auler if (opts::Instrument.getNumOccurrences())
612a34c753fSRafael Auler opts::ForcePatch = true;
613a34c753fSRafael Auler opts::JumpTables = JTS_MOVE;
614a34c753fSRafael Auler opts::InstrumentCalls = false;
615a34c753fSRafael Auler opts::RuntimeInstrumentationLib = "libbolt_rt_instr_osx.a";
616a34c753fSRafael Auler }
617a34c753fSRafael Auler
run()618a34c753fSRafael Auler void MachORewriteInstance::run() {
619a34c753fSRafael Auler adjustCommandLineOptions();
620a34c753fSRafael Auler
621a34c753fSRafael Auler readSpecialSections();
622a34c753fSRafael Auler
623a34c753fSRafael Auler discoverFileObjects();
624a34c753fSRafael Auler
625a34c753fSRafael Auler preprocessProfileData();
626a34c753fSRafael Auler
627a34c753fSRafael Auler disassembleFunctions();
628a34c753fSRafael Auler
629a34c753fSRafael Auler processProfileDataPreCFG();
630a34c753fSRafael Auler
631a34c753fSRafael Auler buildFunctionsCFG();
632a34c753fSRafael Auler
633a34c753fSRafael Auler processProfileData();
634a34c753fSRafael Auler
635a34c753fSRafael Auler postProcessFunctions();
636a34c753fSRafael Auler
637a34c753fSRafael Auler runOptimizationPasses();
638a34c753fSRafael Auler
639a34c753fSRafael Auler emitAndLink();
640a34c753fSRafael Auler
641a34c753fSRafael Auler rewriteFile();
642a34c753fSRafael Auler }
643a34c753fSRafael Auler
~MachORewriteInstance()644a34c753fSRafael Auler MachORewriteInstance::~MachORewriteInstance() {}
645a34c753fSRafael Auler
646a34c753fSRafael Auler } // namespace bolt
647a34c753fSRafael Auler } // namespace llvm
648