1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ProfileData/InstrProfCorrelator.h"
10 #include "llvm/Object/MachO.h"
11 #include "llvm/Support/Debug.h"
12 #include "llvm/Support/FileSystem.h"
13 #include "llvm/Support/Path.h"
14 
15 #define DEBUG_TYPE "correlator"
16 
17 using namespace llvm;
18 
19 /// Get the __llvm_prf_cnts section.
20 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
21   for (auto &Section : Obj.sections())
22     if (auto SectionName = Section.getName())
23       if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
24         return Section;
25   return make_error<InstrProfError>(
26       instrprof_error::unable_to_correlate_profile);
27 }
28 
29 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
30 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
31 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
32 
33 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
34 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
35                                   const object::ObjectFile &Obj) {
36   auto CountersSection = getCountersSection(Obj);
37   if (auto Err = CountersSection.takeError())
38     return std::move(Err);
39   auto C = std::make_unique<Context>();
40   C->Buffer = std::move(Buffer);
41   C->CountersSectionStart = CountersSection->getAddress();
42   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
43   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
44   return Expected<std::unique_ptr<Context>>(std::move(C));
45 }
46 
47 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
48 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
49   auto DsymObjectsOrErr =
50       object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
51   if (auto Err = DsymObjectsOrErr.takeError())
52     return std::move(Err);
53   if (!DsymObjectsOrErr->empty()) {
54     // TODO: Enable profile correlation when there are multiple objects in a
55     // dSYM bundle.
56     if (DsymObjectsOrErr->size() > 1)
57       return createStringError(
58           std::error_code(),
59           "Profile correlation using multiple objects is not yet supported");
60     DebugInfoFilename = *DsymObjectsOrErr->begin();
61   }
62   auto BufferOrErr =
63       errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
64   if (auto Err = BufferOrErr.takeError())
65     return std::move(Err);
66 
67   return get(std::move(*BufferOrErr));
68 }
69 
70 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
71 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
72   auto BinOrErr = object::createBinary(*Buffer);
73   if (auto Err = BinOrErr.takeError())
74     return std::move(Err);
75 
76   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
77     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
78     if (auto Err = CtxOrErr.takeError())
79       return std::move(Err);
80     auto T = Obj->makeTriple();
81     if (T.isArch64Bit())
82       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
83     if (T.isArch32Bit())
84       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
85   }
86   return make_error<InstrProfError>(
87       instrprof_error::unable_to_correlate_profile);
88 }
89 
90 namespace llvm {
91 
92 template <>
93 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
94     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
95     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
96                               std::move(Ctx)) {}
97 template <>
98 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
99     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
100     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
101                               std::move(Ctx)) {}
102 template <>
103 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
104   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
105 }
106 template <>
107 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
108   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
109 }
110 
111 } // end namespace llvm
112 
113 template <class IntPtrT>
114 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
115 InstrProfCorrelatorImpl<IntPtrT>::get(
116     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
117     const object::ObjectFile &Obj) {
118   if (Obj.isELF() || Obj.isMachO()) {
119     auto DICtx = DWARFContext::create(Obj);
120     return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
121                                                                std::move(Ctx));
122   }
123   return make_error<InstrProfError>(instrprof_error::unsupported_debug_format);
124 }
125 
126 template <class IntPtrT>
127 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
128   assert(Data.empty() && Names.empty() && NamesVec.empty());
129   correlateProfileDataImpl();
130   auto Result =
131       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
132   CounterOffsets.clear();
133   NamesVec.clear();
134   return Result;
135 }
136 
137 template <class IntPtrT>
138 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
139                                                 uint64_t CFGHash,
140                                                 IntPtrT CounterOffset,
141                                                 IntPtrT FunctionPtr,
142                                                 uint32_t NumCounters) {
143   // Check if a probe was already added for this counter offset.
144   if (!CounterOffsets.insert(CounterOffset).second)
145     return;
146   Data.push_back({
147       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
148       maybeSwap<uint64_t>(CFGHash),
149       // In this mode, CounterPtr actually stores the section relative address
150       // of the counter.
151       maybeSwap<IntPtrT>(CounterOffset),
152       maybeSwap<IntPtrT>(FunctionPtr),
153       // TODO: Value profiling is not yet supported.
154       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
155       maybeSwap<uint32_t>(NumCounters),
156       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
157   });
158   NamesVec.push_back(FunctionName.str());
159 }
160 
161 template <class IntPtrT>
162 llvm::Optional<uint64_t>
163 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
164   auto Locations = Die.getLocations(dwarf::DW_AT_location);
165   if (!Locations) {
166     consumeError(Locations.takeError());
167     return {};
168   }
169   auto &DU = *Die.getDwarfUnit();
170   auto AddressSize = DU.getAddressByteSize();
171   for (auto &Location : *Locations) {
172     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
173     DWARFExpression Expr(Data, AddressSize);
174     for (auto &Op : Expr) {
175       if (Op.getCode() == dwarf::DW_OP_addr) {
176         return Op.getRawOperand(0);
177       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
178         uint64_t Index = Op.getRawOperand(0);
179         if (auto SA = DU.getAddrOffsetSectionItem(Index))
180           return SA->Address;
181       }
182     }
183   }
184   return {};
185 }
186 
187 template <class IntPtrT>
188 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
189   const auto &ParentDie = Die.getParent();
190   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
191     return false;
192   if (Die.getTag() != dwarf::DW_TAG_variable)
193     return false;
194   if (!ParentDie.isSubprogramDIE())
195     return false;
196   if (!Die.hasChildren())
197     return false;
198   if (const char *Name = Die.getName(DINameKind::ShortName))
199     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
200   return false;
201 }
202 
203 template <class IntPtrT>
204 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
205   auto maybeAddProbe = [&](DWARFDie Die) {
206     if (!isDIEOfProbe(Die))
207       return;
208     Optional<const char *> FunctionName;
209     Optional<uint64_t> CFGHash;
210     Optional<uint64_t> CounterPtr = getLocation(Die);
211     auto FunctionPtr =
212         dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
213     Optional<uint64_t> NumCounters;
214     for (const DWARFDie &Child : Die.children()) {
215       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
216         continue;
217       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
218       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
219       if (!AnnotationFormName || !AnnotationFormValue)
220         continue;
221       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
222       if (auto Err = AnnotationNameOrErr.takeError()) {
223         consumeError(std::move(Err));
224         continue;
225       }
226       StringRef AnnotationName = *AnnotationNameOrErr;
227       if (AnnotationName.compare(
228               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
229         if (auto EC =
230                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
231           consumeError(std::move(EC));
232       } else if (AnnotationName.compare(
233                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
234         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
235       } else if (AnnotationName.compare(
236                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
237         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
238       }
239     }
240     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
241       LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
242                         << FunctionName << "\n\tCFGHash: " << CFGHash
243                         << "\n\tCounterPtr: " << CounterPtr
244                         << "\n\tNumCounters: " << NumCounters);
245       LLVM_DEBUG(Die.dump(dbgs()));
246       return;
247     }
248     uint64_t CountersStart = this->Ctx->CountersSectionStart;
249     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
250     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
251       LLVM_DEBUG(
252           dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
253                  << FunctionName << "\n\tExpected: [0x"
254                  << Twine::utohexstr(CountersStart) << ", 0x"
255                  << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
256                  << Twine::utohexstr(*CounterPtr));
257       LLVM_DEBUG(Die.dump(dbgs()));
258       return;
259     }
260     if (!FunctionPtr) {
261       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
262                         << "\n");
263       LLVM_DEBUG(Die.dump(dbgs()));
264     }
265     this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
266                    FunctionPtr.getValueOr(0), *NumCounters);
267   };
268   for (auto &CU : DICtx->normal_units())
269     for (const auto &Entry : CU->dies())
270       maybeAddProbe(DWARFDie(CU.get(), &Entry));
271   for (auto &CU : DICtx->dwo_units())
272     for (const auto &Entry : CU->dies())
273       maybeAddProbe(DWARFDie(CU.get(), &Entry));
274 }
275