1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ProfileData/InstrProfCorrelator.h"
10 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
11 #include "llvm/Object/MachO.h"
12 #include "llvm/Support/Debug.h"
13 #include "llvm/Support/FileSystem.h"
14 #include "llvm/Support/Path.h"
15 
16 #define DEBUG_TYPE "correlator"
17 
18 using namespace llvm;
19 
20 /// Get the __llvm_prf_cnts section.
21 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
22   for (auto &Section : Obj.sections())
23     if (auto SectionName = Section.getName())
24       if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
25         return Section;
26   return make_error<InstrProfError>(
27       instrprof_error::unable_to_correlate_profile,
28       "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
29 }
30 
31 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
32 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
33 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
34 
35 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
36 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
37                                   const object::ObjectFile &Obj) {
38   auto CountersSection = getCountersSection(Obj);
39   if (auto Err = CountersSection.takeError())
40     return std::move(Err);
41   auto C = std::make_unique<Context>();
42   C->Buffer = std::move(Buffer);
43   C->CountersSectionStart = CountersSection->getAddress();
44   C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
45   C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
46   return Expected<std::unique_ptr<Context>>(std::move(C));
47 }
48 
49 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
50 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
51   auto DsymObjectsOrErr =
52       object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
53   if (auto Err = DsymObjectsOrErr.takeError())
54     return std::move(Err);
55   if (!DsymObjectsOrErr->empty()) {
56     // TODO: Enable profile correlation when there are multiple objects in a
57     // dSYM bundle.
58     if (DsymObjectsOrErr->size() > 1)
59       return make_error<InstrProfError>(
60           instrprof_error::unable_to_correlate_profile,
61           "using multiple objects is not yet supported");
62     DebugInfoFilename = *DsymObjectsOrErr->begin();
63   }
64   auto BufferOrErr =
65       errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
66   if (auto Err = BufferOrErr.takeError())
67     return std::move(Err);
68 
69   return get(std::move(*BufferOrErr));
70 }
71 
72 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
73 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
74   auto BinOrErr = object::createBinary(*Buffer);
75   if (auto Err = BinOrErr.takeError())
76     return std::move(Err);
77 
78   if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
79     auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
80     if (auto Err = CtxOrErr.takeError())
81       return std::move(Err);
82     auto T = Obj->makeTriple();
83     if (T.isArch64Bit())
84       return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
85     if (T.isArch32Bit())
86       return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
87   }
88   return make_error<InstrProfError>(
89       instrprof_error::unable_to_correlate_profile, "not an object file");
90 }
91 
92 Optional<size_t> InstrProfCorrelator::getDataSize() const {
93   if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
94     return C->getDataSize();
95   } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
96     return C->getDataSize();
97   }
98   return {};
99 }
100 
101 namespace llvm {
102 
103 template <>
104 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
105     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
106     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
107                               std::move(Ctx)) {}
108 template <>
109 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
110     std::unique_ptr<InstrProfCorrelator::Context> Ctx)
111     : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
112                               std::move(Ctx)) {}
113 template <>
114 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
115   return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
116 }
117 template <>
118 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
119   return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
120 }
121 
122 } // end namespace llvm
123 
124 template <class IntPtrT>
125 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
126 InstrProfCorrelatorImpl<IntPtrT>::get(
127     std::unique_ptr<InstrProfCorrelator::Context> Ctx,
128     const object::ObjectFile &Obj) {
129   if (Obj.isELF() || Obj.isMachO()) {
130     auto DICtx = DWARFContext::create(Obj);
131     return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
132                                                                std::move(Ctx));
133   }
134   return make_error<InstrProfError>(
135       instrprof_error::unable_to_correlate_profile,
136       "unsupported debug info format (only DWARF is supported)");
137 }
138 
139 template <class IntPtrT>
140 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
141   assert(Data.empty() && Names.empty() && NamesVec.empty());
142   correlateProfileDataImpl();
143   if (Data.empty() || NamesVec.empty())
144     return make_error<InstrProfError>(
145         instrprof_error::unable_to_correlate_profile,
146         "could not find any profile metadata in debug info");
147   auto Result =
148       collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
149   CounterOffsets.clear();
150   NamesVec.clear();
151   return Result;
152 }
153 
154 template <class IntPtrT>
155 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
156                                                 uint64_t CFGHash,
157                                                 IntPtrT CounterOffset,
158                                                 IntPtrT FunctionPtr,
159                                                 uint32_t NumCounters) {
160   // Check if a probe was already added for this counter offset.
161   if (!CounterOffsets.insert(CounterOffset).second)
162     return;
163   Data.push_back({
164       maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
165       maybeSwap<uint64_t>(CFGHash),
166       // In this mode, CounterPtr actually stores the section relative address
167       // of the counter.
168       maybeSwap<IntPtrT>(CounterOffset),
169       maybeSwap<IntPtrT>(FunctionPtr),
170       // TODO: Value profiling is not yet supported.
171       /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
172       maybeSwap<uint32_t>(NumCounters),
173       /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
174   });
175   NamesVec.push_back(FunctionName.str());
176 }
177 
178 template <class IntPtrT>
179 llvm::Optional<uint64_t>
180 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
181   auto Locations = Die.getLocations(dwarf::DW_AT_location);
182   if (!Locations) {
183     consumeError(Locations.takeError());
184     return {};
185   }
186   auto &DU = *Die.getDwarfUnit();
187   auto AddressSize = DU.getAddressByteSize();
188   for (auto &Location : *Locations) {
189     DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
190     DWARFExpression Expr(Data, AddressSize);
191     for (auto &Op : Expr) {
192       if (Op.getCode() == dwarf::DW_OP_addr) {
193         return Op.getRawOperand(0);
194       } else if (Op.getCode() == dwarf::DW_OP_addrx) {
195         uint64_t Index = Op.getRawOperand(0);
196         if (auto SA = DU.getAddrOffsetSectionItem(Index))
197           return SA->Address;
198       }
199     }
200   }
201   return {};
202 }
203 
204 template <class IntPtrT>
205 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
206   const auto &ParentDie = Die.getParent();
207   if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
208     return false;
209   if (Die.getTag() != dwarf::DW_TAG_variable)
210     return false;
211   if (!ParentDie.isSubprogramDIE())
212     return false;
213   if (!Die.hasChildren())
214     return false;
215   if (const char *Name = Die.getName(DINameKind::ShortName))
216     return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
217   return false;
218 }
219 
220 template <class IntPtrT>
221 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
222   auto maybeAddProbe = [&](DWARFDie Die) {
223     if (!isDIEOfProbe(Die))
224       return;
225     Optional<const char *> FunctionName;
226     Optional<uint64_t> CFGHash;
227     Optional<uint64_t> CounterPtr = getLocation(Die);
228     auto FunctionPtr =
229         dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
230     Optional<uint64_t> NumCounters;
231     for (const DWARFDie &Child : Die.children()) {
232       if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
233         continue;
234       auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
235       auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
236       if (!AnnotationFormName || !AnnotationFormValue)
237         continue;
238       auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
239       if (auto Err = AnnotationNameOrErr.takeError()) {
240         consumeError(std::move(Err));
241         continue;
242       }
243       StringRef AnnotationName = *AnnotationNameOrErr;
244       if (AnnotationName.compare(
245               InstrProfCorrelator::FunctionNameAttributeName) == 0) {
246         if (auto EC =
247                 AnnotationFormValue->getAsCString().moveInto(FunctionName))
248           consumeError(std::move(EC));
249       } else if (AnnotationName.compare(
250                      InstrProfCorrelator::CFGHashAttributeName) == 0) {
251         CFGHash = AnnotationFormValue->getAsUnsignedConstant();
252       } else if (AnnotationName.compare(
253                      InstrProfCorrelator::NumCountersAttributeName) == 0) {
254         NumCounters = AnnotationFormValue->getAsUnsignedConstant();
255       }
256     }
257     if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
258       LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
259                         << FunctionName << "\n\tCFGHash: " << CFGHash
260                         << "\n\tCounterPtr: " << CounterPtr
261                         << "\n\tNumCounters: " << NumCounters);
262       LLVM_DEBUG(Die.dump(dbgs()));
263       return;
264     }
265     uint64_t CountersStart = this->Ctx->CountersSectionStart;
266     uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
267     if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
268       LLVM_DEBUG(
269           dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
270                  << FunctionName << "\n\tExpected: [0x"
271                  << Twine::utohexstr(CountersStart) << ", 0x"
272                  << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
273                  << Twine::utohexstr(*CounterPtr));
274       LLVM_DEBUG(Die.dump(dbgs()));
275       return;
276     }
277     if (!FunctionPtr) {
278       LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
279                         << "\n");
280       LLVM_DEBUG(Die.dump(dbgs()));
281     }
282     this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
283                    FunctionPtr.getValueOr(0), *NumCounters);
284   };
285   for (auto &CU : DICtx->normal_units())
286     for (const auto &Entry : CU->dies())
287       maybeAddProbe(DWARFDie(CU.get(), &Entry));
288   for (auto &CU : DICtx->dwo_units())
289     for (const auto &Entry : CU->dies())
290       maybeAddProbe(DWARFDie(CU.get(), &Entry));
291 }
292