1 //===-- InstrProfCorrelator.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ProfileData/InstrProfCorrelator.h"
10 #include "llvm/DebugInfo/DIContext.h"
11 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
12 #include "llvm/DebugInfo/DWARF/DWARFDie.h"
13 #include "llvm/DebugInfo/DWARF/DWARFExpression.h"
14 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
15 #include "llvm/DebugInfo/DWARF/DWARFLocationExpression.h"
16 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
17 #include "llvm/Object/MachO.h"
18 #include "llvm/Support/Debug.h"
19
20 #define DEBUG_TYPE "correlator"
21
22 using namespace llvm;
23
24 /// Get the __llvm_prf_cnts section.
getCountersSection(const object::ObjectFile & Obj)25 Expected<object::SectionRef> getCountersSection(const object::ObjectFile &Obj) {
26 for (auto &Section : Obj.sections())
27 if (auto SectionName = Section.getName())
28 if (SectionName.get() == INSTR_PROF_CNTS_SECT_NAME)
29 return Section;
30 return make_error<InstrProfError>(
31 instrprof_error::unable_to_correlate_profile,
32 "could not find counter section (" INSTR_PROF_CNTS_SECT_NAME ")");
33 }
34
35 const char *InstrProfCorrelator::FunctionNameAttributeName = "Function Name";
36 const char *InstrProfCorrelator::CFGHashAttributeName = "CFG Hash";
37 const char *InstrProfCorrelator::NumCountersAttributeName = "Num Counters";
38
39 llvm::Expected<std::unique_ptr<InstrProfCorrelator::Context>>
get(std::unique_ptr<MemoryBuffer> Buffer,const object::ObjectFile & Obj)40 InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
41 const object::ObjectFile &Obj) {
42 auto CountersSection = getCountersSection(Obj);
43 if (auto Err = CountersSection.takeError())
44 return std::move(Err);
45 auto C = std::make_unique<Context>();
46 C->Buffer = std::move(Buffer);
47 C->CountersSectionStart = CountersSection->getAddress();
48 C->CountersSectionEnd = C->CountersSectionStart + CountersSection->getSize();
49 C->ShouldSwapBytes = Obj.isLittleEndian() != sys::IsLittleEndianHost;
50 return Expected<std::unique_ptr<Context>>(std::move(C));
51 }
52
53 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(StringRef DebugInfoFilename)54 InstrProfCorrelator::get(StringRef DebugInfoFilename) {
55 auto DsymObjectsOrErr =
56 object::MachOObjectFile::findDsymObjectMembers(DebugInfoFilename);
57 if (auto Err = DsymObjectsOrErr.takeError())
58 return std::move(Err);
59 if (!DsymObjectsOrErr->empty()) {
60 // TODO: Enable profile correlation when there are multiple objects in a
61 // dSYM bundle.
62 if (DsymObjectsOrErr->size() > 1)
63 return make_error<InstrProfError>(
64 instrprof_error::unable_to_correlate_profile,
65 "using multiple objects is not yet supported");
66 DebugInfoFilename = *DsymObjectsOrErr->begin();
67 }
68 auto BufferOrErr =
69 errorOrToExpected(MemoryBuffer::getFile(DebugInfoFilename));
70 if (auto Err = BufferOrErr.takeError())
71 return std::move(Err);
72
73 return get(std::move(*BufferOrErr));
74 }
75
76 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(std::unique_ptr<MemoryBuffer> Buffer)77 InstrProfCorrelator::get(std::unique_ptr<MemoryBuffer> Buffer) {
78 auto BinOrErr = object::createBinary(*Buffer);
79 if (auto Err = BinOrErr.takeError())
80 return std::move(Err);
81
82 if (auto *Obj = dyn_cast<object::ObjectFile>(BinOrErr->get())) {
83 auto CtxOrErr = Context::get(std::move(Buffer), *Obj);
84 if (auto Err = CtxOrErr.takeError())
85 return std::move(Err);
86 auto T = Obj->makeTriple();
87 if (T.isArch64Bit())
88 return InstrProfCorrelatorImpl<uint64_t>::get(std::move(*CtxOrErr), *Obj);
89 if (T.isArch32Bit())
90 return InstrProfCorrelatorImpl<uint32_t>::get(std::move(*CtxOrErr), *Obj);
91 }
92 return make_error<InstrProfError>(
93 instrprof_error::unable_to_correlate_profile, "not an object file");
94 }
95
getDataSize() const96 Optional<size_t> InstrProfCorrelator::getDataSize() const {
97 if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint32_t>>(this)) {
98 return C->getDataSize();
99 } else if (auto *C = dyn_cast<InstrProfCorrelatorImpl<uint64_t>>(this)) {
100 return C->getDataSize();
101 }
102 return {};
103 }
104
105 namespace llvm {
106
107 template <>
InstrProfCorrelatorImpl(std::unique_ptr<InstrProfCorrelator::Context> Ctx)108 InstrProfCorrelatorImpl<uint32_t>::InstrProfCorrelatorImpl(
109 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
110 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_32Bit,
111 std::move(Ctx)) {}
112 template <>
InstrProfCorrelatorImpl(std::unique_ptr<InstrProfCorrelator::Context> Ctx)113 InstrProfCorrelatorImpl<uint64_t>::InstrProfCorrelatorImpl(
114 std::unique_ptr<InstrProfCorrelator::Context> Ctx)
115 : InstrProfCorrelatorImpl(InstrProfCorrelatorKind::CK_64Bit,
116 std::move(Ctx)) {}
117 template <>
classof(const InstrProfCorrelator * C)118 bool InstrProfCorrelatorImpl<uint32_t>::classof(const InstrProfCorrelator *C) {
119 return C->getKind() == InstrProfCorrelatorKind::CK_32Bit;
120 }
121 template <>
classof(const InstrProfCorrelator * C)122 bool InstrProfCorrelatorImpl<uint64_t>::classof(const InstrProfCorrelator *C) {
123 return C->getKind() == InstrProfCorrelatorKind::CK_64Bit;
124 }
125
126 } // end namespace llvm
127
128 template <class IntPtrT>
129 llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,const object::ObjectFile & Obj)130 InstrProfCorrelatorImpl<IntPtrT>::get(
131 std::unique_ptr<InstrProfCorrelator::Context> Ctx,
132 const object::ObjectFile &Obj) {
133 if (Obj.isELF() || Obj.isMachO()) {
134 auto DICtx = DWARFContext::create(Obj);
135 return std::make_unique<DwarfInstrProfCorrelator<IntPtrT>>(std::move(DICtx),
136 std::move(Ctx));
137 }
138 return make_error<InstrProfError>(
139 instrprof_error::unable_to_correlate_profile,
140 "unsupported debug info format (only DWARF is supported)");
141 }
142
143 template <class IntPtrT>
correlateProfileData()144 Error InstrProfCorrelatorImpl<IntPtrT>::correlateProfileData() {
145 assert(Data.empty() && Names.empty() && NamesVec.empty());
146 correlateProfileDataImpl();
147 if (Data.empty() || NamesVec.empty())
148 return make_error<InstrProfError>(
149 instrprof_error::unable_to_correlate_profile,
150 "could not find any profile metadata in debug info");
151 auto Result =
152 collectPGOFuncNameStrings(NamesVec, /*doCompression=*/false, Names);
153 CounterOffsets.clear();
154 NamesVec.clear();
155 return Result;
156 }
157
158 template <class IntPtrT>
addProbe(StringRef FunctionName,uint64_t CFGHash,IntPtrT CounterOffset,IntPtrT FunctionPtr,uint32_t NumCounters)159 void InstrProfCorrelatorImpl<IntPtrT>::addProbe(StringRef FunctionName,
160 uint64_t CFGHash,
161 IntPtrT CounterOffset,
162 IntPtrT FunctionPtr,
163 uint32_t NumCounters) {
164 // Check if a probe was already added for this counter offset.
165 if (!CounterOffsets.insert(CounterOffset).second)
166 return;
167 Data.push_back({
168 maybeSwap<uint64_t>(IndexedInstrProf::ComputeHash(FunctionName)),
169 maybeSwap<uint64_t>(CFGHash),
170 // In this mode, CounterPtr actually stores the section relative address
171 // of the counter.
172 maybeSwap<IntPtrT>(CounterOffset),
173 maybeSwap<IntPtrT>(FunctionPtr),
174 // TODO: Value profiling is not yet supported.
175 /*ValuesPtr=*/maybeSwap<IntPtrT>(0),
176 maybeSwap<uint32_t>(NumCounters),
177 /*NumValueSites=*/{maybeSwap<uint16_t>(0), maybeSwap<uint16_t>(0)},
178 });
179 NamesVec.push_back(FunctionName.str());
180 }
181
182 template <class IntPtrT>
183 llvm::Optional<uint64_t>
getLocation(const DWARFDie & Die) const184 DwarfInstrProfCorrelator<IntPtrT>::getLocation(const DWARFDie &Die) const {
185 auto Locations = Die.getLocations(dwarf::DW_AT_location);
186 if (!Locations) {
187 consumeError(Locations.takeError());
188 return {};
189 }
190 auto &DU = *Die.getDwarfUnit();
191 auto AddressSize = DU.getAddressByteSize();
192 for (auto &Location : *Locations) {
193 DataExtractor Data(Location.Expr, DICtx->isLittleEndian(), AddressSize);
194 DWARFExpression Expr(Data, AddressSize);
195 for (auto &Op : Expr) {
196 if (Op.getCode() == dwarf::DW_OP_addr) {
197 return Op.getRawOperand(0);
198 } else if (Op.getCode() == dwarf::DW_OP_addrx) {
199 uint64_t Index = Op.getRawOperand(0);
200 if (auto SA = DU.getAddrOffsetSectionItem(Index))
201 return SA->Address;
202 }
203 }
204 }
205 return {};
206 }
207
208 template <class IntPtrT>
isDIEOfProbe(const DWARFDie & Die)209 bool DwarfInstrProfCorrelator<IntPtrT>::isDIEOfProbe(const DWARFDie &Die) {
210 const auto &ParentDie = Die.getParent();
211 if (!Die.isValid() || !ParentDie.isValid() || Die.isNULL())
212 return false;
213 if (Die.getTag() != dwarf::DW_TAG_variable)
214 return false;
215 if (!ParentDie.isSubprogramDIE())
216 return false;
217 if (!Die.hasChildren())
218 return false;
219 if (const char *Name = Die.getName(DINameKind::ShortName))
220 return StringRef(Name).startswith(getInstrProfCountersVarPrefix());
221 return false;
222 }
223
224 template <class IntPtrT>
correlateProfileDataImpl()225 void DwarfInstrProfCorrelator<IntPtrT>::correlateProfileDataImpl() {
226 auto maybeAddProbe = [&](DWARFDie Die) {
227 if (!isDIEOfProbe(Die))
228 return;
229 Optional<const char *> FunctionName;
230 Optional<uint64_t> CFGHash;
231 Optional<uint64_t> CounterPtr = getLocation(Die);
232 auto FunctionPtr =
233 dwarf::toAddress(Die.getParent().find(dwarf::DW_AT_low_pc));
234 Optional<uint64_t> NumCounters;
235 for (const DWARFDie &Child : Die.children()) {
236 if (Child.getTag() != dwarf::DW_TAG_LLVM_annotation)
237 continue;
238 auto AnnotationFormName = Child.find(dwarf::DW_AT_name);
239 auto AnnotationFormValue = Child.find(dwarf::DW_AT_const_value);
240 if (!AnnotationFormName || !AnnotationFormValue)
241 continue;
242 auto AnnotationNameOrErr = AnnotationFormName->getAsCString();
243 if (auto Err = AnnotationNameOrErr.takeError()) {
244 consumeError(std::move(Err));
245 continue;
246 }
247 StringRef AnnotationName = *AnnotationNameOrErr;
248 if (AnnotationName.compare(
249 InstrProfCorrelator::FunctionNameAttributeName) == 0) {
250 if (auto EC =
251 AnnotationFormValue->getAsCString().moveInto(FunctionName))
252 consumeError(std::move(EC));
253 } else if (AnnotationName.compare(
254 InstrProfCorrelator::CFGHashAttributeName) == 0) {
255 CFGHash = AnnotationFormValue->getAsUnsignedConstant();
256 } else if (AnnotationName.compare(
257 InstrProfCorrelator::NumCountersAttributeName) == 0) {
258 NumCounters = AnnotationFormValue->getAsUnsignedConstant();
259 }
260 }
261 if (!FunctionName || !CFGHash || !CounterPtr || !NumCounters) {
262 LLVM_DEBUG(dbgs() << "Incomplete DIE for probe\n\tFunctionName: "
263 << FunctionName << "\n\tCFGHash: " << CFGHash
264 << "\n\tCounterPtr: " << CounterPtr
265 << "\n\tNumCounters: " << NumCounters);
266 LLVM_DEBUG(Die.dump(dbgs()));
267 return;
268 }
269 uint64_t CountersStart = this->Ctx->CountersSectionStart;
270 uint64_t CountersEnd = this->Ctx->CountersSectionEnd;
271 if (*CounterPtr < CountersStart || *CounterPtr >= CountersEnd) {
272 LLVM_DEBUG(
273 dbgs() << "CounterPtr out of range for probe\n\tFunction Name: "
274 << FunctionName << "\n\tExpected: [0x"
275 << Twine::utohexstr(CountersStart) << ", 0x"
276 << Twine::utohexstr(CountersEnd) << ")\n\tActual: 0x"
277 << Twine::utohexstr(*CounterPtr));
278 LLVM_DEBUG(Die.dump(dbgs()));
279 return;
280 }
281 if (!FunctionPtr) {
282 LLVM_DEBUG(dbgs() << "Could not find address of " << *FunctionName
283 << "\n");
284 LLVM_DEBUG(Die.dump(dbgs()));
285 }
286 this->addProbe(*FunctionName, *CFGHash, *CounterPtr - CountersStart,
287 FunctionPtr.value_or(0), *NumCounters);
288 };
289 for (auto &CU : DICtx->normal_units())
290 for (const auto &Entry : CU->dies())
291 maybeAddProbe(DWARFDie(CU.get(), &Entry));
292 for (auto &CU : DICtx->dwo_units())
293 for (const auto &Entry : CU->dies())
294 maybeAddProbe(DWARFDie(CU.get(), &Entry));
295 }
296