1 //===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass applies cache prefetch instructions based on a profile. The pass
11 // assumes DiscriminateMemOps ran immediately before, to ensure debug info
12 // matches the one used at profile generation time. The profile is encoded in
13 // afdo format (text or binary). It contains prefetch hints recommendations.
14 // Each recommendation is made in terms of debug info locations, a type (i.e.
15 // nta, t{0|1|2}) and a delta. The debug info identifies an instruction with a
16 // memory operand (see X86DiscriminateMemOps). The prefetch will be made for
17 // a location at that memory operand + the delta specified in the
18 // recommendation.
19 //
20 //===----------------------------------------------------------------------===//
21 
22 #include "X86.h"
23 #include "X86InstrBuilder.h"
24 #include "X86InstrInfo.h"
25 #include "X86MachineFunctionInfo.h"
26 #include "X86Subtarget.h"
27 #include "llvm/CodeGen/MachineModuleInfo.h"
28 #include "llvm/IR/DebugInfoMetadata.h"
29 #include "llvm/ProfileData/SampleProf.h"
30 #include "llvm/ProfileData/SampleProfReader.h"
31 #include "llvm/Transforms/IPO/SampleProfile.h"
32 using namespace llvm;
33 using namespace sampleprof;
34 
35 static cl::opt<std::string>
36     PrefetchHintsFile("prefetch-hints-file",
37                       cl::desc("Path to the prefetch hints profile."),
38                       cl::Hidden);
39 namespace {
40 
41 class X86InsertPrefetch : public MachineFunctionPass {
42   void getAnalysisUsage(AnalysisUsage &AU) const override;
43   bool doInitialization(Module &) override;
44 
45   bool runOnMachineFunction(MachineFunction &MF) override;
46   struct PrefetchInfo {
47     unsigned InstructionID;
48     int64_t Delta;
49   };
50   typedef SmallVectorImpl<PrefetchInfo> Prefetches;
51   bool findPrefetchInfo(const FunctionSamples *Samples, const MachineInstr &MI,
52                         Prefetches &prefetches) const;
53 
54 public:
55   static char ID;
56   X86InsertPrefetch(const std::string &PrefetchHintsFilename);
57 
58 private:
59   std::string Filename;
60   std::unique_ptr<SampleProfileReader> Reader;
61 };
62 
63 using PrefetchHints = SampleRecord::CallTargetMap;
64 
65 // Return any prefetching hints for the specified MachineInstruction. The hints
66 // are returned as pairs (name, delta).
67 ErrorOr<PrefetchHints> getPrefetchHints(const FunctionSamples *TopSamples,
68                                         const MachineInstr &MI) {
69   if (const auto &Loc = MI.getDebugLoc())
70     if (const auto *Samples = TopSamples->findFunctionSamples(Loc))
71       return Samples->findCallTargetMapAt(FunctionSamples::getOffset(Loc),
72                                           Loc->getBaseDiscriminator());
73   return std::error_code();
74 }
75 
76 } // end anonymous namespace
77 
78 //===----------------------------------------------------------------------===//
79 //            Implementation
80 //===----------------------------------------------------------------------===//
81 
82 char X86InsertPrefetch::ID = 0;
83 
84 X86InsertPrefetch::X86InsertPrefetch(const std::string &PrefetchHintsFilename)
85     : MachineFunctionPass(ID), Filename(PrefetchHintsFilename) {}
86 
87 /// Return true if the provided MachineInstruction has cache prefetch hints. In
88 /// that case, the prefetch hints are stored, in order, in the Prefetches
89 /// vector.
90 bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
91                                          const MachineInstr &MI,
92                                          Prefetches &Prefetches) const {
93   assert(Prefetches.empty() &&
94          "Expected caller passed empty PrefetchInfo vector.");
95   static const std::pair<const StringRef, unsigned> HintTypes[] = {
96       {"_nta_", X86::PREFETCHNTA},
97       {"_t0_", X86::PREFETCHT0},
98       {"_t1_", X86::PREFETCHT1},
99       {"_t2_", X86::PREFETCHT2},
100   };
101   static const char *SerializedPrefetchPrefix = "__prefetch";
102 
103   const ErrorOr<PrefetchHints> T = getPrefetchHints(TopSamples, MI);
104   if (!T)
105     return false;
106   int16_t max_index = -1;
107   // Convert serialized prefetch hints into PrefetchInfo objects, and populate
108   // the Prefetches vector.
109   for (const auto &S_V : *T) {
110     StringRef Name = S_V.getKey();
111     if (Name.consume_front(SerializedPrefetchPrefix)) {
112       int64_t D = static_cast<int64_t>(S_V.second);
113       unsigned IID = 0;
114       for (const auto &HintType : HintTypes) {
115         if (Name.startswith(HintType.first)) {
116           Name = Name.drop_front(HintType.first.size());
117           IID = HintType.second;
118           break;
119         }
120       }
121       if (IID == 0)
122         return false;
123       uint8_t index = 0;
124       Name.consumeInteger(10, index);
125 
126       if (index >= Prefetches.size())
127         Prefetches.resize(index + 1);
128       Prefetches[index] = {IID, D};
129       max_index = std::max(max_index, static_cast<int16_t>(index));
130     }
131   }
132   assert(max_index + 1 >= 0 &&
133          "Possible overflow: max_index + 1 should be positive.");
134   assert(static_cast<size_t>(max_index + 1) == Prefetches.size() &&
135          "The number of prefetch hints received should match the number of "
136          "PrefetchInfo objects returned");
137   return !Prefetches.empty();
138 }
139 
140 bool X86InsertPrefetch::doInitialization(Module &M) {
141   if (Filename.empty())
142     return false;
143 
144   LLVMContext &Ctx = M.getContext();
145   ErrorOr<std::unique_ptr<SampleProfileReader>> ReaderOrErr =
146       SampleProfileReader::create(Filename, Ctx);
147   if (std::error_code EC = ReaderOrErr.getError()) {
148     std::string Msg = "Could not open profile: " + EC.message();
149     Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg,
150                                              DiagnosticSeverity::DS_Warning));
151     return false;
152   }
153   Reader = std::move(ReaderOrErr.get());
154   Reader->read();
155   return true;
156 }
157 
158 void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
159   AU.setPreservesAll();
160   AU.addRequired<MachineModuleInfo>();
161 }
162 
163 bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
164   if (!Reader)
165     return false;
166   const FunctionSamples *Samples = Reader->getSamplesFor(MF.getFunction());
167   if (!Samples)
168     return false;
169 
170   bool Changed = false;
171 
172   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
173   SmallVector<PrefetchInfo, 4> Prefetches;
174   for (auto &MBB : MF) {
175     for (auto MI = MBB.instr_begin(); MI != MBB.instr_end();) {
176       auto Current = MI;
177       ++MI;
178 
179       int Offset = X86II::getMemoryOperandNo(Current->getDesc().TSFlags);
180       if (Offset < 0)
181         continue;
182       Prefetches.clear();
183       if (!findPrefetchInfo(Samples, *Current, Prefetches))
184         continue;
185       assert(!Prefetches.empty() &&
186              "The Prefetches vector should contain at least a value if "
187              "findPrefetchInfo returned true.");
188       for (auto &PrefInfo : Prefetches) {
189         unsigned PFetchInstrID = PrefInfo.InstructionID;
190         int64_t Delta = PrefInfo.Delta;
191         const MCInstrDesc &Desc = TII->get(PFetchInstrID);
192         MachineInstr *PFetch =
193             MF.CreateMachineInstr(Desc, Current->getDebugLoc(), true);
194         MachineInstrBuilder MIB(MF, PFetch);
195         unsigned Bias = X86II::getOperandBias(Current->getDesc());
196         int MemOpOffset = Offset + Bias;
197 
198         assert(X86::AddrBaseReg == 0 && X86::AddrScaleAmt == 1 &&
199                X86::AddrIndexReg == 2 && X86::AddrDisp == 3 &&
200                X86::AddrSegmentReg == 4 &&
201                "Unexpected change in X86 operand offset order.");
202 
203         // This assumes X86::AddBaseReg = 0, {...}ScaleAmt = 1, etc.
204         // FIXME(mtrofin): consider adding a:
205         //     MachineInstrBuilder::set(unsigned offset, op).
206         MIB.addReg(Current->getOperand(MemOpOffset + X86::AddrBaseReg).getReg())
207             .addImm(
208                 Current->getOperand(MemOpOffset + X86::AddrScaleAmt).getImm())
209             .addReg(
210                 Current->getOperand(MemOpOffset + X86::AddrIndexReg).getReg())
211             .addImm(Current->getOperand(MemOpOffset + X86::AddrDisp).getImm() +
212                     Delta)
213             .addReg(Current->getOperand(MemOpOffset + X86::AddrSegmentReg)
214                         .getReg());
215 
216         if (!Current->memoperands_empty()) {
217           MachineMemOperand *CurrentOp = *(Current->memoperands_begin());
218           MIB.addMemOperand(MF.getMachineMemOperand(
219               CurrentOp, CurrentOp->getOffset() + Delta, CurrentOp->getSize()));
220         }
221 
222         // Insert before Current. This is because Current may clobber some of
223         // the registers used to describe the input memory operand.
224         MBB.insert(Current, PFetch);
225         Changed = true;
226       }
227     }
228   }
229   return Changed;
230 }
231 
232 FunctionPass *llvm::createX86InsertPrefetchPass() {
233   return new X86InsertPrefetch(PrefetchHintsFile);
234 }
235