1 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // This pass changes code to work around Cortex-A53 erratum 835769.
10 // It works around it by inserting a nop instruction in code sequences that
11 // in some circumstances may trigger the erratum.
12 // It inserts a nop instruction between a sequence of the following 2 classes
13 // of instructions:
14 // instr 1: mem-instr (including loads, stores and prefetches).
15 // instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
16 //===----------------------------------------------------------------------===//
17 
18 #include "AArch64.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/CodeGen/MachineFunction.h"
21 #include "llvm/CodeGen/MachineFunctionPass.h"
22 #include "llvm/CodeGen/MachineInstr.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
27 #include "llvm/Target/TargetInstrInfo.h"
28 
29 using namespace llvm;
30 
31 #define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"
32 
33 STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");
34 
35 //===----------------------------------------------------------------------===//
36 // Helper functions
37 
38 // Is the instruction a match for the instruction that comes first in the
39 // sequence of instructions that can trigger the erratum?
40 static bool isFirstInstructionInSequence(MachineInstr *MI) {
41   // Must return true if this instruction is a load, a store or a prefetch.
42   switch (MI->getOpcode()) {
43   case AArch64::PRFMl:
44   case AArch64::PRFMroW:
45   case AArch64::PRFMroX:
46   case AArch64::PRFMui:
47   case AArch64::PRFUMi:
48     return true;
49   default:
50     return MI->mayLoadOrStore();
51   }
52 }
53 
54 // Is the instruction a match for the instruction that comes second in the
55 // sequence that can trigger the erratum?
56 static bool isSecondInstructionInSequence(MachineInstr *MI) {
57   // Must return true for non-SIMD integer multiply-accumulates, writing
58   // to a 64-bit register.
59   switch (MI->getOpcode()) {
60   // Erratum cannot be triggered when the destination register is 32 bits,
61   // therefore only include the following.
62   case AArch64::MSUBXrrr:
63   case AArch64::MADDXrrr:
64   case AArch64::SMADDLrrr:
65   case AArch64::SMSUBLrrr:
66   case AArch64::UMADDLrrr:
67   case AArch64::UMSUBLrrr:
68     // Erratum can only be triggered by multiply-adds, not by regular
69     // non-accumulating multiplies, i.e. when Ra=XZR='11111'
70     return MI->getOperand(3).getReg() != AArch64::XZR;
71   default:
72     return false;
73   }
74 }
75 
76 
77 //===----------------------------------------------------------------------===//
78 
79 namespace {
80 class AArch64A53Fix835769 : public MachineFunctionPass {
81   const TargetInstrInfo *TII;
82 
83 public:
84   static char ID;
85   explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
86     initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
87   }
88 
89   bool runOnMachineFunction(MachineFunction &F) override;
90 
91   MachineFunctionProperties getRequiredProperties() const override {
92     return MachineFunctionProperties().set(
93         MachineFunctionProperties::Property::NoVRegs);
94   }
95 
96   StringRef getPassName() const override {
97     return "Workaround A53 erratum 835769 pass";
98   }
99 
100   void getAnalysisUsage(AnalysisUsage &AU) const override {
101     AU.setPreservesCFG();
102     MachineFunctionPass::getAnalysisUsage(AU);
103   }
104 
105 private:
106   bool runOnBasicBlock(MachineBasicBlock &MBB);
107 };
108 char AArch64A53Fix835769::ID = 0;
109 
110 } // end anonymous namespace
111 
112 INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
113                 "AArch64 fix for A53 erratum 835769", false, false)
114 
115 //===----------------------------------------------------------------------===//
116 
117 bool
118 AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
119   DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
120   bool Changed = false;
121   TII = F.getSubtarget().getInstrInfo();
122 
123   for (auto &MBB : F) {
124     Changed |= runOnBasicBlock(MBB);
125   }
126   return Changed;
127 }
128 
129 // Return the block that was fallen through to get to MBB, if any,
130 // otherwise nullptr.
131 static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
132                                              const TargetInstrInfo *TII) {
133   // Get the previous machine basic block in the function.
134   MachineFunction::iterator MBBI(MBB);
135 
136   // Can't go off top of function.
137   if (MBBI == MBB->getParent()->begin())
138     return nullptr;
139 
140   MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
141   SmallVector<MachineOperand, 2> Cond;
142 
143   MachineBasicBlock *PrevBB = &*std::prev(MBBI);
144   for (MachineBasicBlock *S : MBB->predecessors())
145     if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
146         !FBB)
147       return S;
148 
149   return nullptr;
150 }
151 
152 // Iterate through fallen through blocks trying to find a previous non-pseudo if
153 // there is one, otherwise return nullptr. Only look for instructions in
154 // previous blocks, not the current block, since we only use this to look at
155 // previous blocks.
156 static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
157                                       const TargetInstrInfo *TII) {
158   MachineBasicBlock *FMBB = &MBB;
159 
160   // If there is no non-pseudo in the current block, loop back around and try
161   // the previous block (if there is one).
162   while ((FMBB = getBBFallenThrough(FMBB, TII))) {
163     for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
164       if (!I.isPseudo())
165         return &I;
166   }
167 
168   // There was no previous non-pseudo in the fallen through blocks
169   return nullptr;
170 }
171 
172 static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
173                                        const TargetInstrInfo *TII) {
174   // If we are the first instruction of the block, put the NOP at the end of
175   // the previous fallthrough block
176   if (MI == &MBB.front()) {
177     MachineInstr *I = getLastNonPseudo(MBB, TII);
178     assert(I && "Expected instruction");
179     DebugLoc DL = I->getDebugLoc();
180     BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
181   }
182   else {
183     DebugLoc DL = MI->getDebugLoc();
184     BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
185   }
186 
187   ++NumNopsAdded;
188 }
189 
190 bool
191 AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
192   bool Changed = false;
193   DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n");
194 
195   // First, scan the basic block, looking for a sequence of 2 instructions
196   // that match the conditions under which the erratum may trigger.
197 
198   // List of terminating instructions in matching sequences
199   std::vector<MachineInstr*> Sequences;
200   unsigned Idx = 0;
201   MachineInstr *PrevInstr = nullptr;
202 
203   // Try and find the last non-pseudo instruction in any fallen through blocks,
204   // if there isn't one, then we use nullptr to represent that.
205   PrevInstr = getLastNonPseudo(MBB, TII);
206 
207   for (auto &MI : MBB) {
208     MachineInstr *CurrInstr = &MI;
209     DEBUG(dbgs() << "  Examining: " << MI);
210     if (PrevInstr) {
211       DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
212                    << "    CurrInstr: " << *CurrInstr
213                    << "    isFirstInstructionInSequence(PrevInstr): "
214                    << isFirstInstructionInSequence(PrevInstr) << "\n"
215                    << "    isSecondInstructionInSequence(CurrInstr): "
216                    << isSecondInstructionInSequence(CurrInstr) << "\n");
217       if (isFirstInstructionInSequence(PrevInstr) &&
218           isSecondInstructionInSequence(CurrInstr)) {
219         DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
220         Sequences.push_back(CurrInstr);
221       }
222     }
223     if (!CurrInstr->isPseudo())
224       PrevInstr = CurrInstr;
225     ++Idx;
226   }
227 
228   DEBUG(dbgs() << "Scan complete, " << Sequences.size()
229                << " occurrences of pattern found.\n");
230 
231   // Then update the basic block, inserting nops between the detected sequences.
232   for (auto &MI : Sequences) {
233     Changed = true;
234     insertNopBeforeInstruction(MBB, MI, TII);
235   }
236 
237   return Changed;
238 }
239 
240 // Factory function used by AArch64TargetMachine to add the pass to
241 // the passmanager.
242 FunctionPass *llvm::createAArch64A53Fix835769() {
243   return new AArch64A53Fix835769();
244 }
245