1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ----------=//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
11 // multiple and add / sub instructions) when special VMLx hazards are detected.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #define DEBUG_TYPE "mlx-expansion"
16 #include "ARM.h"
17 #include "ARMBaseInstrInfo.h"
18 #include "llvm/CodeGen/MachineInstr.h"
19 #include "llvm/CodeGen/MachineInstrBuilder.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/CodeGen/MachineRegisterInfo.h"
22 #include "llvm/Target/TargetRegisterInfo.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/Support/CommandLine.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/raw_ostream.h"
27 using namespace llvm;
28 
29 static cl::opt<bool>
30 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
31 static cl::opt<unsigned>
32 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
33 
34 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
35 
36 namespace {
37   struct MLxExpansion : public MachineFunctionPass {
38     static char ID;
39     MLxExpansion() : MachineFunctionPass(ID) {}
40 
41     virtual bool runOnMachineFunction(MachineFunction &Fn);
42 
43     virtual const char *getPassName() const {
44       return "ARM MLA / MLS expansion pass";
45     }
46 
47   private:
48     const ARMBaseInstrInfo *TII;
49     const TargetRegisterInfo *TRI;
50     MachineRegisterInfo *MRI;
51 
52     unsigned MIIdx;
53     MachineInstr* LastMIs[4];
54 
55     void clearStack();
56     void pushStack(MachineInstr *MI);
57     MachineInstr *getAccDefMI(MachineInstr *MI) const;
58     unsigned getDefReg(MachineInstr *MI) const;
59     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
60     bool FindMLxHazard(MachineInstr *MI) const;
61     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
62                                 unsigned MulOpc, unsigned AddSubOpc,
63                                 bool NegAcc, bool HasLane);
64     bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
65   };
66   char MLxExpansion::ID = 0;
67 }
68 
69 void MLxExpansion::clearStack() {
70   std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
71   MIIdx = 0;
72 }
73 
74 void MLxExpansion::pushStack(MachineInstr *MI) {
75   LastMIs[MIIdx] = MI;
76   if (++MIIdx == 4)
77     MIIdx = 0;
78 }
79 
80 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
81   // Look past COPY and INSERT_SUBREG instructions to find the
82   // real definition MI. This is important for _sfp instructions.
83   unsigned Reg = MI->getOperand(1).getReg();
84   if (TargetRegisterInfo::isPhysicalRegister(Reg))
85     return 0;
86 
87   MachineBasicBlock *MBB = MI->getParent();
88   MachineInstr *DefMI = MRI->getVRegDef(Reg);
89   while (true) {
90     if (DefMI->getParent() != MBB)
91       break;
92     if (DefMI->isCopyLike()) {
93       Reg = DefMI->getOperand(1).getReg();
94       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
95         DefMI = MRI->getVRegDef(Reg);
96         continue;
97       }
98     } else if (DefMI->isInsertSubreg()) {
99       Reg = DefMI->getOperand(2).getReg();
100       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
101         DefMI = MRI->getVRegDef(Reg);
102         continue;
103       }
104     }
105     break;
106   }
107   return DefMI;
108 }
109 
110 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
111   unsigned Reg = MI->getOperand(0).getReg();
112   if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
113       !MRI->hasOneNonDBGUse(Reg))
114     return Reg;
115 
116   MachineBasicBlock *MBB = MI->getParent();
117   MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg);
118   if (UseMI->getParent() != MBB)
119     return Reg;
120 
121   while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
122     Reg = UseMI->getOperand(0).getReg();
123     if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
124         !MRI->hasOneNonDBGUse(Reg))
125       return Reg;
126     UseMI = &*MRI->use_nodbg_begin(Reg);
127     if (UseMI->getParent() != MBB)
128       return Reg;
129   }
130 
131   return Reg;
132 }
133 
134 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
135   // FIXME: Detect integer instructions properly.
136   const TargetInstrDesc &TID = MI->getDesc();
137   unsigned Domain = TID.TSFlags & ARMII::DomainMask;
138   if (TID.mayStore())
139     return false;
140   unsigned Opcode = TID.getOpcode();
141   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
142     return false;
143   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
144     return MI->readsRegister(Reg, TRI);
145   return false;
146 }
147 
148 
149 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) const {
150   if (NumExpand >= ExpandLimit)
151     return false;
152 
153   if (ForceExapnd)
154     return true;
155 
156   MachineInstr *DefMI = getAccDefMI(MI);
157   if (TII->isFpMLxInstruction(DefMI->getOpcode()))
158     // r0 = vmla
159     // r3 = vmla r0, r1, r2
160     // takes 16 - 17 cycles
161     //
162     // r0 = vmla
163     // r4 = vmul r1, r2
164     // r3 = vadd r0, r4
165     // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
166     return true;
167 
168   // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
169   // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
170   // preserves the in-order retirement of the instructions.
171   // Look at the next few instructions, if *most* of them can cause hazards,
172   // then the scheduler can't *fix* this, we'd better break up the VMLA.
173   for (unsigned i = 1; i <= 4; ++i) {
174     int Idx = ((int)MIIdx - i + 4) % 4;
175     MachineInstr *NextMI = LastMIs[Idx];
176     if (!NextMI)
177       continue;
178 
179     if (TII->canCauseFpMLxStall(NextMI->getOpcode()))
180       return true;
181 
182     // Look for VMLx RAW hazard.
183     if (hasRAWHazard(getDefReg(MI), NextMI))
184       return true;
185   }
186 
187   return false;
188 }
189 
190 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
191 /// of MUL + ADD / SUB instructions.
192 void
193 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
194                                      unsigned MulOpc, unsigned AddSubOpc,
195                                      bool NegAcc, bool HasLane) {
196   unsigned DstReg = MI->getOperand(0).getReg();
197   bool DstDead = MI->getOperand(0).isDead();
198   unsigned AccReg = MI->getOperand(1).getReg();
199   unsigned Src1Reg = MI->getOperand(2).getReg();
200   unsigned Src2Reg = MI->getOperand(3).getReg();
201   bool Src1Kill = MI->getOperand(2).isKill();
202   bool Src2Kill = MI->getOperand(3).isKill();
203   unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
204   unsigned NextOp = HasLane ? 5 : 4;
205   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
206   unsigned PredReg = MI->getOperand(++NextOp).getReg();
207 
208   const TargetInstrDesc &TID1 = TII->get(MulOpc);
209   const TargetInstrDesc &TID2 = TII->get(AddSubOpc);
210   unsigned TmpReg = MRI->createVirtualRegister(TID1.getRegClass(0, TRI));
211 
212   MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID1, TmpReg)
213     .addReg(Src1Reg, getKillRegState(Src1Kill))
214     .addReg(Src2Reg, getKillRegState(Src2Kill));
215   if (HasLane)
216     MIB.addImm(LaneImm);
217   MIB.addImm(Pred).addReg(PredReg);
218 
219   MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), TID2)
220     .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
221 
222   if (NegAcc) {
223     bool AccKill = MRI->hasOneNonDBGUse(AccReg);
224     MIB.addReg(TmpReg, getKillRegState(true))
225        .addReg(AccReg, getKillRegState(AccKill));
226   } else {
227     MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
228   }
229   MIB.addImm(Pred).addReg(PredReg);
230 
231   DEBUG({
232       dbgs() << "Expanding: " << *MI;
233       dbgs() << "  to:\n";
234       MachineBasicBlock::iterator MII = MI;
235       MII = llvm::prior(MII);
236       MachineInstr &MI2 = *MII;
237       MII = llvm::prior(MII);
238       MachineInstr &MI1 = *MII;
239       dbgs() << "    " << MI1;
240       dbgs() << "    " << MI2;
241    });
242 
243   MI->eraseFromParent();
244   ++NumExpand;
245 }
246 
247 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
248   bool Changed = false;
249 
250   clearStack();
251 
252   unsigned Skip = 0;
253   MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
254   while (MII != E) {
255     MachineInstr *MI = &*MII;
256 
257     if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) {
258       ++MII;
259       continue;
260     }
261 
262     const TargetInstrDesc &TID = MI->getDesc();
263     if (TID.isBarrier()) {
264       clearStack();
265       Skip = 0;
266       ++MII;
267       continue;
268     }
269 
270     unsigned Domain = TID.TSFlags & ARMII::DomainMask;
271     if (Domain == ARMII::DomainGeneral) {
272       if (++Skip == 2)
273         // Assume dual issues of non-VFP / NEON instructions.
274         pushStack(0);
275     } else {
276       Skip = 0;
277 
278       unsigned MulOpc, AddSubOpc;
279       bool NegAcc, HasLane;
280       if (!TII->isFpMLxInstruction(TID.getOpcode(),
281                                    MulOpc, AddSubOpc, NegAcc, HasLane) ||
282           !FindMLxHazard(MI))
283         pushStack(MI);
284       else {
285         ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
286         E = MBB.rend(); // May have changed if MI was the 1st instruction.
287         Changed = true;
288         continue;
289       }
290     }
291 
292     ++MII;
293   }
294 
295   return Changed;
296 }
297 
298 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
299   TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
300   TRI = Fn.getTarget().getRegisterInfo();
301   MRI = &Fn.getRegInfo();
302 
303   bool Modified = false;
304   for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
305        ++MFI) {
306     MachineBasicBlock &MBB = *MFI;
307     Modified |= ExpandFPMLxInstructions(MBB);
308   }
309 
310   return Modified;
311 }
312 
313 FunctionPass *llvm::createMLxExpansionPass() {
314   return new MLxExpansion();
315 }
316