1 //===-- MVEVPTBlockPass.cpp - Insert MVE VPT blocks -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ARM.h"
10 #include "ARMMachineFunctionInfo.h"
11 #include "ARMSubtarget.h"
12 #include "MCTargetDesc/ARMBaseInfo.h"
13 #include "Thumb2InstrInfo.h"
14 #include "llvm/ADT/SmallSet.h"
15 #include "llvm/ADT/SmallVector.h"
16 #include "llvm/ADT/Statistic.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/CodeGen/MachineBasicBlock.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/CodeGen/MachineInstr.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineInstrBundle.h"
24 #include "llvm/CodeGen/MachineOperand.h"
25 #include "llvm/IR/DebugLoc.h"
26 #include "llvm/MC/MCInstrDesc.h"
27 #include "llvm/MC/MCRegisterInfo.h"
28 #include "llvm/Support/Debug.h"
29 #include <cassert>
30 #include <new>
31 
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "arm-mve-vpt"
35 
36 namespace {
37 class MVEVPTBlock : public MachineFunctionPass {
38 public:
39   static char ID;
40   const Thumb2InstrInfo *TII;
41   const TargetRegisterInfo *TRI;
42 
43   MVEVPTBlock() : MachineFunctionPass(ID) {}
44 
45   bool runOnMachineFunction(MachineFunction &Fn) override;
46 
47   MachineFunctionProperties getRequiredProperties() const override {
48     return MachineFunctionProperties().set(
49         MachineFunctionProperties::Property::NoVRegs);
50   }
51 
52   StringRef getPassName() const override {
53     return "MVE VPT block insertion pass";
54   }
55 
56 private:
57   bool InsertVPTBlocks(MachineBasicBlock &MBB);
58 };
59 
60 char MVEVPTBlock::ID = 0;
61 
62 } // end anonymous namespace
63 
64 INITIALIZE_PASS(MVEVPTBlock, DEBUG_TYPE, "ARM MVE VPT block pass", false, false)
65 
66 static MachineInstr *findVCMPToFoldIntoVPST(MachineBasicBlock::iterator MI,
67                                             const TargetRegisterInfo *TRI,
68                                             unsigned &NewOpcode) {
69   // Search backwards to the instruction that defines VPR. This may or not
70   // be a VCMP, we check that after this loop. If we find another instruction
71   // that reads cpsr, we return nullptr.
72   MachineBasicBlock::iterator CmpMI = MI;
73   while (CmpMI != MI->getParent()->begin()) {
74     --CmpMI;
75     if (CmpMI->modifiesRegister(ARM::VPR, TRI))
76       break;
77     if (CmpMI->readsRegister(ARM::VPR, TRI))
78       break;
79   }
80 
81   if (CmpMI == MI)
82     return nullptr;
83   NewOpcode = VCMPOpcodeToVPT(CmpMI->getOpcode());
84   if (NewOpcode == 0)
85     return nullptr;
86 
87   // Search forward from CmpMI to MI, checking if either register was def'd
88   if (registerDefinedBetween(CmpMI->getOperand(1).getReg(), std::next(CmpMI),
89                              MI, TRI))
90     return nullptr;
91   if (registerDefinedBetween(CmpMI->getOperand(2).getReg(), std::next(CmpMI),
92                              MI, TRI))
93     return nullptr;
94   return &*CmpMI;
95 }
96 
97 static ARM::PredBlockMask ExpandBlockMask(ARM::PredBlockMask BlockMask,
98                                           ARMVCC::VPTCodes Kind) {
99   using PredBlockMask = ARM::PredBlockMask;
100   assert(Kind != ARMVCC::None && "Cannot expand mask with 'None'");
101   assert(countTrailingZeros((unsigned)BlockMask) != 0 &&
102          "Mask is already full");
103 
104   auto ChooseMask = [&](PredBlockMask AddedThen, PredBlockMask AddedElse) {
105     return (Kind == ARMVCC::Then) ? AddedThen : AddedElse;
106   };
107 
108   switch (BlockMask) {
109   case PredBlockMask::T:
110     return ChooseMask(PredBlockMask::TT, PredBlockMask::TE);
111   case PredBlockMask::TT:
112     return ChooseMask(PredBlockMask::TTT, PredBlockMask::TTE);
113   case PredBlockMask::TE:
114     return ChooseMask(PredBlockMask::TET, PredBlockMask::TEE);
115   case PredBlockMask::TTT:
116     return ChooseMask(PredBlockMask::TTTT, PredBlockMask::TTTE);
117   case PredBlockMask::TTE:
118     return ChooseMask(PredBlockMask::TTET, PredBlockMask::TTEE);
119   case PredBlockMask::TET:
120     return ChooseMask(PredBlockMask::TETT, PredBlockMask::TETE);
121   case PredBlockMask::TEE:
122     return ChooseMask(PredBlockMask::TEET, PredBlockMask::TEEE);
123   default:
124     llvm_unreachable("Unknown Mask");
125   }
126 }
127 
128 // Advances Iter past a block of predicated instructions.
129 // Returns true if it successfully skipped the whole block of predicated
130 // instructions. Returns false when it stopped early (due to MaxSteps), or if
131 // Iter didn't point to a predicated instruction.
132 static bool StepOverPredicatedInstrs(MachineBasicBlock::instr_iterator &Iter,
133                                      MachineBasicBlock::instr_iterator EndIter,
134                                      unsigned MaxSteps,
135                                      unsigned &NumInstrsSteppedOver) {
136   ARMVCC::VPTCodes NextPred = ARMVCC::None;
137   unsigned PredReg;
138   NumInstrsSteppedOver = 0;
139 
140   while (Iter != EndIter) {
141     NextPred = getVPTInstrPredicate(*Iter, PredReg);
142     assert(NextPred != ARMVCC::Else &&
143            "VPT block pass does not expect Else preds");
144     if (NextPred == ARMVCC::None || MaxSteps == 0)
145       break;
146     --MaxSteps;
147     ++Iter;
148     ++NumInstrsSteppedOver;
149   };
150 
151   return NumInstrsSteppedOver != 0 &&
152          (NextPred == ARMVCC::None || Iter == EndIter);
153 }
154 
155 // Returns true if at least one instruction in the range [Iter, End) defines
156 // or kills VPR.
157 static bool IsVPRDefinedOrKilledByBlock(MachineBasicBlock::iterator Iter,
158                                         MachineBasicBlock::iterator End) {
159   for (; Iter != End; ++Iter)
160     if (Iter->definesRegister(ARM::VPR) || Iter->killsRegister(ARM::VPR))
161       return true;
162   return false;
163 }
164 
165 // Given an iterator (Iter) that points at an instruction with a "Then"
166 // predicate, tries to create the largest block of continuous predicated
167 // instructions possible, and returns the VPT Block Mask of that block.
168 //
169 // This will try to perform some minor optimization in order to maximize the
170 // size of the block.
171 static ARM::PredBlockMask
172 CreateVPTBlock(MachineBasicBlock::instr_iterator &Iter,
173                MachineBasicBlock::instr_iterator EndIter,
174                SmallVectorImpl<MachineInstr *> &DeadInstructions) {
175   MachineBasicBlock::instr_iterator BlockBeg = Iter;
176   (void)BlockBeg;
177   assert(getVPTInstrPredicate(*Iter) == ARMVCC::Then &&
178          "Expected a Predicated Instruction");
179 
180   LLVM_DEBUG(dbgs() << "VPT block created for: "; Iter->dump());
181 
182   unsigned BlockSize;
183   StepOverPredicatedInstrs(Iter, EndIter, 4, BlockSize);
184 
185   LLVM_DEBUG(for (MachineBasicBlock::instr_iterator AddedInstIter =
186                       std::next(BlockBeg);
187                   AddedInstIter != Iter; ++AddedInstIter) {
188     dbgs() << "  adding: ";
189     AddedInstIter->dump();
190   });
191 
192   // Generate the initial BlockMask
193   ARM::PredBlockMask BlockMask = getARMVPTBlockMask(BlockSize);
194 
195   // Remove VPNOTs while there's still room in the block, so we can make the
196   // largest block possible.
197   ARMVCC::VPTCodes CurrentPredicate = ARMVCC::Then;
198   while (BlockSize < 4 && Iter != EndIter &&
199          Iter->getOpcode() == ARM::MVE_VPNOT) {
200 
201     // Try to skip all of the predicated instructions after the VPNOT, stopping
202     // after (4 - BlockSize). If we can't skip them all, stop.
203     unsigned ElseInstCnt = 0;
204     MachineBasicBlock::instr_iterator VPNOTBlockEndIter = std::next(Iter);
205     if (!StepOverPredicatedInstrs(VPNOTBlockEndIter, EndIter, (4 - BlockSize),
206                                   ElseInstCnt))
207       break;
208 
209     // Check if this VPNOT can be removed or not: It can only be removed if at
210     // least one of the predicated instruction that follows it kills or sets
211     // VPR.
212     if (!IsVPRDefinedOrKilledByBlock(Iter, VPNOTBlockEndIter))
213       break;
214 
215     LLVM_DEBUG(dbgs() << "  removing VPNOT: "; Iter->dump(););
216 
217     // Record the new size of the block
218     BlockSize += ElseInstCnt;
219     assert(BlockSize <= 4 && "Block is too large!");
220 
221     // Record the VPNot to remove it later.
222     DeadInstructions.push_back(&*Iter);
223     ++Iter;
224 
225     // Replace "then" by "elses" in the block until we find an instruction that
226     // defines VPR, then after that leave everything to "t".
227     // Note that we are using "Iter" to iterate over the block so we can update
228     // it at the same time.
229     bool ChangeToElse = (CurrentPredicate == ARMVCC::Then);
230     for (; Iter != VPNOTBlockEndIter; ++Iter) {
231       // Find the register in which the predicate is
232       int OpIdx = findFirstVPTPredOperandIdx(*Iter);
233       assert(OpIdx != -1);
234 
235       // Update the mask + change the predicate to an else if needed.
236       if (ChangeToElse) {
237         // Change the predicate and update the mask
238         Iter->getOperand(OpIdx).setImm(ARMVCC::Else);
239         BlockMask = ExpandBlockMask(BlockMask, ARMVCC::Else);
240         // Reset back to a "then" predicate if this instruction defines VPR.
241         if (Iter->definesRegister(ARM::VPR))
242           ChangeToElse = false;
243       } else
244         BlockMask = ExpandBlockMask(BlockMask, ARMVCC::Then);
245 
246       LLVM_DEBUG(dbgs() << "  adding: "; Iter->dump());
247     }
248 
249     CurrentPredicate =
250         (CurrentPredicate == ARMVCC::Then ? ARMVCC::Else : ARMVCC::Then);
251   }
252   return BlockMask;
253 }
254 
255 bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
256   bool Modified = false;
257   MachineBasicBlock::instr_iterator MBIter = Block.instr_begin();
258   MachineBasicBlock::instr_iterator EndIter = Block.instr_end();
259 
260   SmallVector<MachineInstr *, 4> DeadInstructions;
261 
262   while (MBIter != EndIter) {
263     MachineInstr *MI = &*MBIter;
264     unsigned PredReg = 0;
265     DebugLoc DL = MI->getDebugLoc();
266 
267     ARMVCC::VPTCodes Pred = getVPTInstrPredicate(*MI, PredReg);
268 
269     // The idea of the predicate is that None, Then and Else are for use when
270     // handling assembly language: they correspond to the three possible
271     // suffixes "", "t" and "e" on the mnemonic. So when instructions are read
272     // from assembly source or disassembled from object code, you expect to
273     // see a mixture whenever there's a long VPT block. But in code
274     // generation, we hope we'll never generate an Else as input to this pass.
275     assert(Pred != ARMVCC::Else && "VPT block pass does not expect Else preds");
276 
277     if (Pred == ARMVCC::None) {
278       ++MBIter;
279       continue;
280     }
281 
282     ARM::PredBlockMask BlockMask =
283         CreateVPTBlock(MBIter, EndIter, DeadInstructions);
284 
285     // Search back for a VCMP that can be folded to create a VPT, or else
286     // create a VPST directly
287     MachineInstrBuilder MIBuilder;
288     unsigned NewOpcode;
289     LLVM_DEBUG(dbgs() << "  final block mask: " << (unsigned)BlockMask << "\n");
290     if (MachineInstr *VCMP = findVCMPToFoldIntoVPST(MI, TRI, NewOpcode)) {
291       LLVM_DEBUG(dbgs() << "  folding VCMP into VPST: "; VCMP->dump());
292       MIBuilder = BuildMI(Block, MI, DL, TII->get(NewOpcode));
293       MIBuilder.addImm((uint64_t)BlockMask);
294       MIBuilder.add(VCMP->getOperand(1));
295       MIBuilder.add(VCMP->getOperand(2));
296       MIBuilder.add(VCMP->getOperand(3));
297       VCMP->eraseFromParent();
298     } else {
299       MIBuilder = BuildMI(Block, MI, DL, TII->get(ARM::MVE_VPST));
300       MIBuilder.addImm((uint64_t)BlockMask);
301     }
302 
303     finalizeBundle(
304         Block, MachineBasicBlock::instr_iterator(MIBuilder.getInstr()), MBIter);
305 
306     Modified = true;
307   }
308 
309   // Erase all dead instructions
310   for (MachineInstr *DeadMI : DeadInstructions) {
311     if (DeadMI->isInsideBundle())
312       DeadMI->eraseFromBundle();
313     else
314       DeadMI->eraseFromParent();
315   }
316 
317   return Modified;
318 }
319 
320 bool MVEVPTBlock::runOnMachineFunction(MachineFunction &Fn) {
321   const ARMSubtarget &STI =
322       static_cast<const ARMSubtarget &>(Fn.getSubtarget());
323 
324   if (!STI.isThumb2() || !STI.hasMVEIntegerOps())
325     return false;
326 
327   TII = static_cast<const Thumb2InstrInfo *>(STI.getInstrInfo());
328   TRI = STI.getRegisterInfo();
329 
330   LLVM_DEBUG(dbgs() << "********** ARM MVE VPT BLOCKS **********\n"
331                     << "********** Function: " << Fn.getName() << '\n');
332 
333   bool Modified = false;
334   for (MachineBasicBlock &MBB : Fn)
335     Modified |= InsertVPTBlocks(MBB);
336 
337   LLVM_DEBUG(dbgs() << "**************************************\n");
338   return Modified;
339 }
340 
341 /// createMVEVPTBlock - Returns an instance of the MVE VPT block
342 /// insertion pass.
343 FunctionPass *llvm::createMVEVPTBlockPass() { return new MVEVPTBlock(); }
344