1 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the RISCV implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVInstrInfo.h"
14 #include "MCTargetDesc/RISCVMatInt.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVSubtarget.h"
18 #include "RISCVTargetMachine.h"
19 #include "llvm/ADT/STLExtras.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/Analysis/MemoryLocation.h"
22 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #include "llvm/CodeGen/MachineInstrBuilder.h"
24 #include "llvm/CodeGen/MachineRegisterInfo.h"
25 #include "llvm/CodeGen/RegisterScavenging.h"
26 #include "llvm/Support/ErrorHandling.h"
27 #include "llvm/Support/TargetRegistry.h"
28 
29 using namespace llvm;
30 
31 #define GEN_CHECK_COMPRESS_INSTR
32 #include "RISCVGenCompressInstEmitter.inc"
33 
34 #define GET_INSTRINFO_CTOR_DTOR
35 #include "RISCVGenInstrInfo.inc"
36 
37 namespace llvm {
38 namespace RISCVVPseudosTable {
39 
40 using namespace RISCV;
41 
42 #define GET_RISCVVPseudosTable_IMPL
43 #include "RISCVGenSearchableTables.inc"
44 
45 } // namespace RISCVVPseudosTable
46 } // namespace llvm
47 
48 RISCVInstrInfo::RISCVInstrInfo(RISCVSubtarget &STI)
49     : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
50       STI(STI) {}
51 
52 unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
53                                              int &FrameIndex) const {
54   switch (MI.getOpcode()) {
55   default:
56     return 0;
57   case RISCV::LB:
58   case RISCV::LBU:
59   case RISCV::LH:
60   case RISCV::LHU:
61   case RISCV::FLH:
62   case RISCV::LW:
63   case RISCV::FLW:
64   case RISCV::LWU:
65   case RISCV::LD:
66   case RISCV::FLD:
67     break;
68   }
69 
70   if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
71       MI.getOperand(2).getImm() == 0) {
72     FrameIndex = MI.getOperand(1).getIndex();
73     return MI.getOperand(0).getReg();
74   }
75 
76   return 0;
77 }
78 
79 unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
80                                             int &FrameIndex) const {
81   switch (MI.getOpcode()) {
82   default:
83     return 0;
84   case RISCV::SB:
85   case RISCV::SH:
86   case RISCV::SW:
87   case RISCV::FSH:
88   case RISCV::FSW:
89   case RISCV::SD:
90   case RISCV::FSD:
91     break;
92   }
93 
94   if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
95       MI.getOperand(2).getImm() == 0) {
96     FrameIndex = MI.getOperand(1).getIndex();
97     return MI.getOperand(0).getReg();
98   }
99 
100   return 0;
101 }
102 
103 void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
104                                  MachineBasicBlock::iterator MBBI,
105                                  const DebugLoc &DL, MCRegister DstReg,
106                                  MCRegister SrcReg, bool KillSrc) const {
107   if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
108     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
109         .addReg(SrcReg, getKillRegState(KillSrc))
110         .addImm(0);
111     return;
112   }
113 
114   // FPR->FPR copies and VR->VR copies.
115   unsigned Opc;
116   bool IsScalableVector = false;
117   if (RISCV::FPR16RegClass.contains(DstReg, SrcReg))
118     Opc = RISCV::FSGNJ_H;
119   else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg))
120     Opc = RISCV::FSGNJ_S;
121   else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg))
122     Opc = RISCV::FSGNJ_D;
123   else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) {
124     Opc = RISCV::PseudoVMV1R_V;
125     IsScalableVector = true;
126   } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) {
127     Opc = RISCV::PseudoVMV2R_V;
128     IsScalableVector = true;
129   } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) {
130     Opc = RISCV::PseudoVMV4R_V;
131     IsScalableVector = true;
132   } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) {
133     Opc = RISCV::PseudoVMV8R_V;
134     IsScalableVector = true;
135   } else
136     llvm_unreachable("Impossible reg-to-reg copy");
137 
138   if (IsScalableVector)
139     BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
140         .addReg(SrcReg, getKillRegState(KillSrc));
141   else
142     BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
143         .addReg(SrcReg, getKillRegState(KillSrc))
144         .addReg(SrcReg, getKillRegState(KillSrc));
145 }
146 
147 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
148                                          MachineBasicBlock::iterator I,
149                                          Register SrcReg, bool IsKill, int FI,
150                                          const TargetRegisterClass *RC,
151                                          const TargetRegisterInfo *TRI) const {
152   DebugLoc DL;
153   if (I != MBB.end())
154     DL = I->getDebugLoc();
155 
156   MachineFunction *MF = MBB.getParent();
157   MachineFrameInfo &MFI = MF->getFrameInfo();
158 
159   unsigned Opcode;
160   bool IsScalableVector = false;
161   if (RISCV::GPRRegClass.hasSubClassEq(RC))
162     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
163              RISCV::SW : RISCV::SD;
164   else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
165     Opcode = RISCV::FSH;
166   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
167     Opcode = RISCV::FSW;
168   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
169     Opcode = RISCV::FSD;
170   else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
171     Opcode = RISCV::PseudoVSPILL_M1;
172     IsScalableVector = true;
173   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
174     Opcode = RISCV::PseudoVSPILL_M2;
175     IsScalableVector = true;
176   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
177     Opcode = RISCV::PseudoVSPILL_M4;
178     IsScalableVector = true;
179   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
180     Opcode = RISCV::PseudoVSPILL_M8;
181     IsScalableVector = true;
182   } else
183     llvm_unreachable("Can't store this register to stack slot");
184 
185   if (IsScalableVector) {
186     MachineMemOperand *MMO = MF->getMachineMemOperand(
187         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
188         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
189 
190     MFI.setStackID(FI, TargetStackID::ScalableVector);
191     BuildMI(MBB, I, DL, get(Opcode))
192         .addReg(SrcReg, getKillRegState(IsKill))
193         .addFrameIndex(FI)
194         .addMemOperand(MMO);
195   } else {
196     MachineMemOperand *MMO = MF->getMachineMemOperand(
197         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,
198         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
199 
200     BuildMI(MBB, I, DL, get(Opcode))
201         .addReg(SrcReg, getKillRegState(IsKill))
202         .addFrameIndex(FI)
203         .addImm(0)
204         .addMemOperand(MMO);
205   }
206 }
207 
208 void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
209                                           MachineBasicBlock::iterator I,
210                                           Register DstReg, int FI,
211                                           const TargetRegisterClass *RC,
212                                           const TargetRegisterInfo *TRI) const {
213   DebugLoc DL;
214   if (I != MBB.end())
215     DL = I->getDebugLoc();
216 
217   MachineFunction *MF = MBB.getParent();
218   MachineFrameInfo &MFI = MF->getFrameInfo();
219 
220   unsigned Opcode;
221   bool IsScalableVector = false;
222   if (RISCV::GPRRegClass.hasSubClassEq(RC))
223     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
224              RISCV::LW : RISCV::LD;
225   else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
226     Opcode = RISCV::FLH;
227   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
228     Opcode = RISCV::FLW;
229   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
230     Opcode = RISCV::FLD;
231   else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
232     Opcode = RISCV::PseudoVRELOAD_M1;
233     IsScalableVector = true;
234   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
235     Opcode = RISCV::PseudoVRELOAD_M2;
236     IsScalableVector = true;
237   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
238     Opcode = RISCV::PseudoVRELOAD_M4;
239     IsScalableVector = true;
240   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
241     Opcode = RISCV::PseudoVRELOAD_M8;
242     IsScalableVector = true;
243   } else
244     llvm_unreachable("Can't load this register from stack slot");
245 
246   if (IsScalableVector) {
247     MachineMemOperand *MMO = MF->getMachineMemOperand(
248         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
249         MemoryLocation::UnknownSize, MFI.getObjectAlign(FI));
250 
251     MFI.setStackID(FI, TargetStackID::ScalableVector);
252     BuildMI(MBB, I, DL, get(Opcode), DstReg)
253         .addFrameIndex(FI)
254         .addMemOperand(MMO);
255   } else {
256     MachineMemOperand *MMO = MF->getMachineMemOperand(
257         MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,
258         MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
259 
260     BuildMI(MBB, I, DL, get(Opcode), DstReg)
261         .addFrameIndex(FI)
262         .addImm(0)
263         .addMemOperand(MMO);
264   }
265 }
266 
267 void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
268                             MachineBasicBlock::iterator MBBI,
269                             const DebugLoc &DL, Register DstReg, uint64_t Val,
270                             MachineInstr::MIFlag Flag) const {
271   MachineFunction *MF = MBB.getParent();
272   MachineRegisterInfo &MRI = MF->getRegInfo();
273   bool IsRV64 = MF->getSubtarget<RISCVSubtarget>().is64Bit();
274   Register SrcReg = RISCV::X0;
275   Register Result = MRI.createVirtualRegister(&RISCV::GPRRegClass);
276   unsigned Num = 0;
277 
278   if (!IsRV64 && !isInt<32>(Val))
279     report_fatal_error("Should only materialize 32-bit constants for RV32");
280 
281   RISCVMatInt::InstSeq Seq;
282   RISCVMatInt::generateInstSeq(Val, IsRV64, Seq);
283   assert(Seq.size() > 0);
284 
285   for (RISCVMatInt::Inst &Inst : Seq) {
286     // Write the final result to DstReg if it's the last instruction in the Seq.
287     // Otherwise, write the result to the temp register.
288     if (++Num == Seq.size())
289       Result = DstReg;
290 
291     if (Inst.Opc == RISCV::LUI) {
292       BuildMI(MBB, MBBI, DL, get(RISCV::LUI), Result)
293           .addImm(Inst.Imm)
294           .setMIFlag(Flag);
295     } else {
296       BuildMI(MBB, MBBI, DL, get(Inst.Opc), Result)
297           .addReg(SrcReg, RegState::Kill)
298           .addImm(Inst.Imm)
299           .setMIFlag(Flag);
300     }
301     // Only the first instruction has X0 as its source.
302     SrcReg = Result;
303   }
304 }
305 
306 // The contents of values added to Cond are not examined outside of
307 // RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
308 // push BranchOpcode, Reg1, Reg2.
309 static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
310                             SmallVectorImpl<MachineOperand> &Cond) {
311   // Block ends with fall-through condbranch.
312   assert(LastInst.getDesc().isConditionalBranch() &&
313          "Unknown conditional branch");
314   Target = LastInst.getOperand(2).getMBB();
315   Cond.push_back(MachineOperand::CreateImm(LastInst.getOpcode()));
316   Cond.push_back(LastInst.getOperand(0));
317   Cond.push_back(LastInst.getOperand(1));
318 }
319 
320 static unsigned getOppositeBranchOpcode(int Opc) {
321   switch (Opc) {
322   default:
323     llvm_unreachable("Unrecognized conditional branch");
324   case RISCV::BEQ:
325     return RISCV::BNE;
326   case RISCV::BNE:
327     return RISCV::BEQ;
328   case RISCV::BLT:
329     return RISCV::BGE;
330   case RISCV::BGE:
331     return RISCV::BLT;
332   case RISCV::BLTU:
333     return RISCV::BGEU;
334   case RISCV::BGEU:
335     return RISCV::BLTU;
336   }
337 }
338 
339 bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
340                                    MachineBasicBlock *&TBB,
341                                    MachineBasicBlock *&FBB,
342                                    SmallVectorImpl<MachineOperand> &Cond,
343                                    bool AllowModify) const {
344   TBB = FBB = nullptr;
345   Cond.clear();
346 
347   // If the block has no terminators, it just falls into the block after it.
348   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
349   if (I == MBB.end() || !isUnpredicatedTerminator(*I))
350     return false;
351 
352   // Count the number of terminators and find the first unconditional or
353   // indirect branch.
354   MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
355   int NumTerminators = 0;
356   for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
357        J++) {
358     NumTerminators++;
359     if (J->getDesc().isUnconditionalBranch() ||
360         J->getDesc().isIndirectBranch()) {
361       FirstUncondOrIndirectBr = J.getReverse();
362     }
363   }
364 
365   // If AllowModify is true, we can erase any terminators after
366   // FirstUncondOrIndirectBR.
367   if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
368     while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
369       std::next(FirstUncondOrIndirectBr)->eraseFromParent();
370       NumTerminators--;
371     }
372     I = FirstUncondOrIndirectBr;
373   }
374 
375   // We can't handle blocks that end in an indirect branch.
376   if (I->getDesc().isIndirectBranch())
377     return true;
378 
379   // We can't handle blocks with more than 2 terminators.
380   if (NumTerminators > 2)
381     return true;
382 
383   // Handle a single unconditional branch.
384   if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
385     TBB = getBranchDestBlock(*I);
386     return false;
387   }
388 
389   // Handle a single conditional branch.
390   if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
391     parseCondBranch(*I, TBB, Cond);
392     return false;
393   }
394 
395   // Handle a conditional branch followed by an unconditional branch.
396   if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
397       I->getDesc().isUnconditionalBranch()) {
398     parseCondBranch(*std::prev(I), TBB, Cond);
399     FBB = getBranchDestBlock(*I);
400     return false;
401   }
402 
403   // Otherwise, we can't handle this.
404   return true;
405 }
406 
407 unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
408                                       int *BytesRemoved) const {
409   if (BytesRemoved)
410     *BytesRemoved = 0;
411   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
412   if (I == MBB.end())
413     return 0;
414 
415   if (!I->getDesc().isUnconditionalBranch() &&
416       !I->getDesc().isConditionalBranch())
417     return 0;
418 
419   // Remove the branch.
420   if (BytesRemoved)
421     *BytesRemoved += getInstSizeInBytes(*I);
422   I->eraseFromParent();
423 
424   I = MBB.end();
425 
426   if (I == MBB.begin())
427     return 1;
428   --I;
429   if (!I->getDesc().isConditionalBranch())
430     return 1;
431 
432   // Remove the branch.
433   if (BytesRemoved)
434     *BytesRemoved += getInstSizeInBytes(*I);
435   I->eraseFromParent();
436   return 2;
437 }
438 
439 // Inserts a branch into the end of the specific MachineBasicBlock, returning
440 // the number of instructions inserted.
441 unsigned RISCVInstrInfo::insertBranch(
442     MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
443     ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
444   if (BytesAdded)
445     *BytesAdded = 0;
446 
447   // Shouldn't be a fall through.
448   assert(TBB && "insertBranch must not be told to insert a fallthrough");
449   assert((Cond.size() == 3 || Cond.size() == 0) &&
450          "RISCV branch conditions have two components!");
451 
452   // Unconditional branch.
453   if (Cond.empty()) {
454     MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
455     if (BytesAdded)
456       *BytesAdded += getInstSizeInBytes(MI);
457     return 1;
458   }
459 
460   // Either a one or two-way conditional branch.
461   unsigned Opc = Cond[0].getImm();
462   MachineInstr &CondMI =
463       *BuildMI(&MBB, DL, get(Opc)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
464   if (BytesAdded)
465     *BytesAdded += getInstSizeInBytes(CondMI);
466 
467   // One-way conditional branch.
468   if (!FBB)
469     return 1;
470 
471   // Two-way conditional branch.
472   MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
473   if (BytesAdded)
474     *BytesAdded += getInstSizeInBytes(MI);
475   return 2;
476 }
477 
478 unsigned RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
479                                               MachineBasicBlock &DestBB,
480                                               const DebugLoc &DL,
481                                               int64_t BrOffset,
482                                               RegScavenger *RS) const {
483   assert(RS && "RegScavenger required for long branching");
484   assert(MBB.empty() &&
485          "new block should be inserted for expanding unconditional branch");
486   assert(MBB.pred_size() == 1);
487 
488   MachineFunction *MF = MBB.getParent();
489   MachineRegisterInfo &MRI = MF->getRegInfo();
490 
491   if (!isInt<32>(BrOffset))
492     report_fatal_error(
493         "Branch offsets outside of the signed 32-bit range not supported");
494 
495   // FIXME: A virtual register must be used initially, as the register
496   // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
497   // uses the same workaround).
498   Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
499   auto II = MBB.end();
500 
501   MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
502                           .addReg(ScratchReg, RegState::Define | RegState::Dead)
503                           .addMBB(&DestBB, RISCVII::MO_CALL);
504 
505   RS->enterBasicBlockEnd(MBB);
506   unsigned Scav = RS->scavengeRegisterBackwards(RISCV::GPRRegClass,
507                                                 MI.getIterator(), false, 0);
508   MRI.replaceRegWith(ScratchReg, Scav);
509   MRI.clearVirtRegs();
510   RS->setRegUsed(Scav);
511   return 8;
512 }
513 
514 bool RISCVInstrInfo::reverseBranchCondition(
515     SmallVectorImpl<MachineOperand> &Cond) const {
516   assert((Cond.size() == 3) && "Invalid branch condition!");
517   Cond[0].setImm(getOppositeBranchOpcode(Cond[0].getImm()));
518   return false;
519 }
520 
521 MachineBasicBlock *
522 RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
523   assert(MI.getDesc().isBranch() && "Unexpected opcode!");
524   // The branch target is always the last operand.
525   int NumOp = MI.getNumExplicitOperands();
526   return MI.getOperand(NumOp - 1).getMBB();
527 }
528 
529 bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
530                                            int64_t BrOffset) const {
531   unsigned XLen = STI.getXLen();
532   // Ideally we could determine the supported branch offset from the
533   // RISCVII::FormMask, but this can't be used for Pseudo instructions like
534   // PseudoBR.
535   switch (BranchOp) {
536   default:
537     llvm_unreachable("Unexpected opcode!");
538   case RISCV::BEQ:
539   case RISCV::BNE:
540   case RISCV::BLT:
541   case RISCV::BGE:
542   case RISCV::BLTU:
543   case RISCV::BGEU:
544     return isIntN(13, BrOffset);
545   case RISCV::JAL:
546   case RISCV::PseudoBR:
547     return isIntN(21, BrOffset);
548   case RISCV::PseudoJump:
549     return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
550   }
551 }
552 
553 unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
554   unsigned Opcode = MI.getOpcode();
555 
556   switch (Opcode) {
557   default: {
558     if (MI.getParent() && MI.getParent()->getParent()) {
559       const auto MF = MI.getMF();
560       const auto &TM = static_cast<const RISCVTargetMachine &>(MF->getTarget());
561       const MCRegisterInfo &MRI = *TM.getMCRegisterInfo();
562       const MCSubtargetInfo &STI = *TM.getMCSubtargetInfo();
563       const RISCVSubtarget &ST = MF->getSubtarget<RISCVSubtarget>();
564       if (isCompressibleInst(MI, &ST, MRI, STI))
565         return 2;
566     }
567     return get(Opcode).getSize();
568   }
569   case TargetOpcode::EH_LABEL:
570   case TargetOpcode::IMPLICIT_DEF:
571   case TargetOpcode::KILL:
572   case TargetOpcode::DBG_VALUE:
573     return 0;
574   // These values are determined based on RISCVExpandAtomicPseudoInsts,
575   // RISCVExpandPseudoInsts and RISCVMCCodeEmitter, depending on where the
576   // pseudos are expanded.
577   case RISCV::PseudoCALLReg:
578   case RISCV::PseudoCALL:
579   case RISCV::PseudoJump:
580   case RISCV::PseudoTAIL:
581   case RISCV::PseudoLLA:
582   case RISCV::PseudoLA:
583   case RISCV::PseudoLA_TLS_IE:
584   case RISCV::PseudoLA_TLS_GD:
585     return 8;
586   case RISCV::PseudoAtomicLoadNand32:
587   case RISCV::PseudoAtomicLoadNand64:
588     return 20;
589   case RISCV::PseudoMaskedAtomicSwap32:
590   case RISCV::PseudoMaskedAtomicLoadAdd32:
591   case RISCV::PseudoMaskedAtomicLoadSub32:
592     return 28;
593   case RISCV::PseudoMaskedAtomicLoadNand32:
594     return 32;
595   case RISCV::PseudoMaskedAtomicLoadMax32:
596   case RISCV::PseudoMaskedAtomicLoadMin32:
597     return 44;
598   case RISCV::PseudoMaskedAtomicLoadUMax32:
599   case RISCV::PseudoMaskedAtomicLoadUMin32:
600     return 36;
601   case RISCV::PseudoCmpXchg32:
602   case RISCV::PseudoCmpXchg64:
603     return 16;
604   case RISCV::PseudoMaskedCmpXchg32:
605     return 32;
606   case TargetOpcode::INLINEASM:
607   case TargetOpcode::INLINEASM_BR: {
608     const MachineFunction &MF = *MI.getParent()->getParent();
609     const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
610     return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
611                               *TM.getMCAsmInfo());
612   }
613   }
614 }
615 
616 bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
617   const unsigned Opcode = MI.getOpcode();
618   switch (Opcode) {
619   default:
620     break;
621   case RISCV::FSGNJ_D:
622   case RISCV::FSGNJ_S:
623     // The canonical floating-point move is fsgnj rd, rs, rs.
624     return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
625            MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
626   case RISCV::ADDI:
627   case RISCV::ORI:
628   case RISCV::XORI:
629     return (MI.getOperand(1).isReg() &&
630             MI.getOperand(1).getReg() == RISCV::X0) ||
631            (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
632   }
633   return MI.isAsCheapAsAMove();
634 }
635 
636 Optional<DestSourcePair>
637 RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
638   if (MI.isMoveReg())
639     return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
640   switch (MI.getOpcode()) {
641   default:
642     break;
643   case RISCV::ADDI:
644     // Operand 1 can be a frameindex but callers expect registers
645     if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
646         MI.getOperand(2).getImm() == 0)
647       return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
648     break;
649   case RISCV::FSGNJ_D:
650   case RISCV::FSGNJ_S:
651     // The canonical floating-point move is fsgnj rd, rs, rs.
652     if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
653         MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
654       return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
655     break;
656   }
657   return None;
658 }
659 
660 bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
661                                        StringRef &ErrInfo) const {
662   const MCInstrInfo *MCII = STI.getInstrInfo();
663   MCInstrDesc const &Desc = MCII->get(MI.getOpcode());
664 
665   for (auto &OI : enumerate(Desc.operands())) {
666     unsigned OpType = OI.value().OperandType;
667     if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
668         OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
669       const MachineOperand &MO = MI.getOperand(OI.index());
670       if (MO.isImm()) {
671         int64_t Imm = MO.getImm();
672         bool Ok;
673         switch (OpType) {
674         default:
675           llvm_unreachable("Unexpected operand type");
676         case RISCVOp::OPERAND_UIMM4:
677           Ok = isUInt<4>(Imm);
678           break;
679         case RISCVOp::OPERAND_UIMM5:
680           Ok = isUInt<5>(Imm);
681           break;
682         case RISCVOp::OPERAND_UIMM12:
683           Ok = isUInt<12>(Imm);
684           break;
685         case RISCVOp::OPERAND_SIMM12:
686           Ok = isInt<12>(Imm);
687           break;
688         case RISCVOp::OPERAND_UIMM20:
689           Ok = isUInt<20>(Imm);
690           break;
691         case RISCVOp::OPERAND_UIMMLOG2XLEN:
692           if (STI.getTargetTriple().isArch64Bit())
693             Ok = isUInt<6>(Imm);
694           else
695             Ok = isUInt<5>(Imm);
696           break;
697         }
698         if (!Ok) {
699           ErrInfo = "Invalid immediate";
700           return false;
701         }
702       }
703     }
704   }
705 
706   return true;
707 }
708 
709 // Return true if get the base operand, byte offset of an instruction and the
710 // memory width. Width is the size of memory that is being loaded/stored.
711 bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
712     const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
713     unsigned &Width, const TargetRegisterInfo *TRI) const {
714   if (!LdSt.mayLoadOrStore())
715     return false;
716 
717   // Here we assume the standard RISC-V ISA, which uses a base+offset
718   // addressing mode. You'll need to relax these conditions to support custom
719   // load/stores instructions.
720   if (LdSt.getNumExplicitOperands() != 3)
721     return false;
722   if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm())
723     return false;
724 
725   if (!LdSt.hasOneMemOperand())
726     return false;
727 
728   Width = (*LdSt.memoperands_begin())->getSize();
729   BaseReg = &LdSt.getOperand(1);
730   Offset = LdSt.getOperand(2).getImm();
731   return true;
732 }
733 
734 bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
735     const MachineInstr &MIa, const MachineInstr &MIb) const {
736   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
737   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
738 
739   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
740       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
741     return false;
742 
743   // Retrieve the base register, offset from the base register and width. Width
744   // is the size of memory that is being loaded/stored (e.g. 1, 2, 4).  If
745   // base registers are identical, and the offset of a lower memory access +
746   // the width doesn't overlap the offset of a higher memory access,
747   // then the memory accesses are different.
748   const TargetRegisterInfo *TRI = STI.getRegisterInfo();
749   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
750   int64_t OffsetA = 0, OffsetB = 0;
751   unsigned int WidthA = 0, WidthB = 0;
752   if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
753       getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
754     if (BaseOpA->isIdenticalTo(*BaseOpB)) {
755       int LowOffset = std::min(OffsetA, OffsetB);
756       int HighOffset = std::max(OffsetA, OffsetB);
757       int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
758       if (LowOffset + LowWidth <= HighOffset)
759         return true;
760     }
761   }
762   return false;
763 }
764 
765 std::pair<unsigned, unsigned>
766 RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
767   const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
768   return std::make_pair(TF & Mask, TF & ~Mask);
769 }
770 
771 ArrayRef<std::pair<unsigned, const char *>>
772 RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
773   using namespace RISCVII;
774   static const std::pair<unsigned, const char *> TargetFlags[] = {
775       {MO_CALL, "riscv-call"},
776       {MO_PLT, "riscv-plt"},
777       {MO_LO, "riscv-lo"},
778       {MO_HI, "riscv-hi"},
779       {MO_PCREL_LO, "riscv-pcrel-lo"},
780       {MO_PCREL_HI, "riscv-pcrel-hi"},
781       {MO_GOT_HI, "riscv-got-hi"},
782       {MO_TPREL_LO, "riscv-tprel-lo"},
783       {MO_TPREL_HI, "riscv-tprel-hi"},
784       {MO_TPREL_ADD, "riscv-tprel-add"},
785       {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
786       {MO_TLS_GD_HI, "riscv-tls-gd-hi"}};
787   return makeArrayRef(TargetFlags);
788 }
789 bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
790     MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
791   const Function &F = MF.getFunction();
792 
793   // Can F be deduplicated by the linker? If it can, don't outline from it.
794   if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
795     return false;
796 
797   // Don't outline from functions with section markings; the program could
798   // expect that all the code is in the named section.
799   if (F.hasSection())
800     return false;
801 
802   // It's safe to outline from MF.
803   return true;
804 }
805 
806 bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
807                                             unsigned &Flags) const {
808   // More accurate safety checking is done in getOutliningCandidateInfo.
809   return true;
810 }
811 
812 // Enum values indicating how an outlined call should be constructed.
813 enum MachineOutlinerConstructionID {
814   MachineOutlinerDefault
815 };
816 
817 outliner::OutlinedFunction RISCVInstrInfo::getOutliningCandidateInfo(
818     std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
819 
820   // First we need to filter out candidates where the X5 register (IE t0) can't
821   // be used to setup the function call.
822   auto CannotInsertCall = [](outliner::Candidate &C) {
823     const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
824 
825     C.initLRU(*TRI);
826     LiveRegUnits LRU = C.LRU;
827     return !LRU.available(RISCV::X5);
828   };
829 
830   llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall);
831 
832   // If the sequence doesn't have enough candidates left, then we're done.
833   if (RepeatedSequenceLocs.size() < 2)
834     return outliner::OutlinedFunction();
835 
836   unsigned SequenceSize = 0;
837 
838   auto I = RepeatedSequenceLocs[0].front();
839   auto E = std::next(RepeatedSequenceLocs[0].back());
840   for (; I != E; ++I)
841     SequenceSize += getInstSizeInBytes(*I);
842 
843   // call t0, function = 8 bytes.
844   unsigned CallOverhead = 8;
845   for (auto &C : RepeatedSequenceLocs)
846     C.setCallInfo(MachineOutlinerDefault, CallOverhead);
847 
848   // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
849   unsigned FrameOverhead = 4;
850   if (RepeatedSequenceLocs[0].getMF()->getSubtarget()
851           .getFeatureBits()[RISCV::FeatureStdExtC])
852     FrameOverhead = 2;
853 
854   return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
855                                     FrameOverhead, MachineOutlinerDefault);
856 }
857 
858 outliner::InstrType
859 RISCVInstrInfo::getOutliningType(MachineBasicBlock::iterator &MBBI,
860                                  unsigned Flags) const {
861   MachineInstr &MI = *MBBI;
862   MachineBasicBlock *MBB = MI.getParent();
863   const TargetRegisterInfo *TRI =
864       MBB->getParent()->getSubtarget().getRegisterInfo();
865 
866   // Positions generally can't safely be outlined.
867   if (MI.isPosition()) {
868     // We can manually strip out CFI instructions later.
869     if (MI.isCFIInstruction())
870       return outliner::InstrType::Invisible;
871 
872     return outliner::InstrType::Illegal;
873   }
874 
875   // Don't trust the user to write safe inline assembly.
876   if (MI.isInlineAsm())
877     return outliner::InstrType::Illegal;
878 
879   // We can't outline branches to other basic blocks.
880   if (MI.isTerminator() && !MBB->succ_empty())
881     return outliner::InstrType::Illegal;
882 
883   // We need support for tail calls to outlined functions before return
884   // statements can be allowed.
885   if (MI.isReturn())
886     return outliner::InstrType::Illegal;
887 
888   // Don't allow modifying the X5 register which we use for return addresses for
889   // these outlined functions.
890   if (MI.modifiesRegister(RISCV::X5, TRI) ||
891       MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5))
892     return outliner::InstrType::Illegal;
893 
894   // Make sure the operands don't reference something unsafe.
895   for (const auto &MO : MI.operands())
896     if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI())
897       return outliner::InstrType::Illegal;
898 
899   // Don't allow instructions which won't be materialized to impact outlining
900   // analysis.
901   if (MI.isMetaInstruction())
902     return outliner::InstrType::Invisible;
903 
904   return outliner::InstrType::Legal;
905 }
906 
907 void RISCVInstrInfo::buildOutlinedFrame(
908     MachineBasicBlock &MBB, MachineFunction &MF,
909     const outliner::OutlinedFunction &OF) const {
910 
911   // Strip out any CFI instructions
912   bool Changed = true;
913   while (Changed) {
914     Changed = false;
915     auto I = MBB.begin();
916     auto E = MBB.end();
917     for (; I != E; ++I) {
918       if (I->isCFIInstruction()) {
919         I->removeFromParent();
920         Changed = true;
921         break;
922       }
923     }
924   }
925 
926   MBB.addLiveIn(RISCV::X5);
927 
928   // Add in a return instruction to the end of the outlined frame.
929   MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
930       .addReg(RISCV::X0, RegState::Define)
931       .addReg(RISCV::X5)
932       .addImm(0));
933 }
934 
935 MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
936     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
937     MachineFunction &MF, const outliner::Candidate &C) const {
938 
939   // Add in a call instruction to the outlined function at the given location.
940   It = MBB.insert(It,
941                   BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
942                       .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
943                                         RISCVII::MO_CALL));
944   return It;
945 }
946 
947 // clang-format off
948 #define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL)                                \
949   RISCV::PseudoV##OP##_##TYPE##_##LMUL##_COMMUTABLE
950 
951 #define CASE_VFMA_OPCODE_LMULS(OP, TYPE)                                       \
952   CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8):                                      \
953   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4):                                 \
954   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2):                                 \
955   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1):                                  \
956   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2):                                  \
957   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4):                                  \
958   case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8)
959 
960 #define CASE_VFMA_SPLATS(OP)                                                   \
961   CASE_VFMA_OPCODE_LMULS(OP, VF16):                                            \
962   case CASE_VFMA_OPCODE_LMULS(OP, VF32):                                       \
963   case CASE_VFMA_OPCODE_LMULS(OP, VF64)
964 // clang-format on
965 
966 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
967                                            unsigned &SrcOpIdx1,
968                                            unsigned &SrcOpIdx2) const {
969   const MCInstrDesc &Desc = MI.getDesc();
970   if (!Desc.isCommutable())
971     return false;
972 
973   switch (MI.getOpcode()) {
974   case CASE_VFMA_SPLATS(FMADD):
975   case CASE_VFMA_SPLATS(FMSUB):
976   case CASE_VFMA_SPLATS(FMACC):
977   case CASE_VFMA_SPLATS(FMSAC):
978   case CASE_VFMA_SPLATS(FNMADD):
979   case CASE_VFMA_SPLATS(FNMSUB):
980   case CASE_VFMA_SPLATS(FNMACC):
981   case CASE_VFMA_SPLATS(FNMSAC):
982   case CASE_VFMA_OPCODE_LMULS(FMACC, VV):
983   case CASE_VFMA_OPCODE_LMULS(FMSAC, VV):
984   case CASE_VFMA_OPCODE_LMULS(FNMACC, VV):
985   case CASE_VFMA_OPCODE_LMULS(FNMSAC, VV): {
986     // For these instructions we can only swap operand 1 and operand 3 by
987     // changing the opcode.
988     unsigned CommutableOpIdx1 = 1;
989     unsigned CommutableOpIdx2 = 3;
990     if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
991                               CommutableOpIdx2))
992       return false;
993     return true;
994   }
995   case CASE_VFMA_OPCODE_LMULS(FMADD, VV):
996   case CASE_VFMA_OPCODE_LMULS(FMSUB, VV):
997   case CASE_VFMA_OPCODE_LMULS(FNMADD, VV):
998   case CASE_VFMA_OPCODE_LMULS(FNMSUB, VV): {
999     // For these instructions we have more freedom. We can commute with the
1000     // other multiplicand or with the addend/subtrahend/minuend.
1001 
1002     // Any fixed operand must be from source 1, 2 or 3.
1003     if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
1004       return false;
1005     if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
1006       return false;
1007 
1008     // It both ops are fixed one must be the tied source.
1009     if (SrcOpIdx1 != CommuteAnyOperandIndex &&
1010         SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
1011       return false;
1012 
1013     // Look for two different register operands assumed to be commutable
1014     // regardless of the FMA opcode. The FMA opcode is adjusted later if
1015     // needed.
1016     if (SrcOpIdx1 == CommuteAnyOperandIndex ||
1017         SrcOpIdx2 == CommuteAnyOperandIndex) {
1018       // At least one of operands to be commuted is not specified and
1019       // this method is free to choose appropriate commutable operands.
1020       unsigned CommutableOpIdx1 = SrcOpIdx1;
1021       if (SrcOpIdx1 == SrcOpIdx2) {
1022         // Both of operands are not fixed. Set one of commutable
1023         // operands to the tied source.
1024         CommutableOpIdx1 = 1;
1025       } else if (SrcOpIdx1 == CommutableOpIdx1) {
1026         // Only one of the operands is not fixed.
1027         CommutableOpIdx1 = SrcOpIdx2;
1028       }
1029 
1030       // CommutableOpIdx1 is well defined now. Let's choose another commutable
1031       // operand and assign its index to CommutableOpIdx2.
1032       unsigned CommutableOpIdx2;
1033       if (CommutableOpIdx1 != 1) {
1034         // If we haven't already used the tied source, we must use it now.
1035         CommutableOpIdx2 = 1;
1036       } else {
1037         Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
1038 
1039         // The commuted operands should have different registers.
1040         // Otherwise, the commute transformation does not change anything and
1041         // is useless. We use this as a hint to make our decision.
1042         if (Op1Reg != MI.getOperand(2).getReg())
1043           CommutableOpIdx2 = 2;
1044         else
1045           CommutableOpIdx2 = 3;
1046       }
1047 
1048       // Assign the found pair of commutable indices to SrcOpIdx1 and
1049       // SrcOpIdx2 to return those values.
1050       if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
1051                                 CommutableOpIdx2))
1052         return false;
1053     }
1054 
1055     return true;
1056   }
1057   }
1058 
1059   return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1060 }
1061 
1062 #define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL)               \
1063   case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_COMMUTABLE:                   \
1064     Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_COMMUTABLE;                \
1065     break;
1066 
1067 #define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                      \
1068   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                      \
1069   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                      \
1070   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                      \
1071   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                       \
1072   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                       \
1073   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                       \
1074   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
1075 
1076 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                           \
1077   CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF16)                            \
1078   CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF32)                            \
1079   CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, VF64)
1080 
1081 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
1082                                                      bool NewMI,
1083                                                      unsigned OpIdx1,
1084                                                      unsigned OpIdx2) const {
1085   auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
1086     if (NewMI)
1087       return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
1088     return MI;
1089   };
1090 
1091   switch (MI.getOpcode()) {
1092   case CASE_VFMA_SPLATS(FMACC):
1093   case CASE_VFMA_SPLATS(FMADD):
1094   case CASE_VFMA_SPLATS(FMSAC):
1095   case CASE_VFMA_SPLATS(FMSUB):
1096   case CASE_VFMA_SPLATS(FNMACC):
1097   case CASE_VFMA_SPLATS(FNMADD):
1098   case CASE_VFMA_SPLATS(FNMSAC):
1099   case CASE_VFMA_SPLATS(FNMSUB):
1100   case CASE_VFMA_OPCODE_LMULS(FMACC, VV):
1101   case CASE_VFMA_OPCODE_LMULS(FMSAC, VV):
1102   case CASE_VFMA_OPCODE_LMULS(FNMACC, VV):
1103   case CASE_VFMA_OPCODE_LMULS(FNMSAC, VV): {
1104     // It only make sense to toggle these between clobbering the
1105     // addend/subtrahend/minuend one of the multiplicands.
1106     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
1107     assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
1108     unsigned Opc;
1109     switch (MI.getOpcode()) {
1110       default:
1111         llvm_unreachable("Unexpected opcode");
1112       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
1113       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
1114       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
1115       CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
1116       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
1117       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
1118       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
1119       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
1120       CASE_VFMA_CHANGE_OPCODE_LMULS(FMACC, FMADD, VV)
1121       CASE_VFMA_CHANGE_OPCODE_LMULS(FMSAC, FMSUB, VV)
1122       CASE_VFMA_CHANGE_OPCODE_LMULS(FNMACC, FNMADD, VV)
1123       CASE_VFMA_CHANGE_OPCODE_LMULS(FNMSAC, FNMSUB, VV)
1124     }
1125 
1126     auto &WorkingMI = cloneIfNew(MI);
1127     WorkingMI.setDesc(get(Opc));
1128     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
1129                                                    OpIdx1, OpIdx2);
1130   }
1131   case CASE_VFMA_OPCODE_LMULS(FMADD, VV):
1132   case CASE_VFMA_OPCODE_LMULS(FMSUB, VV):
1133   case CASE_VFMA_OPCODE_LMULS(FNMADD, VV):
1134   case CASE_VFMA_OPCODE_LMULS(FNMSUB, VV): {
1135     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
1136     // If one of the operands, is the addend we need to change opcode.
1137     // Otherwise we're just swapping 2 of the multiplicands.
1138     if (OpIdx1 == 3 || OpIdx2 == 3) {
1139       unsigned Opc;
1140       switch (MI.getOpcode()) {
1141         default:
1142           llvm_unreachable("Unexpected opcode");
1143         CASE_VFMA_CHANGE_OPCODE_LMULS(FMADD, FMACC, VV)
1144         CASE_VFMA_CHANGE_OPCODE_LMULS(FMSUB, FMSAC, VV)
1145         CASE_VFMA_CHANGE_OPCODE_LMULS(FNMADD, FNMACC, VV)
1146         CASE_VFMA_CHANGE_OPCODE_LMULS(FNMSUB, FNMSAC, VV)
1147       }
1148 
1149       auto &WorkingMI = cloneIfNew(MI);
1150       WorkingMI.setDesc(get(Opc));
1151       return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
1152                                                      OpIdx1, OpIdx2);
1153     }
1154     // Let the default code handle it.
1155     break;
1156   }
1157   }
1158 
1159   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1160 }
1161 
1162 #undef CASE_VFMA_CHANGE_OPCODE_SPLATS
1163 #undef CASE_VFMA_CHANGE_OPCODE_LMULS
1164 #undef CASE_VFMA_CHANGE_OPCODE_COMMON
1165 #undef CASE_VFMA_SPLATS
1166 #undef CASE_VFMA_OPCODE_LMULS
1167 #undef CASE_VFMA_OPCODE_COMMON
1168 
1169 Register RISCVInstrInfo::getVLENFactoredAmount(MachineFunction &MF,
1170                                                MachineBasicBlock &MBB,
1171                                                MachineBasicBlock::iterator II,
1172                                                int64_t Amount) const {
1173   assert(Amount > 0 && "There is no need to get VLEN scaled value.");
1174   assert(Amount % 8 == 0 &&
1175          "Reserve the stack by the multiple of one vector size.");
1176 
1177   MachineRegisterInfo &MRI = MF.getRegInfo();
1178   const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1179   DebugLoc DL = II->getDebugLoc();
1180   int64_t NumOfVReg = Amount / 8;
1181 
1182   Register SizeOfVector = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1183   BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), SizeOfVector);
1184   Register FactorRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1185   assert(isInt<12>(NumOfVReg) &&
1186          "Expect the number of vector registers within 12-bits.");
1187   if (isPowerOf2_32(NumOfVReg)) {
1188     uint32_t ShiftAmount = Log2_32(NumOfVReg);
1189     if (ShiftAmount == 0)
1190       return SizeOfVector;
1191     BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), FactorRegister)
1192         .addReg(SizeOfVector, RegState::Kill)
1193         .addImm(ShiftAmount);
1194   } else {
1195     Register VN = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1196     BuildMI(MBB, II, DL, TII->get(RISCV::ADDI), VN)
1197         .addReg(RISCV::X0)
1198         .addImm(NumOfVReg);
1199     assert(MF.getSubtarget<RISCVSubtarget>().hasStdExtM() &&
1200            "M-extension must be enabled to calculate the vscaled size/offset.");
1201     BuildMI(MBB, II, DL, TII->get(RISCV::MUL), FactorRegister)
1202         .addReg(SizeOfVector, RegState::Kill)
1203         .addReg(VN, RegState::Kill);
1204   }
1205 
1206   return FactorRegister;
1207 }
1208