1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the MachineIRBuidler class.
10 //===----------------------------------------------------------------------===//
11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
13 
14 #include "llvm/CodeGen/MachineFunction.h"
15 #include "llvm/CodeGen/MachineInstr.h"
16 #include "llvm/CodeGen/MachineInstrBuilder.h"
17 #include "llvm/CodeGen/MachineRegisterInfo.h"
18 #include "llvm/CodeGen/TargetInstrInfo.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 #include "llvm/CodeGen/TargetOpcodes.h"
21 #include "llvm/CodeGen/TargetSubtargetInfo.h"
22 #include "llvm/IR/DebugInfo.h"
23 
24 using namespace llvm;
25 
26 void MachineIRBuilder::setMF(MachineFunction &MF) {
27   State.MF = &MF;
28   State.MBB = nullptr;
29   State.MRI = &MF.getRegInfo();
30   State.TII = MF.getSubtarget().getInstrInfo();
31   State.DL = DebugLoc();
32   State.II = MachineBasicBlock::iterator();
33   State.Observer = nullptr;
34 }
35 
36 void MachineIRBuilder::setMBB(MachineBasicBlock &MBB) {
37   State.MBB = &MBB;
38   State.II = MBB.end();
39   assert(&getMF() == MBB.getParent() &&
40          "Basic block is in a different function");
41 }
42 
43 void MachineIRBuilder::setInstr(MachineInstr &MI) {
44   assert(MI.getParent() && "Instruction is not part of a basic block");
45   setMBB(*MI.getParent());
46   State.II = MI.getIterator();
47 }
48 
49 void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; }
50 
51 void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB,
52                                    MachineBasicBlock::iterator II) {
53   assert(MBB.getParent() == &getMF() &&
54          "Basic block is in a different function");
55   State.MBB = &MBB;
56   State.II = II;
57 }
58 
59 void MachineIRBuilder::recordInsertion(MachineInstr *InsertedInstr) const {
60   if (State.Observer)
61     State.Observer->createdInstr(*InsertedInstr);
62 }
63 
64 void MachineIRBuilder::setChangeObserver(GISelChangeObserver &Observer) {
65   State.Observer = &Observer;
66 }
67 
68 void MachineIRBuilder::stopObservingChanges() { State.Observer = nullptr; }
69 
70 //------------------------------------------------------------------------------
71 // Build instruction variants.
72 //------------------------------------------------------------------------------
73 
74 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opcode) {
75   return insertInstr(buildInstrNoInsert(Opcode));
76 }
77 
78 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
79   MachineInstrBuilder MIB = BuildMI(getMF(), getDL(), getTII().get(Opcode));
80   return MIB;
81 }
82 
83 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
84   getMBB().insert(getInsertPt(), MIB);
85   recordInsertion(MIB);
86   return MIB;
87 }
88 
89 MachineInstrBuilder
90 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
91                                       const MDNode *Expr) {
92   assert(isa<DILocalVariable>(Variable) && "not a variable");
93   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
94   assert(
95       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
96       "Expected inlined-at fields to agree");
97   return insertInstr(BuildMI(getMF(), getDL(),
98                              getTII().get(TargetOpcode::DBG_VALUE),
99                              /*IsIndirect*/ false, Reg, Variable, Expr));
100 }
101 
102 MachineInstrBuilder
103 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
104                                         const MDNode *Expr) {
105   assert(isa<DILocalVariable>(Variable) && "not a variable");
106   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
107   assert(
108       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
109       "Expected inlined-at fields to agree");
110   // DBG_VALUE insts now carry IR-level indirection in their DIExpression
111   // rather than encoding it in the instruction itself.
112   const DIExpression *DIExpr = cast<DIExpression>(Expr);
113   DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref});
114   return insertInstr(BuildMI(getMF(), getDL(),
115                              getTII().get(TargetOpcode::DBG_VALUE),
116                              /*IsIndirect*/ false, Reg, Variable, DIExpr));
117 }
118 
119 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
120                                                       const MDNode *Variable,
121                                                       const MDNode *Expr) {
122   assert(isa<DILocalVariable>(Variable) && "not a variable");
123   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
124   assert(
125       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
126       "Expected inlined-at fields to agree");
127   // DBG_VALUE insts now carry IR-level indirection in their DIExpression
128   // rather than encoding it in the instruction itself.
129   const DIExpression *DIExpr = cast<DIExpression>(Expr);
130   DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref});
131   return buildInstr(TargetOpcode::DBG_VALUE)
132       .addFrameIndex(FI)
133       .addReg(0)
134       .addMetadata(Variable)
135       .addMetadata(DIExpr);
136 }
137 
138 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
139                                                          const MDNode *Variable,
140                                                          const MDNode *Expr) {
141   assert(isa<DILocalVariable>(Variable) && "not a variable");
142   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
143   assert(
144       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
145       "Expected inlined-at fields to agree");
146   auto MIB = buildInstr(TargetOpcode::DBG_VALUE);
147   if (auto *CI = dyn_cast<ConstantInt>(&C)) {
148     if (CI->getBitWidth() > 64)
149       MIB.addCImm(CI);
150     else
151       MIB.addImm(CI->getZExtValue());
152   } else if (auto *CFP = dyn_cast<ConstantFP>(&C)) {
153     MIB.addFPImm(CFP);
154   } else {
155     // Insert %noreg if we didn't find a usable constant and had to drop it.
156     MIB.addReg(0U);
157   }
158 
159   return MIB.addReg(0).addMetadata(Variable).addMetadata(Expr);
160 }
161 
162 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
163   assert(isa<DILabel>(Label) && "not a label");
164   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
165          "Expected inlined-at fields to agree");
166   auto MIB = buildInstr(TargetOpcode::DBG_LABEL);
167 
168   return MIB.addMetadata(Label);
169 }
170 
171 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
172                                                          const SrcOp &Size,
173                                                          unsigned Align) {
174   assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
175   auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC);
176   Res.addDefToMIB(*getMRI(), MIB);
177   Size.addSrcToMIB(MIB);
178   MIB.addImm(Align);
179   return MIB;
180 }
181 
182 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
183                                                       int Idx) {
184   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
185   auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX);
186   Res.addDefToMIB(*getMRI(), MIB);
187   MIB.addFrameIndex(Idx);
188   return MIB;
189 }
190 
191 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
192                                                        const GlobalValue *GV) {
193   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
194   assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
195              GV->getType()->getAddressSpace() &&
196          "address space mismatch");
197 
198   auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE);
199   Res.addDefToMIB(*getMRI(), MIB);
200   MIB.addGlobalAddress(GV);
201   return MIB;
202 }
203 
204 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
205                                                      unsigned JTI) {
206   return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
207       .addJumpTableIndex(JTI);
208 }
209 
210 void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0,
211                                         const LLT &Op1) {
212   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
213   assert((Res == Op0 && Res == Op1) && "type mismatch");
214 }
215 
216 void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0,
217                                        const LLT &Op1) {
218   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
219   assert((Res == Op0) && "type mismatch");
220 }
221 
222 MachineInstrBuilder MachineIRBuilder::buildPtrAdd(const DstOp &Res,
223                                                   const SrcOp &Op0,
224                                                   const SrcOp &Op1) {
225   assert(Res.getLLTTy(*getMRI()).getScalarType().isPointer() &&
226          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
227   assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
228 
229   return buildInstr(TargetOpcode::G_PTR_ADD, {Res}, {Op0, Op1});
230 }
231 
232 Optional<MachineInstrBuilder>
233 MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
234                                     const LLT &ValueTy, uint64_t Value) {
235   assert(Res == 0 && "Res is a result argument");
236   assert(ValueTy.isScalar()  && "invalid offset type");
237 
238   if (Value == 0) {
239     Res = Op0;
240     return None;
241   }
242 
243   Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
244   auto Cst = buildConstant(ValueTy, Value);
245   return buildPtrAdd(Res, Op0, Cst.getReg(0));
246 }
247 
248 MachineInstrBuilder MachineIRBuilder::buildPtrMask(const DstOp &Res,
249                                                    const SrcOp &Op0,
250                                                    uint32_t NumBits) {
251   assert(Res.getLLTTy(*getMRI()).isPointer() &&
252          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
253 
254   auto MIB = buildInstr(TargetOpcode::G_PTR_MASK);
255   Res.addDefToMIB(*getMRI(), MIB);
256   Op0.addSrcToMIB(MIB);
257   MIB.addImm(NumBits);
258   return MIB;
259 }
260 
261 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
262   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
263 }
264 
265 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
266   assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
267   return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt);
268 }
269 
270 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
271                                                 unsigned JTI,
272                                                 Register IndexReg) {
273   assert(getMRI()->getType(TablePtr).isPointer() &&
274          "Table reg must be a pointer");
275   return buildInstr(TargetOpcode::G_BRJT)
276       .addUse(TablePtr)
277       .addJumpTableIndex(JTI)
278       .addUse(IndexReg);
279 }
280 
281 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
282                                                 const SrcOp &Op) {
283   return buildInstr(TargetOpcode::COPY, Res, Op);
284 }
285 
286 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
287                                                     const ConstantInt &Val) {
288   LLT Ty = Res.getLLTTy(*getMRI());
289   LLT EltTy = Ty.getScalarType();
290   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
291          "creating constant with the wrong size");
292 
293   if (Ty.isVector()) {
294     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
295     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
296     .addCImm(&Val);
297     return buildSplatVector(Res, Const);
298   }
299 
300   auto Const = buildInstr(TargetOpcode::G_CONSTANT);
301   Res.addDefToMIB(*getMRI(), Const);
302   Const.addCImm(&Val);
303   return Const;
304 }
305 
306 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
307                                                     int64_t Val) {
308   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
309                                Res.getLLTTy(*getMRI()).getScalarSizeInBits());
310   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
311   return buildConstant(Res, *CI);
312 }
313 
314 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
315                                                      const ConstantFP &Val) {
316   LLT Ty = Res.getLLTTy(*getMRI());
317   LLT EltTy = Ty.getScalarType();
318 
319   assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
320          == EltTy.getSizeInBits() &&
321          "creating fconstant with the wrong size");
322 
323   assert(!Ty.isPointer() && "invalid operand type");
324 
325   if (Ty.isVector()) {
326     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
327     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
328     .addFPImm(&Val);
329 
330     return buildSplatVector(Res, Const);
331   }
332 
333   auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
334   Res.addDefToMIB(*getMRI(), Const);
335   Const.addFPImm(&Val);
336   return Const;
337 }
338 
339 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
340                                                     const APInt &Val) {
341   ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
342   return buildConstant(Res, *CI);
343 }
344 
345 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
346                                                      double Val) {
347   LLT DstTy = Res.getLLTTy(*getMRI());
348   auto &Ctx = getMF().getFunction().getContext();
349   auto *CFP =
350       ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
351   return buildFConstant(Res, *CFP);
352 }
353 
354 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
355                                                      const APFloat &Val) {
356   auto &Ctx = getMF().getFunction().getContext();
357   auto *CFP = ConstantFP::get(Ctx, Val);
358   return buildFConstant(Res, *CFP);
359 }
360 
361 MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst,
362                                                   MachineBasicBlock &Dest) {
363   assert(getMRI()->getType(Tst).isScalar() && "invalid operand type");
364 
365   return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest);
366 }
367 
368 MachineInstrBuilder MachineIRBuilder::buildLoad(const DstOp &Res,
369                                                 const SrcOp &Addr,
370                                                 MachineMemOperand &MMO) {
371   return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO);
372 }
373 
374 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
375                                                      const DstOp &Res,
376                                                      const SrcOp &Addr,
377                                                      MachineMemOperand &MMO) {
378   assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
379   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
380 
381   auto MIB = buildInstr(Opcode);
382   Res.addDefToMIB(*getMRI(), MIB);
383   Addr.addSrcToMIB(MIB);
384   MIB.addMemOperand(&MMO);
385   return MIB;
386 }
387 
388 MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
389                                                  const SrcOp &Addr,
390                                                  MachineMemOperand &MMO) {
391   assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
392   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
393 
394   auto MIB = buildInstr(TargetOpcode::G_STORE);
395   Val.addSrcToMIB(MIB);
396   Addr.addSrcToMIB(MIB);
397   MIB.addMemOperand(&MMO);
398   return MIB;
399 }
400 
401 MachineInstrBuilder MachineIRBuilder::buildUAddo(const DstOp &Res,
402                                                  const DstOp &CarryOut,
403                                                  const SrcOp &Op0,
404                                                  const SrcOp &Op1) {
405   return buildInstr(TargetOpcode::G_UADDO, {Res, CarryOut}, {Op0, Op1});
406 }
407 
408 MachineInstrBuilder MachineIRBuilder::buildUAdde(const DstOp &Res,
409                                                  const DstOp &CarryOut,
410                                                  const SrcOp &Op0,
411                                                  const SrcOp &Op1,
412                                                  const SrcOp &CarryIn) {
413   return buildInstr(TargetOpcode::G_UADDE, {Res, CarryOut},
414                     {Op0, Op1, CarryIn});
415 }
416 
417 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
418                                                   const SrcOp &Op) {
419   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
420 }
421 
422 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
423                                                 const SrcOp &Op) {
424   return buildInstr(TargetOpcode::G_SEXT, Res, Op);
425 }
426 
427 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
428                                                 const SrcOp &Op) {
429   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
430 }
431 
432 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
433   const auto *TLI = getMF().getSubtarget().getTargetLowering();
434   switch (TLI->getBooleanContents(IsVec, IsFP)) {
435   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
436     return TargetOpcode::G_SEXT;
437   case TargetLoweringBase::ZeroOrOneBooleanContent:
438     return TargetOpcode::G_ZEXT;
439   default:
440     return TargetOpcode::G_ANYEXT;
441   }
442 }
443 
444 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
445                                                    const SrcOp &Op,
446                                                    bool IsFP) {
447   unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
448   return buildInstr(ExtOp, Res, Op);
449 }
450 
451 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
452                                                       const DstOp &Res,
453                                                       const SrcOp &Op) {
454   assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
455           TargetOpcode::G_SEXT == ExtOpc) &&
456          "Expecting Extending Opc");
457   assert(Res.getLLTTy(*getMRI()).isScalar() ||
458          Res.getLLTTy(*getMRI()).isVector());
459   assert(Res.getLLTTy(*getMRI()).isScalar() ==
460          Op.getLLTTy(*getMRI()).isScalar());
461 
462   unsigned Opcode = TargetOpcode::COPY;
463   if (Res.getLLTTy(*getMRI()).getSizeInBits() >
464       Op.getLLTTy(*getMRI()).getSizeInBits())
465     Opcode = ExtOpc;
466   else if (Res.getLLTTy(*getMRI()).getSizeInBits() <
467            Op.getLLTTy(*getMRI()).getSizeInBits())
468     Opcode = TargetOpcode::G_TRUNC;
469   else
470     assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
471 
472   return buildInstr(Opcode, Res, Op);
473 }
474 
475 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
476                                                        const SrcOp &Op) {
477   return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op);
478 }
479 
480 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
481                                                        const SrcOp &Op) {
482   return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op);
483 }
484 
485 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
486                                                          const SrcOp &Op) {
487   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
488 }
489 
490 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
491                                                 const SrcOp &Src) {
492   LLT SrcTy = Src.getLLTTy(*getMRI());
493   LLT DstTy = Dst.getLLTTy(*getMRI());
494   if (SrcTy == DstTy)
495     return buildCopy(Dst, Src);
496 
497   unsigned Opcode;
498   if (SrcTy.isPointer() && DstTy.isScalar())
499     Opcode = TargetOpcode::G_PTRTOINT;
500   else if (DstTy.isPointer() && SrcTy.isScalar())
501     Opcode = TargetOpcode::G_INTTOPTR;
502   else {
503     assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
504     Opcode = TargetOpcode::G_BITCAST;
505   }
506 
507   return buildInstr(Opcode, Dst, Src);
508 }
509 
510 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
511                                                    const SrcOp &Src,
512                                                    uint64_t Index) {
513   LLT SrcTy = Src.getLLTTy(*getMRI());
514   LLT DstTy = Dst.getLLTTy(*getMRI());
515 
516 #ifndef NDEBUG
517   assert(SrcTy.isValid() && "invalid operand type");
518   assert(DstTy.isValid() && "invalid operand type");
519   assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
520          "extracting off end of register");
521 #endif
522 
523   if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
524     assert(Index == 0 && "insertion past the end of a register");
525     return buildCast(Dst, Src);
526   }
527 
528   auto Extract = buildInstr(TargetOpcode::G_EXTRACT);
529   Dst.addDefToMIB(*getMRI(), Extract);
530   Src.addSrcToMIB(Extract);
531   Extract.addImm(Index);
532   return Extract;
533 }
534 
535 void MachineIRBuilder::buildSequence(Register Res, ArrayRef<Register> Ops,
536                                      ArrayRef<uint64_t> Indices) {
537 #ifndef NDEBUG
538   assert(Ops.size() == Indices.size() && "incompatible args");
539   assert(!Ops.empty() && "invalid trivial sequence");
540   assert(std::is_sorted(Indices.begin(), Indices.end()) &&
541          "sequence offsets must be in ascending order");
542 
543   assert(getMRI()->getType(Res).isValid() && "invalid operand type");
544   for (auto Op : Ops)
545     assert(getMRI()->getType(Op).isValid() && "invalid operand type");
546 #endif
547 
548   LLT ResTy = getMRI()->getType(Res);
549   LLT OpTy = getMRI()->getType(Ops[0]);
550   unsigned OpSize = OpTy.getSizeInBits();
551   bool MaybeMerge = true;
552   for (unsigned i = 0; i < Ops.size(); ++i) {
553     if (getMRI()->getType(Ops[i]) != OpTy || Indices[i] != i * OpSize) {
554       MaybeMerge = false;
555       break;
556     }
557   }
558 
559   if (MaybeMerge && Ops.size() * OpSize == ResTy.getSizeInBits()) {
560     buildMerge(Res, Ops);
561     return;
562   }
563 
564   Register ResIn = getMRI()->createGenericVirtualRegister(ResTy);
565   buildUndef(ResIn);
566 
567   for (unsigned i = 0; i < Ops.size(); ++i) {
568     Register ResOut = i + 1 == Ops.size()
569                           ? Res
570                           : getMRI()->createGenericVirtualRegister(ResTy);
571     buildInsert(ResOut, ResIn, Ops[i], Indices[i]);
572     ResIn = ResOut;
573   }
574 }
575 
576 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
577   return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {});
578 }
579 
580 MachineInstrBuilder MachineIRBuilder::buildMerge(const DstOp &Res,
581                                                  ArrayRef<Register> Ops) {
582   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
583   // we need some temporary storage for the DstOp objects. Here we use a
584   // sufficiently large SmallVector to not go through the heap.
585   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
586   assert(TmpVec.size() > 1);
587   return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec);
588 }
589 
590 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
591                                                    const SrcOp &Op) {
592   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
593   // we need some temporary storage for the DstOp objects. Here we use a
594   // sufficiently large SmallVector to not go through the heap.
595   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
596   assert(TmpVec.size() > 1);
597   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
598 }
599 
600 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
601                                                    const SrcOp &Op) {
602   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
603   SmallVector<Register, 8> TmpVec;
604   for (unsigned I = 0; I != NumReg; ++I)
605     TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res));
606   return buildUnmerge(TmpVec, Op);
607 }
608 
609 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
610                                                    const SrcOp &Op) {
611   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
612   // we need some temporary storage for the DstOp objects. Here we use a
613   // sufficiently large SmallVector to not go through the heap.
614   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
615   assert(TmpVec.size() > 1);
616   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
617 }
618 
619 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
620                                                        ArrayRef<Register> Ops) {
621   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
622   // we need some temporary storage for the DstOp objects. Here we use a
623   // sufficiently large SmallVector to not go through the heap.
624   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
625   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
626 }
627 
628 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
629                                                        const SrcOp &Src) {
630   SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
631   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
632 }
633 
634 MachineInstrBuilder
635 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
636                                         ArrayRef<Register> Ops) {
637   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
638   // we need some temporary storage for the DstOp objects. Here we use a
639   // sufficiently large SmallVector to not go through the heap.
640   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
641   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
642 }
643 
644 MachineInstrBuilder
645 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
646   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
647   // we need some temporary storage for the DstOp objects. Here we use a
648   // sufficiently large SmallVector to not go through the heap.
649   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
650   return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec);
651 }
652 
653 MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
654                                                   const SrcOp &Src,
655                                                   const SrcOp &Op,
656                                                   unsigned Index) {
657   assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
658              Res.getLLTTy(*getMRI()).getSizeInBits() &&
659          "insertion past the end of a register");
660 
661   if (Res.getLLTTy(*getMRI()).getSizeInBits() ==
662       Op.getLLTTy(*getMRI()).getSizeInBits()) {
663     return buildCast(Res, Op);
664   }
665 
666   return buildInstr(TargetOpcode::G_INSERT, Res, {Src, Op, uint64_t(Index)});
667 }
668 
669 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
670                                                      ArrayRef<Register> ResultRegs,
671                                                      bool HasSideEffects) {
672   auto MIB =
673       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
674                                 : TargetOpcode::G_INTRINSIC);
675   for (unsigned ResultReg : ResultRegs)
676     MIB.addDef(ResultReg);
677   MIB.addIntrinsicID(ID);
678   return MIB;
679 }
680 
681 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
682                                                      ArrayRef<DstOp> Results,
683                                                      bool HasSideEffects) {
684   auto MIB =
685       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
686                                 : TargetOpcode::G_INTRINSIC);
687   for (DstOp Result : Results)
688     Result.addDefToMIB(*getMRI(), MIB);
689   MIB.addIntrinsicID(ID);
690   return MIB;
691 }
692 
693 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
694                                                  const SrcOp &Op) {
695   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
696 }
697 
698 MachineInstrBuilder MachineIRBuilder::buildFPTrunc(const DstOp &Res,
699                                                    const SrcOp &Op,
700                                                    Optional<unsigned> Flags) {
701   return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op, Flags);
702 }
703 
704 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
705                                                 const DstOp &Res,
706                                                 const SrcOp &Op0,
707                                                 const SrcOp &Op1) {
708   return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
709 }
710 
711 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
712                                                 const DstOp &Res,
713                                                 const SrcOp &Op0,
714                                                 const SrcOp &Op1,
715                                                 Optional<unsigned> Flags) {
716 
717   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags);
718 }
719 
720 MachineInstrBuilder MachineIRBuilder::buildSelect(const DstOp &Res,
721                                                   const SrcOp &Tst,
722                                                   const SrcOp &Op0,
723                                                   const SrcOp &Op1,
724                                                   Optional<unsigned> Flags) {
725 
726   return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1}, Flags);
727 }
728 
729 MachineInstrBuilder
730 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
731                                            const SrcOp &Elt, const SrcOp &Idx) {
732   return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx});
733 }
734 
735 MachineInstrBuilder
736 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
737                                             const SrcOp &Idx) {
738   return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx});
739 }
740 
741 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
742     Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal,
743     Register NewVal, MachineMemOperand &MMO) {
744 #ifndef NDEBUG
745   LLT OldValResTy = getMRI()->getType(OldValRes);
746   LLT SuccessResTy = getMRI()->getType(SuccessRes);
747   LLT AddrTy = getMRI()->getType(Addr);
748   LLT CmpValTy = getMRI()->getType(CmpVal);
749   LLT NewValTy = getMRI()->getType(NewVal);
750   assert(OldValResTy.isScalar() && "invalid operand type");
751   assert(SuccessResTy.isScalar() && "invalid operand type");
752   assert(AddrTy.isPointer() && "invalid operand type");
753   assert(CmpValTy.isValid() && "invalid operand type");
754   assert(NewValTy.isValid() && "invalid operand type");
755   assert(OldValResTy == CmpValTy && "type mismatch");
756   assert(OldValResTy == NewValTy && "type mismatch");
757 #endif
758 
759   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
760       .addDef(OldValRes)
761       .addDef(SuccessRes)
762       .addUse(Addr)
763       .addUse(CmpVal)
764       .addUse(NewVal)
765       .addMemOperand(&MMO);
766 }
767 
768 MachineInstrBuilder
769 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr,
770                                      Register CmpVal, Register NewVal,
771                                      MachineMemOperand &MMO) {
772 #ifndef NDEBUG
773   LLT OldValResTy = getMRI()->getType(OldValRes);
774   LLT AddrTy = getMRI()->getType(Addr);
775   LLT CmpValTy = getMRI()->getType(CmpVal);
776   LLT NewValTy = getMRI()->getType(NewVal);
777   assert(OldValResTy.isScalar() && "invalid operand type");
778   assert(AddrTy.isPointer() && "invalid operand type");
779   assert(CmpValTy.isValid() && "invalid operand type");
780   assert(NewValTy.isValid() && "invalid operand type");
781   assert(OldValResTy == CmpValTy && "type mismatch");
782   assert(OldValResTy == NewValTy && "type mismatch");
783 #endif
784 
785   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG)
786       .addDef(OldValRes)
787       .addUse(Addr)
788       .addUse(CmpVal)
789       .addUse(NewVal)
790       .addMemOperand(&MMO);
791 }
792 
793 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
794   unsigned Opcode, const DstOp &OldValRes,
795   const SrcOp &Addr, const SrcOp &Val,
796   MachineMemOperand &MMO) {
797 
798 #ifndef NDEBUG
799   LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
800   LLT AddrTy = Addr.getLLTTy(*getMRI());
801   LLT ValTy = Val.getLLTTy(*getMRI());
802   assert(OldValResTy.isScalar() && "invalid operand type");
803   assert(AddrTy.isPointer() && "invalid operand type");
804   assert(ValTy.isValid() && "invalid operand type");
805   assert(OldValResTy == ValTy && "type mismatch");
806   assert(MMO.isAtomic() && "not atomic mem operand");
807 #endif
808 
809   auto MIB = buildInstr(Opcode);
810   OldValRes.addDefToMIB(*getMRI(), MIB);
811   Addr.addSrcToMIB(MIB);
812   Val.addSrcToMIB(MIB);
813   MIB.addMemOperand(&MMO);
814   return MIB;
815 }
816 
817 MachineInstrBuilder
818 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
819                                      Register Val, MachineMemOperand &MMO) {
820   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
821                         MMO);
822 }
823 MachineInstrBuilder
824 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
825                                     Register Val, MachineMemOperand &MMO) {
826   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
827                         MMO);
828 }
829 MachineInstrBuilder
830 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
831                                     Register Val, MachineMemOperand &MMO) {
832   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
833                         MMO);
834 }
835 MachineInstrBuilder
836 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
837                                     Register Val, MachineMemOperand &MMO) {
838   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
839                         MMO);
840 }
841 MachineInstrBuilder
842 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
843                                      Register Val, MachineMemOperand &MMO) {
844   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
845                         MMO);
846 }
847 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
848                                                        Register Addr,
849                                                        Register Val,
850                                                        MachineMemOperand &MMO) {
851   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
852                         MMO);
853 }
854 MachineInstrBuilder
855 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
856                                     Register Val, MachineMemOperand &MMO) {
857   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
858                         MMO);
859 }
860 MachineInstrBuilder
861 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
862                                     Register Val, MachineMemOperand &MMO) {
863   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
864                         MMO);
865 }
866 MachineInstrBuilder
867 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
868                                     Register Val, MachineMemOperand &MMO) {
869   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
870                         MMO);
871 }
872 MachineInstrBuilder
873 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
874                                      Register Val, MachineMemOperand &MMO) {
875   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
876                         MMO);
877 }
878 MachineInstrBuilder
879 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
880                                      Register Val, MachineMemOperand &MMO) {
881   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
882                         MMO);
883 }
884 
885 MachineInstrBuilder
886 MachineIRBuilder::buildAtomicRMWFAdd(
887   const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
888   MachineMemOperand &MMO) {
889   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
890                         MMO);
891 }
892 
893 MachineInstrBuilder
894 MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
895                                      MachineMemOperand &MMO) {
896   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
897                         MMO);
898 }
899 
900 MachineInstrBuilder
901 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
902   return buildInstr(TargetOpcode::G_FENCE)
903     .addImm(Ordering)
904     .addImm(Scope);
905 }
906 
907 MachineInstrBuilder
908 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
909 #ifndef NDEBUG
910   assert(getMRI()->getType(Res).isPointer() && "invalid res type");
911 #endif
912 
913   return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA);
914 }
915 
916 void MachineIRBuilder::validateTruncExt(const LLT &DstTy, const LLT &SrcTy,
917                                         bool IsExtend) {
918 #ifndef NDEBUG
919   if (DstTy.isVector()) {
920     assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
921     assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
922            "different number of elements in a trunc/ext");
923   } else
924     assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
925 
926   if (IsExtend)
927     assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
928            "invalid narrowing extend");
929   else
930     assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() &&
931            "invalid widening trunc");
932 #endif
933 }
934 
935 void MachineIRBuilder::validateSelectOp(const LLT &ResTy, const LLT &TstTy,
936                                         const LLT &Op0Ty, const LLT &Op1Ty) {
937 #ifndef NDEBUG
938   assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
939          "invalid operand type");
940   assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
941   if (ResTy.isScalar() || ResTy.isPointer())
942     assert(TstTy.isScalar() && "type mismatch");
943   else
944     assert((TstTy.isScalar() ||
945             (TstTy.isVector() &&
946              TstTy.getNumElements() == Op0Ty.getNumElements())) &&
947            "type mismatch");
948 #endif
949 }
950 
951 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
952                                                  ArrayRef<DstOp> DstOps,
953                                                  ArrayRef<SrcOp> SrcOps,
954                                                  Optional<unsigned> Flags) {
955   switch (Opc) {
956   default:
957     break;
958   case TargetOpcode::G_SELECT: {
959     assert(DstOps.size() == 1 && "Invalid select");
960     assert(SrcOps.size() == 3 && "Invalid select");
961     validateSelectOp(
962         DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()),
963         SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI()));
964     break;
965   }
966   case TargetOpcode::G_ADD:
967   case TargetOpcode::G_AND:
968   case TargetOpcode::G_MUL:
969   case TargetOpcode::G_OR:
970   case TargetOpcode::G_SUB:
971   case TargetOpcode::G_XOR:
972   case TargetOpcode::G_UDIV:
973   case TargetOpcode::G_SDIV:
974   case TargetOpcode::G_UREM:
975   case TargetOpcode::G_SREM:
976   case TargetOpcode::G_SMIN:
977   case TargetOpcode::G_SMAX:
978   case TargetOpcode::G_UMIN:
979   case TargetOpcode::G_UMAX: {
980     // All these are binary ops.
981     assert(DstOps.size() == 1 && "Invalid Dst");
982     assert(SrcOps.size() == 2 && "Invalid Srcs");
983     validateBinaryOp(DstOps[0].getLLTTy(*getMRI()),
984                      SrcOps[0].getLLTTy(*getMRI()),
985                      SrcOps[1].getLLTTy(*getMRI()));
986     break;
987   }
988   case TargetOpcode::G_SHL:
989   case TargetOpcode::G_ASHR:
990   case TargetOpcode::G_LSHR: {
991     assert(DstOps.size() == 1 && "Invalid Dst");
992     assert(SrcOps.size() == 2 && "Invalid Srcs");
993     validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
994                     SrcOps[0].getLLTTy(*getMRI()),
995                     SrcOps[1].getLLTTy(*getMRI()));
996     break;
997   }
998   case TargetOpcode::G_SEXT:
999   case TargetOpcode::G_ZEXT:
1000   case TargetOpcode::G_ANYEXT:
1001     assert(DstOps.size() == 1 && "Invalid Dst");
1002     assert(SrcOps.size() == 1 && "Invalid Srcs");
1003     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1004                      SrcOps[0].getLLTTy(*getMRI()), true);
1005     break;
1006   case TargetOpcode::G_TRUNC:
1007   case TargetOpcode::G_FPTRUNC: {
1008     assert(DstOps.size() == 1 && "Invalid Dst");
1009     assert(SrcOps.size() == 1 && "Invalid Srcs");
1010     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1011                      SrcOps[0].getLLTTy(*getMRI()), false);
1012     break;
1013   }
1014   case TargetOpcode::G_BITCAST: {
1015     assert(DstOps.size() == 1 && "Invalid Dst");
1016     assert(SrcOps.size() == 1 && "Invalid Srcs");
1017     assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1018            SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1019     break;
1020   }
1021   case TargetOpcode::COPY:
1022     assert(DstOps.size() == 1 && "Invalid Dst");
1023     // If the caller wants to add a subreg source it has to be done separately
1024     // so we may not have any SrcOps at this point yet.
1025     break;
1026   case TargetOpcode::G_FCMP:
1027   case TargetOpcode::G_ICMP: {
1028     assert(DstOps.size() == 1 && "Invalid Dst Operands");
1029     assert(SrcOps.size() == 3 && "Invalid Src Operands");
1030     // For F/ICMP, the first src operand is the predicate, followed by
1031     // the two comparands.
1032     assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1033            "Expecting predicate");
1034     assert([&]() -> bool {
1035       CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1036       return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1037                                          : CmpInst::isFPPredicate(Pred);
1038     }() && "Invalid predicate");
1039     assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1040            "Type mismatch");
1041     assert([&]() -> bool {
1042       LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1043       LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1044       if (Op0Ty.isScalar() || Op0Ty.isPointer())
1045         return DstTy.isScalar();
1046       else
1047         return DstTy.isVector() &&
1048                DstTy.getNumElements() == Op0Ty.getNumElements();
1049     }() && "Type Mismatch");
1050     break;
1051   }
1052   case TargetOpcode::G_UNMERGE_VALUES: {
1053     assert(!DstOps.empty() && "Invalid trivial sequence");
1054     assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1055     assert(std::all_of(DstOps.begin(), DstOps.end(),
1056                        [&, this](const DstOp &Op) {
1057                          return Op.getLLTTy(*getMRI()) ==
1058                                 DstOps[0].getLLTTy(*getMRI());
1059                        }) &&
1060            "type mismatch in output list");
1061     assert(DstOps.size() * DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1062                SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1063            "input operands do not cover output register");
1064     break;
1065   }
1066   case TargetOpcode::G_MERGE_VALUES: {
1067     assert(!SrcOps.empty() && "invalid trivial sequence");
1068     assert(DstOps.size() == 1 && "Invalid Dst");
1069     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1070                        [&, this](const SrcOp &Op) {
1071                          return Op.getLLTTy(*getMRI()) ==
1072                                 SrcOps[0].getLLTTy(*getMRI());
1073                        }) &&
1074            "type mismatch in input list");
1075     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1076                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1077            "input operands do not cover output register");
1078     if (SrcOps.size() == 1)
1079       return buildCast(DstOps[0], SrcOps[0]);
1080     if (DstOps[0].getLLTTy(*getMRI()).isVector()) {
1081       if (SrcOps[0].getLLTTy(*getMRI()).isVector())
1082         return buildInstr(TargetOpcode::G_CONCAT_VECTORS, DstOps, SrcOps);
1083       return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps);
1084     }
1085     break;
1086   }
1087   case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1088     assert(DstOps.size() == 1 && "Invalid Dst size");
1089     assert(SrcOps.size() == 2 && "Invalid Src size");
1090     assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1091     assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1092             DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1093            "Invalid operand type");
1094     assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1095     assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1096                DstOps[0].getLLTTy(*getMRI()) &&
1097            "Type mismatch");
1098     break;
1099   }
1100   case TargetOpcode::G_INSERT_VECTOR_ELT: {
1101     assert(DstOps.size() == 1 && "Invalid dst size");
1102     assert(SrcOps.size() == 3 && "Invalid src size");
1103     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1104            SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1105     assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1106                SrcOps[1].getLLTTy(*getMRI()) &&
1107            "Type mismatch");
1108     assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1109     assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() ==
1110                SrcOps[0].getLLTTy(*getMRI()).getNumElements() &&
1111            "Type mismatch");
1112     break;
1113   }
1114   case TargetOpcode::G_BUILD_VECTOR: {
1115     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1116            "Must have at least 2 operands");
1117     assert(DstOps.size() == 1 && "Invalid DstOps");
1118     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1119            "Res type must be a vector");
1120     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1121                        [&, this](const SrcOp &Op) {
1122                          return Op.getLLTTy(*getMRI()) ==
1123                                 SrcOps[0].getLLTTy(*getMRI());
1124                        }) &&
1125            "type mismatch in input list");
1126     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1127                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1128            "input scalars do not exactly cover the output vector register");
1129     break;
1130   }
1131   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1132     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1133            "Must have at least 2 operands");
1134     assert(DstOps.size() == 1 && "Invalid DstOps");
1135     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1136            "Res type must be a vector");
1137     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1138                        [&, this](const SrcOp &Op) {
1139                          return Op.getLLTTy(*getMRI()) ==
1140                                 SrcOps[0].getLLTTy(*getMRI());
1141                        }) &&
1142            "type mismatch in input list");
1143     if (SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1144         DstOps[0].getLLTTy(*getMRI()).getElementType().getSizeInBits())
1145       return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps);
1146     break;
1147   }
1148   case TargetOpcode::G_CONCAT_VECTORS: {
1149     assert(DstOps.size() == 1 && "Invalid DstOps");
1150     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1151            "Must have at least 2 operands");
1152     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1153                        [&, this](const SrcOp &Op) {
1154                          return (Op.getLLTTy(*getMRI()).isVector() &&
1155                                  Op.getLLTTy(*getMRI()) ==
1156                                      SrcOps[0].getLLTTy(*getMRI()));
1157                        }) &&
1158            "type mismatch in input list");
1159     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1160                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1161            "input vectors do not exactly cover the output vector register");
1162     break;
1163   }
1164   case TargetOpcode::G_UADDE: {
1165     assert(DstOps.size() == 2 && "Invalid no of dst operands");
1166     assert(SrcOps.size() == 3 && "Invalid no of src operands");
1167     assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1168     assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1169            (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1170            "Invalid operand");
1171     assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1172     assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1173            "type mismatch");
1174     break;
1175   }
1176   }
1177 
1178   auto MIB = buildInstr(Opc);
1179   for (const DstOp &Op : DstOps)
1180     Op.addDefToMIB(*getMRI(), MIB);
1181   for (const SrcOp &Op : SrcOps)
1182     Op.addSrcToMIB(MIB);
1183   if (Flags)
1184     MIB->setFlags(*Flags);
1185   return MIB;
1186 }
1187