1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the MachineIRBuidler class.
10 //===----------------------------------------------------------------------===//
11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
13 
14 #include "llvm/CodeGen/MachineFunction.h"
15 #include "llvm/CodeGen/MachineInstr.h"
16 #include "llvm/CodeGen/MachineInstrBuilder.h"
17 #include "llvm/CodeGen/MachineRegisterInfo.h"
18 #include "llvm/CodeGen/TargetInstrInfo.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 #include "llvm/CodeGen/TargetOpcodes.h"
21 #include "llvm/CodeGen/TargetSubtargetInfo.h"
22 #include "llvm/IR/DebugInfo.h"
23 
24 using namespace llvm;
25 
26 void MachineIRBuilder::setMF(MachineFunction &MF) {
27   State.MF = &MF;
28   State.MBB = nullptr;
29   State.MRI = &MF.getRegInfo();
30   State.TII = MF.getSubtarget().getInstrInfo();
31   State.DL = DebugLoc();
32   State.II = MachineBasicBlock::iterator();
33   State.Observer = nullptr;
34 }
35 
36 void MachineIRBuilder::setMBB(MachineBasicBlock &MBB) {
37   State.MBB = &MBB;
38   State.II = MBB.end();
39   assert(&getMF() == MBB.getParent() &&
40          "Basic block is in a different function");
41 }
42 
43 void MachineIRBuilder::setInstr(MachineInstr &MI) {
44   assert(MI.getParent() && "Instruction is not part of a basic block");
45   setMBB(*MI.getParent());
46   State.II = MI.getIterator();
47 }
48 
49 void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; }
50 
51 void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB,
52                                    MachineBasicBlock::iterator II) {
53   assert(MBB.getParent() == &getMF() &&
54          "Basic block is in a different function");
55   State.MBB = &MBB;
56   State.II = II;
57 }
58 
59 void MachineIRBuilder::recordInsertion(MachineInstr *InsertedInstr) const {
60   if (State.Observer)
61     State.Observer->createdInstr(*InsertedInstr);
62 }
63 
64 void MachineIRBuilder::setChangeObserver(GISelChangeObserver &Observer) {
65   State.Observer = &Observer;
66 }
67 
68 void MachineIRBuilder::stopObservingChanges() { State.Observer = nullptr; }
69 
70 //------------------------------------------------------------------------------
71 // Build instruction variants.
72 //------------------------------------------------------------------------------
73 
74 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opcode) {
75   return insertInstr(buildInstrNoInsert(Opcode));
76 }
77 
78 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
79   MachineInstrBuilder MIB = BuildMI(getMF(), getDL(), getTII().get(Opcode));
80   return MIB;
81 }
82 
83 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
84   getMBB().insert(getInsertPt(), MIB);
85   recordInsertion(MIB);
86   return MIB;
87 }
88 
89 MachineInstrBuilder
90 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
91                                       const MDNode *Expr) {
92   assert(isa<DILocalVariable>(Variable) && "not a variable");
93   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
94   assert(
95       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
96       "Expected inlined-at fields to agree");
97   return insertInstr(BuildMI(getMF(), getDL(),
98                              getTII().get(TargetOpcode::DBG_VALUE),
99                              /*IsIndirect*/ false, Reg, Variable, Expr));
100 }
101 
102 MachineInstrBuilder
103 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
104                                         const MDNode *Expr) {
105   assert(isa<DILocalVariable>(Variable) && "not a variable");
106   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
107   assert(
108       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
109       "Expected inlined-at fields to agree");
110   return insertInstr(BuildMI(getMF(), getDL(),
111                              getTII().get(TargetOpcode::DBG_VALUE),
112                              /*IsIndirect*/ true, Reg, Variable, Expr));
113 }
114 
115 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
116                                                       const MDNode *Variable,
117                                                       const MDNode *Expr) {
118   assert(isa<DILocalVariable>(Variable) && "not a variable");
119   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
120   assert(
121       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
122       "Expected inlined-at fields to agree");
123   return buildInstr(TargetOpcode::DBG_VALUE)
124       .addFrameIndex(FI)
125       .addImm(0)
126       .addMetadata(Variable)
127       .addMetadata(Expr);
128 }
129 
130 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
131                                                          const MDNode *Variable,
132                                                          const MDNode *Expr) {
133   assert(isa<DILocalVariable>(Variable) && "not a variable");
134   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
135   assert(
136       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
137       "Expected inlined-at fields to agree");
138   auto MIB = buildInstr(TargetOpcode::DBG_VALUE);
139   if (auto *CI = dyn_cast<ConstantInt>(&C)) {
140     if (CI->getBitWidth() > 64)
141       MIB.addCImm(CI);
142     else
143       MIB.addImm(CI->getZExtValue());
144   } else if (auto *CFP = dyn_cast<ConstantFP>(&C)) {
145     MIB.addFPImm(CFP);
146   } else {
147     // Insert %noreg if we didn't find a usable constant and had to drop it.
148     MIB.addReg(0U);
149   }
150 
151   return MIB.addImm(0).addMetadata(Variable).addMetadata(Expr);
152 }
153 
154 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
155   assert(isa<DILabel>(Label) && "not a label");
156   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
157          "Expected inlined-at fields to agree");
158   auto MIB = buildInstr(TargetOpcode::DBG_LABEL);
159 
160   return MIB.addMetadata(Label);
161 }
162 
163 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(Register Res, int Idx) {
164   assert(getMRI()->getType(Res).isPointer() && "invalid operand type");
165   return buildInstr(TargetOpcode::G_FRAME_INDEX)
166       .addDef(Res)
167       .addFrameIndex(Idx);
168 }
169 
170 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(Register Res,
171                                                        const GlobalValue *GV) {
172   assert(getMRI()->getType(Res).isPointer() && "invalid operand type");
173   assert(getMRI()->getType(Res).getAddressSpace() ==
174              GV->getType()->getAddressSpace() &&
175          "address space mismatch");
176 
177   return buildInstr(TargetOpcode::G_GLOBAL_VALUE)
178       .addDef(Res)
179       .addGlobalAddress(GV);
180 }
181 
182 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
183                                                      unsigned JTI) {
184   return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
185       .addJumpTableIndex(JTI);
186 }
187 
188 void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0,
189                                         const LLT &Op1) {
190   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
191   assert((Res == Op0 && Res == Op1) && "type mismatch");
192 }
193 
194 void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0,
195                                        const LLT &Op1) {
196   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
197   assert((Res == Op0) && "type mismatch");
198 }
199 
200 MachineInstrBuilder MachineIRBuilder::buildGEP(Register Res, Register Op0,
201                                                Register Op1) {
202   assert(getMRI()->getType(Res).isPointer() &&
203          getMRI()->getType(Res) == getMRI()->getType(Op0) && "type mismatch");
204   assert(getMRI()->getType(Op1).isScalar() && "invalid offset type");
205 
206   return buildInstr(TargetOpcode::G_GEP)
207       .addDef(Res)
208       .addUse(Op0)
209       .addUse(Op1);
210 }
211 
212 Optional<MachineInstrBuilder>
213 MachineIRBuilder::materializeGEP(Register &Res, Register Op0,
214                                  const LLT &ValueTy, uint64_t Value) {
215   assert(Res == 0 && "Res is a result argument");
216   assert(ValueTy.isScalar()  && "invalid offset type");
217 
218   if (Value == 0) {
219     Res = Op0;
220     return None;
221   }
222 
223   Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
224   auto Cst = buildConstant(ValueTy, Value);
225   return buildGEP(Res, Op0, Cst.getReg(0));
226 }
227 
228 MachineInstrBuilder MachineIRBuilder::buildPtrMask(Register Res, Register Op0,
229                                                    uint32_t NumBits) {
230   assert(getMRI()->getType(Res).isPointer() &&
231          getMRI()->getType(Res) == getMRI()->getType(Op0) && "type mismatch");
232 
233   return buildInstr(TargetOpcode::G_PTR_MASK)
234       .addDef(Res)
235       .addUse(Op0)
236       .addImm(NumBits);
237 }
238 
239 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
240   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
241 }
242 
243 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
244   assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
245   return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt);
246 }
247 
248 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
249                                                 unsigned JTI,
250                                                 Register IndexReg) {
251   assert(getMRI()->getType(TablePtr).isPointer() &&
252          "Table reg must be a pointer");
253   return buildInstr(TargetOpcode::G_BRJT)
254       .addUse(TablePtr)
255       .addJumpTableIndex(JTI)
256       .addUse(IndexReg);
257 }
258 
259 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
260                                                 const SrcOp &Op) {
261   return buildInstr(TargetOpcode::COPY, Res, Op);
262 }
263 
264 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
265                                                     const ConstantInt &Val) {
266   LLT Ty = Res.getLLTTy(*getMRI());
267   LLT EltTy = Ty.getScalarType();
268   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
269          "creating constant with the wrong size");
270 
271   if (Ty.isVector()) {
272     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
273     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
274     .addCImm(&Val);
275     return buildSplatVector(Res, Const);
276   }
277 
278   auto Const = buildInstr(TargetOpcode::G_CONSTANT);
279   Res.addDefToMIB(*getMRI(), Const);
280   Const.addCImm(&Val);
281   return Const;
282 }
283 
284 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
285                                                     int64_t Val) {
286   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
287                                Res.getLLTTy(*getMRI()).getScalarSizeInBits());
288   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
289   return buildConstant(Res, *CI);
290 }
291 
292 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
293                                                      const ConstantFP &Val) {
294   LLT Ty = Res.getLLTTy(*getMRI());
295   LLT EltTy = Ty.getScalarType();
296 
297   assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
298          == EltTy.getSizeInBits() &&
299          "creating fconstant with the wrong size");
300 
301   assert(!Ty.isPointer() && "invalid operand type");
302 
303   if (Ty.isVector()) {
304     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
305     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
306     .addFPImm(&Val);
307 
308     return buildSplatVector(Res, Const);
309   }
310 
311   auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
312   Res.addDefToMIB(*getMRI(), Const);
313   Const.addFPImm(&Val);
314   return Const;
315 }
316 
317 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
318                                                     const APInt &Val) {
319   ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
320   return buildConstant(Res, *CI);
321 }
322 
323 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
324                                                      double Val) {
325   LLT DstTy = Res.getLLTTy(*getMRI());
326   auto &Ctx = getMF().getFunction().getContext();
327   auto *CFP =
328       ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
329   return buildFConstant(Res, *CFP);
330 }
331 
332 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
333                                                      const APFloat &Val) {
334   auto &Ctx = getMF().getFunction().getContext();
335   auto *CFP = ConstantFP::get(Ctx, Val);
336   return buildFConstant(Res, *CFP);
337 }
338 
339 MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst,
340                                                   MachineBasicBlock &Dest) {
341   assert(getMRI()->getType(Tst).isScalar() && "invalid operand type");
342 
343   return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest);
344 }
345 
346 MachineInstrBuilder MachineIRBuilder::buildLoad(Register Res, Register Addr,
347                                                 MachineMemOperand &MMO) {
348   return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO);
349 }
350 
351 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
352                                                      Register Res,
353                                                      Register Addr,
354                                                      MachineMemOperand &MMO) {
355   assert(getMRI()->getType(Res).isValid() && "invalid operand type");
356   assert(getMRI()->getType(Addr).isPointer() && "invalid operand type");
357 
358   return buildInstr(Opcode)
359       .addDef(Res)
360       .addUse(Addr)
361       .addMemOperand(&MMO);
362 }
363 
364 MachineInstrBuilder MachineIRBuilder::buildStore(Register Val, Register Addr,
365                                                  MachineMemOperand &MMO) {
366   assert(getMRI()->getType(Val).isValid() && "invalid operand type");
367   assert(getMRI()->getType(Addr).isPointer() && "invalid operand type");
368 
369   return buildInstr(TargetOpcode::G_STORE)
370       .addUse(Val)
371       .addUse(Addr)
372       .addMemOperand(&MMO);
373 }
374 
375 MachineInstrBuilder MachineIRBuilder::buildUAddo(const DstOp &Res,
376                                                  const DstOp &CarryOut,
377                                                  const SrcOp &Op0,
378                                                  const SrcOp &Op1) {
379   return buildInstr(TargetOpcode::G_UADDO, {Res, CarryOut}, {Op0, Op1});
380 }
381 
382 MachineInstrBuilder MachineIRBuilder::buildUAdde(const DstOp &Res,
383                                                  const DstOp &CarryOut,
384                                                  const SrcOp &Op0,
385                                                  const SrcOp &Op1,
386                                                  const SrcOp &CarryIn) {
387   return buildInstr(TargetOpcode::G_UADDE, {Res, CarryOut},
388                     {Op0, Op1, CarryIn});
389 }
390 
391 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
392                                                   const SrcOp &Op) {
393   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
394 }
395 
396 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
397                                                 const SrcOp &Op) {
398   return buildInstr(TargetOpcode::G_SEXT, Res, Op);
399 }
400 
401 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
402                                                 const SrcOp &Op) {
403   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
404 }
405 
406 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
407   const auto *TLI = getMF().getSubtarget().getTargetLowering();
408   switch (TLI->getBooleanContents(IsVec, IsFP)) {
409   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
410     return TargetOpcode::G_SEXT;
411   case TargetLoweringBase::ZeroOrOneBooleanContent:
412     return TargetOpcode::G_ZEXT;
413   default:
414     return TargetOpcode::G_ANYEXT;
415   }
416 }
417 
418 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
419                                                    const SrcOp &Op,
420                                                    bool IsFP) {
421   unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
422   return buildInstr(ExtOp, Res, Op);
423 }
424 
425 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
426                                                       const DstOp &Res,
427                                                       const SrcOp &Op) {
428   assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
429           TargetOpcode::G_SEXT == ExtOpc) &&
430          "Expecting Extending Opc");
431   assert(Res.getLLTTy(*getMRI()).isScalar() ||
432          Res.getLLTTy(*getMRI()).isVector());
433   assert(Res.getLLTTy(*getMRI()).isScalar() ==
434          Op.getLLTTy(*getMRI()).isScalar());
435 
436   unsigned Opcode = TargetOpcode::COPY;
437   if (Res.getLLTTy(*getMRI()).getSizeInBits() >
438       Op.getLLTTy(*getMRI()).getSizeInBits())
439     Opcode = ExtOpc;
440   else if (Res.getLLTTy(*getMRI()).getSizeInBits() <
441            Op.getLLTTy(*getMRI()).getSizeInBits())
442     Opcode = TargetOpcode::G_TRUNC;
443   else
444     assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
445 
446   return buildInstr(Opcode, Res, Op);
447 }
448 
449 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
450                                                        const SrcOp &Op) {
451   return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op);
452 }
453 
454 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
455                                                        const SrcOp &Op) {
456   return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op);
457 }
458 
459 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
460                                                          const SrcOp &Op) {
461   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
462 }
463 
464 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
465                                                 const SrcOp &Src) {
466   LLT SrcTy = Src.getLLTTy(*getMRI());
467   LLT DstTy = Dst.getLLTTy(*getMRI());
468   if (SrcTy == DstTy)
469     return buildCopy(Dst, Src);
470 
471   unsigned Opcode;
472   if (SrcTy.isPointer() && DstTy.isScalar())
473     Opcode = TargetOpcode::G_PTRTOINT;
474   else if (DstTy.isPointer() && SrcTy.isScalar())
475     Opcode = TargetOpcode::G_INTTOPTR;
476   else {
477     assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
478     Opcode = TargetOpcode::G_BITCAST;
479   }
480 
481   return buildInstr(Opcode, Dst, Src);
482 }
483 
484 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
485                                                    const SrcOp &Src,
486                                                    uint64_t Index) {
487   LLT SrcTy = Src.getLLTTy(*getMRI());
488   LLT DstTy = Dst.getLLTTy(*getMRI());
489 
490 #ifndef NDEBUG
491   assert(SrcTy.isValid() && "invalid operand type");
492   assert(DstTy.isValid() && "invalid operand type");
493   assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
494          "extracting off end of register");
495 #endif
496 
497   if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
498     assert(Index == 0 && "insertion past the end of a register");
499     return buildCast(Dst, Src);
500   }
501 
502   auto Extract = buildInstr(TargetOpcode::G_EXTRACT);
503   Dst.addDefToMIB(*getMRI(), Extract);
504   Src.addSrcToMIB(Extract);
505   Extract.addImm(Index);
506   return Extract;
507 }
508 
509 void MachineIRBuilder::buildSequence(Register Res, ArrayRef<Register> Ops,
510                                      ArrayRef<uint64_t> Indices) {
511 #ifndef NDEBUG
512   assert(Ops.size() == Indices.size() && "incompatible args");
513   assert(!Ops.empty() && "invalid trivial sequence");
514   assert(std::is_sorted(Indices.begin(), Indices.end()) &&
515          "sequence offsets must be in ascending order");
516 
517   assert(getMRI()->getType(Res).isValid() && "invalid operand type");
518   for (auto Op : Ops)
519     assert(getMRI()->getType(Op).isValid() && "invalid operand type");
520 #endif
521 
522   LLT ResTy = getMRI()->getType(Res);
523   LLT OpTy = getMRI()->getType(Ops[0]);
524   unsigned OpSize = OpTy.getSizeInBits();
525   bool MaybeMerge = true;
526   for (unsigned i = 0; i < Ops.size(); ++i) {
527     if (getMRI()->getType(Ops[i]) != OpTy || Indices[i] != i * OpSize) {
528       MaybeMerge = false;
529       break;
530     }
531   }
532 
533   if (MaybeMerge && Ops.size() * OpSize == ResTy.getSizeInBits()) {
534     buildMerge(Res, Ops);
535     return;
536   }
537 
538   Register ResIn = getMRI()->createGenericVirtualRegister(ResTy);
539   buildUndef(ResIn);
540 
541   for (unsigned i = 0; i < Ops.size(); ++i) {
542     Register ResOut = i + 1 == Ops.size()
543                           ? Res
544                           : getMRI()->createGenericVirtualRegister(ResTy);
545     buildInsert(ResOut, ResIn, Ops[i], Indices[i]);
546     ResIn = ResOut;
547   }
548 }
549 
550 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
551   return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {});
552 }
553 
554 MachineInstrBuilder MachineIRBuilder::buildMerge(const DstOp &Res,
555                                                  ArrayRef<Register> Ops) {
556   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
557   // we need some temporary storage for the DstOp objects. Here we use a
558   // sufficiently large SmallVector to not go through the heap.
559   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
560   return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec);
561 }
562 
563 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
564                                                    const SrcOp &Op) {
565   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
566   // we need some temporary storage for the DstOp objects. Here we use a
567   // sufficiently large SmallVector to not go through the heap.
568   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
569   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
570 }
571 
572 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
573                                                    const SrcOp &Op) {
574   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
575   SmallVector<Register, 8> TmpVec;
576   for (unsigned I = 0; I != NumReg; ++I)
577     TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res));
578   return buildUnmerge(TmpVec, Op);
579 }
580 
581 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
582                                                    const SrcOp &Op) {
583   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
584   // we need some temporary storage for the DstOp objects. Here we use a
585   // sufficiently large SmallVector to not go through the heap.
586   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
587   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
588 }
589 
590 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
591                                                        ArrayRef<Register> Ops) {
592   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
593   // we need some temporary storage for the DstOp objects. Here we use a
594   // sufficiently large SmallVector to not go through the heap.
595   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
596   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
597 }
598 
599 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
600                                                        const SrcOp &Src) {
601   SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
602   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
603 }
604 
605 MachineInstrBuilder
606 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
607                                         ArrayRef<Register> Ops) {
608   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
609   // we need some temporary storage for the DstOp objects. Here we use a
610   // sufficiently large SmallVector to not go through the heap.
611   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
612   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
613 }
614 
615 MachineInstrBuilder
616 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
617   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
618   // we need some temporary storage for the DstOp objects. Here we use a
619   // sufficiently large SmallVector to not go through the heap.
620   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
621   return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec);
622 }
623 
624 MachineInstrBuilder MachineIRBuilder::buildInsert(Register Res, Register Src,
625                                                   Register Op, unsigned Index) {
626   assert(Index + getMRI()->getType(Op).getSizeInBits() <=
627              getMRI()->getType(Res).getSizeInBits() &&
628          "insertion past the end of a register");
629 
630   if (getMRI()->getType(Res).getSizeInBits() ==
631       getMRI()->getType(Op).getSizeInBits()) {
632     return buildCast(Res, Op);
633   }
634 
635   return buildInstr(TargetOpcode::G_INSERT)
636       .addDef(Res)
637       .addUse(Src)
638       .addUse(Op)
639       .addImm(Index);
640 }
641 
642 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
643                                                      ArrayRef<Register> ResultRegs,
644                                                      bool HasSideEffects) {
645   auto MIB =
646       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
647                                 : TargetOpcode::G_INTRINSIC);
648   for (unsigned ResultReg : ResultRegs)
649     MIB.addDef(ResultReg);
650   MIB.addIntrinsicID(ID);
651   return MIB;
652 }
653 
654 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
655                                                      ArrayRef<DstOp> Results,
656                                                      bool HasSideEffects) {
657   auto MIB =
658       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
659                                 : TargetOpcode::G_INTRINSIC);
660   for (DstOp Result : Results)
661     Result.addDefToMIB(*getMRI(), MIB);
662   MIB.addIntrinsicID(ID);
663   return MIB;
664 }
665 
666 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
667                                                  const SrcOp &Op) {
668   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
669 }
670 
671 MachineInstrBuilder MachineIRBuilder::buildFPTrunc(const DstOp &Res,
672                                                    const SrcOp &Op) {
673   return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op);
674 }
675 
676 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
677                                                 const DstOp &Res,
678                                                 const SrcOp &Op0,
679                                                 const SrcOp &Op1) {
680   return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
681 }
682 
683 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
684                                                 const DstOp &Res,
685                                                 const SrcOp &Op0,
686                                                 const SrcOp &Op1) {
687 
688   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1});
689 }
690 
691 MachineInstrBuilder MachineIRBuilder::buildSelect(const DstOp &Res,
692                                                   const SrcOp &Tst,
693                                                   const SrcOp &Op0,
694                                                   const SrcOp &Op1) {
695 
696   return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1});
697 }
698 
699 MachineInstrBuilder
700 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
701                                            const SrcOp &Elt, const SrcOp &Idx) {
702   return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx});
703 }
704 
705 MachineInstrBuilder
706 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
707                                             const SrcOp &Idx) {
708   return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx});
709 }
710 
711 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
712     Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal,
713     Register NewVal, MachineMemOperand &MMO) {
714 #ifndef NDEBUG
715   LLT OldValResTy = getMRI()->getType(OldValRes);
716   LLT SuccessResTy = getMRI()->getType(SuccessRes);
717   LLT AddrTy = getMRI()->getType(Addr);
718   LLT CmpValTy = getMRI()->getType(CmpVal);
719   LLT NewValTy = getMRI()->getType(NewVal);
720   assert(OldValResTy.isScalar() && "invalid operand type");
721   assert(SuccessResTy.isScalar() && "invalid operand type");
722   assert(AddrTy.isPointer() && "invalid operand type");
723   assert(CmpValTy.isValid() && "invalid operand type");
724   assert(NewValTy.isValid() && "invalid operand type");
725   assert(OldValResTy == CmpValTy && "type mismatch");
726   assert(OldValResTy == NewValTy && "type mismatch");
727 #endif
728 
729   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
730       .addDef(OldValRes)
731       .addDef(SuccessRes)
732       .addUse(Addr)
733       .addUse(CmpVal)
734       .addUse(NewVal)
735       .addMemOperand(&MMO);
736 }
737 
738 MachineInstrBuilder
739 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr,
740                                      Register CmpVal, Register NewVal,
741                                      MachineMemOperand &MMO) {
742 #ifndef NDEBUG
743   LLT OldValResTy = getMRI()->getType(OldValRes);
744   LLT AddrTy = getMRI()->getType(Addr);
745   LLT CmpValTy = getMRI()->getType(CmpVal);
746   LLT NewValTy = getMRI()->getType(NewVal);
747   assert(OldValResTy.isScalar() && "invalid operand type");
748   assert(AddrTy.isPointer() && "invalid operand type");
749   assert(CmpValTy.isValid() && "invalid operand type");
750   assert(NewValTy.isValid() && "invalid operand type");
751   assert(OldValResTy == CmpValTy && "type mismatch");
752   assert(OldValResTy == NewValTy && "type mismatch");
753 #endif
754 
755   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG)
756       .addDef(OldValRes)
757       .addUse(Addr)
758       .addUse(CmpVal)
759       .addUse(NewVal)
760       .addMemOperand(&MMO);
761 }
762 
763 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(unsigned Opcode,
764                                                      Register OldValRes,
765                                                      Register Addr,
766                                                      Register Val,
767                                                      MachineMemOperand &MMO) {
768 #ifndef NDEBUG
769   LLT OldValResTy = getMRI()->getType(OldValRes);
770   LLT AddrTy = getMRI()->getType(Addr);
771   LLT ValTy = getMRI()->getType(Val);
772   assert(OldValResTy.isScalar() && "invalid operand type");
773   assert(AddrTy.isPointer() && "invalid operand type");
774   assert(ValTy.isValid() && "invalid operand type");
775   assert(OldValResTy == ValTy && "type mismatch");
776 #endif
777 
778   return buildInstr(Opcode)
779       .addDef(OldValRes)
780       .addUse(Addr)
781       .addUse(Val)
782       .addMemOperand(&MMO);
783 }
784 
785 MachineInstrBuilder
786 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
787                                      Register Val, MachineMemOperand &MMO) {
788   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
789                         MMO);
790 }
791 MachineInstrBuilder
792 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
793                                     Register Val, MachineMemOperand &MMO) {
794   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
795                         MMO);
796 }
797 MachineInstrBuilder
798 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
799                                     Register Val, MachineMemOperand &MMO) {
800   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
801                         MMO);
802 }
803 MachineInstrBuilder
804 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
805                                     Register Val, MachineMemOperand &MMO) {
806   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
807                         MMO);
808 }
809 MachineInstrBuilder
810 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
811                                      Register Val, MachineMemOperand &MMO) {
812   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
813                         MMO);
814 }
815 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
816                                                        Register Addr,
817                                                        Register Val,
818                                                        MachineMemOperand &MMO) {
819   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
820                         MMO);
821 }
822 MachineInstrBuilder
823 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
824                                     Register Val, MachineMemOperand &MMO) {
825   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
826                         MMO);
827 }
828 MachineInstrBuilder
829 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
830                                     Register Val, MachineMemOperand &MMO) {
831   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
832                         MMO);
833 }
834 MachineInstrBuilder
835 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
836                                     Register Val, MachineMemOperand &MMO) {
837   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
838                         MMO);
839 }
840 MachineInstrBuilder
841 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
842                                      Register Val, MachineMemOperand &MMO) {
843   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
844                         MMO);
845 }
846 MachineInstrBuilder
847 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
848                                      Register Val, MachineMemOperand &MMO) {
849   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
850                         MMO);
851 }
852 
853 MachineInstrBuilder
854 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
855   return buildInstr(TargetOpcode::G_FENCE)
856     .addImm(Ordering)
857     .addImm(Scope);
858 }
859 
860 MachineInstrBuilder
861 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
862 #ifndef NDEBUG
863   assert(getMRI()->getType(Res).isPointer() && "invalid res type");
864 #endif
865 
866   return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA);
867 }
868 
869 void MachineIRBuilder::validateTruncExt(const LLT &DstTy, const LLT &SrcTy,
870                                         bool IsExtend) {
871 #ifndef NDEBUG
872   if (DstTy.isVector()) {
873     assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
874     assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
875            "different number of elements in a trunc/ext");
876   } else
877     assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
878 
879   if (IsExtend)
880     assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
881            "invalid narrowing extend");
882   else
883     assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() &&
884            "invalid widening trunc");
885 #endif
886 }
887 
888 void MachineIRBuilder::validateSelectOp(const LLT &ResTy, const LLT &TstTy,
889                                         const LLT &Op0Ty, const LLT &Op1Ty) {
890 #ifndef NDEBUG
891   assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
892          "invalid operand type");
893   assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
894   if (ResTy.isScalar() || ResTy.isPointer())
895     assert(TstTy.isScalar() && "type mismatch");
896   else
897     assert((TstTy.isScalar() ||
898             (TstTy.isVector() &&
899              TstTy.getNumElements() == Op0Ty.getNumElements())) &&
900            "type mismatch");
901 #endif
902 }
903 
904 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
905                                                  ArrayRef<DstOp> DstOps,
906                                                  ArrayRef<SrcOp> SrcOps,
907                                                  Optional<unsigned> Flags) {
908   switch (Opc) {
909   default:
910     break;
911   case TargetOpcode::G_SELECT: {
912     assert(DstOps.size() == 1 && "Invalid select");
913     assert(SrcOps.size() == 3 && "Invalid select");
914     validateSelectOp(
915         DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()),
916         SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI()));
917     break;
918   }
919   case TargetOpcode::G_ADD:
920   case TargetOpcode::G_AND:
921   case TargetOpcode::G_MUL:
922   case TargetOpcode::G_OR:
923   case TargetOpcode::G_SUB:
924   case TargetOpcode::G_XOR:
925   case TargetOpcode::G_UDIV:
926   case TargetOpcode::G_SDIV:
927   case TargetOpcode::G_UREM:
928   case TargetOpcode::G_SREM:
929   case TargetOpcode::G_SMIN:
930   case TargetOpcode::G_SMAX:
931   case TargetOpcode::G_UMIN:
932   case TargetOpcode::G_UMAX: {
933     // All these are binary ops.
934     assert(DstOps.size() == 1 && "Invalid Dst");
935     assert(SrcOps.size() == 2 && "Invalid Srcs");
936     validateBinaryOp(DstOps[0].getLLTTy(*getMRI()),
937                      SrcOps[0].getLLTTy(*getMRI()),
938                      SrcOps[1].getLLTTy(*getMRI()));
939     break;
940   }
941   case TargetOpcode::G_SHL:
942   case TargetOpcode::G_ASHR:
943   case TargetOpcode::G_LSHR: {
944     assert(DstOps.size() == 1 && "Invalid Dst");
945     assert(SrcOps.size() == 2 && "Invalid Srcs");
946     validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
947                     SrcOps[0].getLLTTy(*getMRI()),
948                     SrcOps[1].getLLTTy(*getMRI()));
949     break;
950   }
951   case TargetOpcode::G_SEXT:
952   case TargetOpcode::G_ZEXT:
953   case TargetOpcode::G_ANYEXT:
954     assert(DstOps.size() == 1 && "Invalid Dst");
955     assert(SrcOps.size() == 1 && "Invalid Srcs");
956     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
957                      SrcOps[0].getLLTTy(*getMRI()), true);
958     break;
959   case TargetOpcode::G_TRUNC:
960   case TargetOpcode::G_FPTRUNC: {
961     assert(DstOps.size() == 1 && "Invalid Dst");
962     assert(SrcOps.size() == 1 && "Invalid Srcs");
963     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
964                      SrcOps[0].getLLTTy(*getMRI()), false);
965     break;
966   }
967   case TargetOpcode::COPY:
968     assert(DstOps.size() == 1 && "Invalid Dst");
969     // If the caller wants to add a subreg source it has to be done separately
970     // so we may not have any SrcOps at this point yet.
971     break;
972   case TargetOpcode::G_FCMP:
973   case TargetOpcode::G_ICMP: {
974     assert(DstOps.size() == 1 && "Invalid Dst Operands");
975     assert(SrcOps.size() == 3 && "Invalid Src Operands");
976     // For F/ICMP, the first src operand is the predicate, followed by
977     // the two comparands.
978     assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
979            "Expecting predicate");
980     assert([&]() -> bool {
981       CmpInst::Predicate Pred = SrcOps[0].getPredicate();
982       return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
983                                          : CmpInst::isFPPredicate(Pred);
984     }() && "Invalid predicate");
985     assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
986            "Type mismatch");
987     assert([&]() -> bool {
988       LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
989       LLT DstTy = DstOps[0].getLLTTy(*getMRI());
990       if (Op0Ty.isScalar() || Op0Ty.isPointer())
991         return DstTy.isScalar();
992       else
993         return DstTy.isVector() &&
994                DstTy.getNumElements() == Op0Ty.getNumElements();
995     }() && "Type Mismatch");
996     break;
997   }
998   case TargetOpcode::G_UNMERGE_VALUES: {
999     assert(!DstOps.empty() && "Invalid trivial sequence");
1000     assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1001     assert(std::all_of(DstOps.begin(), DstOps.end(),
1002                        [&, this](const DstOp &Op) {
1003                          return Op.getLLTTy(*getMRI()) ==
1004                                 DstOps[0].getLLTTy(*getMRI());
1005                        }) &&
1006            "type mismatch in output list");
1007     assert(DstOps.size() * DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1008                SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1009            "input operands do not cover output register");
1010     break;
1011   }
1012   case TargetOpcode::G_MERGE_VALUES: {
1013     assert(!SrcOps.empty() && "invalid trivial sequence");
1014     assert(DstOps.size() == 1 && "Invalid Dst");
1015     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1016                        [&, this](const SrcOp &Op) {
1017                          return Op.getLLTTy(*getMRI()) ==
1018                                 SrcOps[0].getLLTTy(*getMRI());
1019                        }) &&
1020            "type mismatch in input list");
1021     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1022                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1023            "input operands do not cover output register");
1024     if (SrcOps.size() == 1)
1025       return buildCast(DstOps[0], SrcOps[0]);
1026     if (DstOps[0].getLLTTy(*getMRI()).isVector())
1027       return buildInstr(TargetOpcode::G_CONCAT_VECTORS, DstOps, SrcOps);
1028     break;
1029   }
1030   case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1031     assert(DstOps.size() == 1 && "Invalid Dst size");
1032     assert(SrcOps.size() == 2 && "Invalid Src size");
1033     assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1034     assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1035             DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1036            "Invalid operand type");
1037     assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1038     assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1039                DstOps[0].getLLTTy(*getMRI()) &&
1040            "Type mismatch");
1041     break;
1042   }
1043   case TargetOpcode::G_INSERT_VECTOR_ELT: {
1044     assert(DstOps.size() == 1 && "Invalid dst size");
1045     assert(SrcOps.size() == 3 && "Invalid src size");
1046     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1047            SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1048     assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1049                SrcOps[1].getLLTTy(*getMRI()) &&
1050            "Type mismatch");
1051     assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1052     assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() ==
1053                SrcOps[0].getLLTTy(*getMRI()).getNumElements() &&
1054            "Type mismatch");
1055     break;
1056   }
1057   case TargetOpcode::G_BUILD_VECTOR: {
1058     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1059            "Must have at least 2 operands");
1060     assert(DstOps.size() == 1 && "Invalid DstOps");
1061     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1062            "Res type must be a vector");
1063     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1064                        [&, this](const SrcOp &Op) {
1065                          return Op.getLLTTy(*getMRI()) ==
1066                                 SrcOps[0].getLLTTy(*getMRI());
1067                        }) &&
1068            "type mismatch in input list");
1069     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1070                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1071            "input scalars do not exactly cover the output vector register");
1072     break;
1073   }
1074   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1075     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1076            "Must have at least 2 operands");
1077     assert(DstOps.size() == 1 && "Invalid DstOps");
1078     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1079            "Res type must be a vector");
1080     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1081                        [&, this](const SrcOp &Op) {
1082                          return Op.getLLTTy(*getMRI()) ==
1083                                 SrcOps[0].getLLTTy(*getMRI());
1084                        }) &&
1085            "type mismatch in input list");
1086     if (SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1087         DstOps[0].getLLTTy(*getMRI()).getElementType().getSizeInBits())
1088       return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps);
1089     break;
1090   }
1091   case TargetOpcode::G_CONCAT_VECTORS: {
1092     assert(DstOps.size() == 1 && "Invalid DstOps");
1093     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1094            "Must have at least 2 operands");
1095     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1096                        [&, this](const SrcOp &Op) {
1097                          return (Op.getLLTTy(*getMRI()).isVector() &&
1098                                  Op.getLLTTy(*getMRI()) ==
1099                                      SrcOps[0].getLLTTy(*getMRI()));
1100                        }) &&
1101            "type mismatch in input list");
1102     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1103                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1104            "input vectors do not exactly cover the output vector register");
1105     break;
1106   }
1107   case TargetOpcode::G_UADDE: {
1108     assert(DstOps.size() == 2 && "Invalid no of dst operands");
1109     assert(SrcOps.size() == 3 && "Invalid no of src operands");
1110     assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1111     assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1112            (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1113            "Invalid operand");
1114     assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1115     assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1116            "type mismatch");
1117     break;
1118   }
1119   }
1120 
1121   auto MIB = buildInstr(Opc);
1122   for (const DstOp &Op : DstOps)
1123     Op.addDefToMIB(*getMRI(), MIB);
1124   for (const SrcOp &Op : SrcOps)
1125     Op.addSrcToMIB(MIB);
1126   if (Flags)
1127     MIB->setFlags(*Flags);
1128   return MIB;
1129 }
1130