1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file implements the LegalizerHelper class to legalize
11 /// individual instructions and the LegalizeMachineIR wrapper pass for the
12 /// primary legalization.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
17 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/CodeGen/TargetSubtargetInfo.h"
22 #include "llvm/Support/Debug.h"
23 #include "llvm/Support/raw_ostream.h"
24 
25 
26 #define DEBUG_TYPE "legalizer"
27 
28 using namespace llvm;
29 using namespace LegalizeActions;
30 
31 LegalizerHelper::LegalizerHelper(MachineFunction &MF)
32     : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()) {
33   MIRBuilder.setMF(MF);
34 }
35 
36 LegalizerHelper::LegalizeResult
37 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
38   DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
39 
40   auto Step = LI.getAction(MI, MRI);
41   switch (Step.Action) {
42   case Legal:
43     DEBUG(dbgs() << ".. Already legal\n");
44     return AlreadyLegal;
45   case Libcall:
46     DEBUG(dbgs() << ".. Convert to libcall\n");
47     return libcall(MI);
48   case NarrowScalar:
49     DEBUG(dbgs() << ".. Narrow scalar\n");
50     return narrowScalar(MI, Step.TypeIdx, Step.NewType);
51   case WidenScalar:
52     DEBUG(dbgs() << ".. Widen scalar\n");
53     return widenScalar(MI, Step.TypeIdx, Step.NewType);
54   case Lower:
55     DEBUG(dbgs() << ".. Lower\n");
56     return lower(MI, Step.TypeIdx, Step.NewType);
57   case FewerElements:
58     DEBUG(dbgs() << ".. Reduce number of elements\n");
59     return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
60   case Custom:
61     DEBUG(dbgs() << ".. Custom legalization\n");
62     return LI.legalizeCustom(MI, MRI, MIRBuilder) ? Legalized
63                                                   : UnableToLegalize;
64   default:
65     DEBUG(dbgs() << ".. Unable to legalize\n");
66     return UnableToLegalize;
67   }
68 }
69 
70 void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
71                                    SmallVectorImpl<unsigned> &VRegs) {
72   for (int i = 0; i < NumParts; ++i)
73     VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
74   MIRBuilder.buildUnmerge(VRegs, Reg);
75 }
76 
77 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
78   switch (Opcode) {
79   case TargetOpcode::G_SDIV:
80     assert(Size == 32 && "Unsupported size");
81     return RTLIB::SDIV_I32;
82   case TargetOpcode::G_UDIV:
83     assert(Size == 32 && "Unsupported size");
84     return RTLIB::UDIV_I32;
85   case TargetOpcode::G_SREM:
86     assert(Size == 32 && "Unsupported size");
87     return RTLIB::SREM_I32;
88   case TargetOpcode::G_UREM:
89     assert(Size == 32 && "Unsupported size");
90     return RTLIB::UREM_I32;
91   case TargetOpcode::G_FADD:
92     assert((Size == 32 || Size == 64) && "Unsupported size");
93     return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
94   case TargetOpcode::G_FSUB:
95     assert((Size == 32 || Size == 64) && "Unsupported size");
96     return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
97   case TargetOpcode::G_FMUL:
98     assert((Size == 32 || Size == 64) && "Unsupported size");
99     return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
100   case TargetOpcode::G_FDIV:
101     assert((Size == 32 || Size == 64) && "Unsupported size");
102     return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
103   case TargetOpcode::G_FREM:
104     return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
105   case TargetOpcode::G_FPOW:
106     return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
107   case TargetOpcode::G_FMA:
108     assert((Size == 32 || Size == 64) && "Unsupported size");
109     return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
110   }
111   llvm_unreachable("Unknown libcall function");
112 }
113 
114 LegalizerHelper::LegalizeResult
115 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
116                     const CallLowering::ArgInfo &Result,
117                     ArrayRef<CallLowering::ArgInfo> Args) {
118   auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
119   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
120   const char *Name = TLI.getLibcallName(Libcall);
121 
122   MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
123   if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
124                      MachineOperand::CreateES(Name), Result, Args))
125     return LegalizerHelper::UnableToLegalize;
126 
127   return LegalizerHelper::Legalized;
128 }
129 
130 // Useful for libcalls where all operands have the same type.
131 static LegalizerHelper::LegalizeResult
132 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
133               Type *OpType) {
134   auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
135 
136   SmallVector<CallLowering::ArgInfo, 3> Args;
137   for (unsigned i = 1; i < MI.getNumOperands(); i++)
138     Args.push_back({MI.getOperand(i).getReg(), OpType});
139   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
140                        Args);
141 }
142 
143 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
144                                        Type *FromType) {
145   auto ToMVT = MVT::getVT(ToType);
146   auto FromMVT = MVT::getVT(FromType);
147 
148   switch (Opcode) {
149   case TargetOpcode::G_FPEXT:
150     return RTLIB::getFPEXT(FromMVT, ToMVT);
151   case TargetOpcode::G_FPTRUNC:
152     return RTLIB::getFPROUND(FromMVT, ToMVT);
153   case TargetOpcode::G_FPTOSI:
154     return RTLIB::getFPTOSINT(FromMVT, ToMVT);
155   case TargetOpcode::G_FPTOUI:
156     return RTLIB::getFPTOUINT(FromMVT, ToMVT);
157   case TargetOpcode::G_SITOFP:
158     return RTLIB::getSINTTOFP(FromMVT, ToMVT);
159   case TargetOpcode::G_UITOFP:
160     return RTLIB::getUINTTOFP(FromMVT, ToMVT);
161   }
162   llvm_unreachable("Unsupported libcall function");
163 }
164 
165 static LegalizerHelper::LegalizeResult
166 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
167                   Type *FromType) {
168   RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
169   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
170                        {{MI.getOperand(1).getReg(), FromType}});
171 }
172 
173 LegalizerHelper::LegalizeResult
174 LegalizerHelper::libcall(MachineInstr &MI) {
175   LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
176   unsigned Size = LLTy.getSizeInBits();
177   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
178 
179   MIRBuilder.setInstr(MI);
180 
181   switch (MI.getOpcode()) {
182   default:
183     return UnableToLegalize;
184   case TargetOpcode::G_SDIV:
185   case TargetOpcode::G_UDIV:
186   case TargetOpcode::G_SREM:
187   case TargetOpcode::G_UREM: {
188     Type *HLTy = Type::getInt32Ty(Ctx);
189     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
190     if (Status != Legalized)
191       return Status;
192     break;
193   }
194   case TargetOpcode::G_FADD:
195   case TargetOpcode::G_FSUB:
196   case TargetOpcode::G_FMUL:
197   case TargetOpcode::G_FDIV:
198   case TargetOpcode::G_FMA:
199   case TargetOpcode::G_FPOW:
200   case TargetOpcode::G_FREM: {
201     Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
202     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
203     if (Status != Legalized)
204       return Status;
205     break;
206   }
207   case TargetOpcode::G_FPEXT: {
208     // FIXME: Support other floating point types (half, fp128 etc)
209     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
210     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
211     if (ToSize != 64 || FromSize != 32)
212       return UnableToLegalize;
213     LegalizeResult Status = conversionLibcall(
214         MI, MIRBuilder, Type::getDoubleTy(Ctx), Type::getFloatTy(Ctx));
215     if (Status != Legalized)
216       return Status;
217     break;
218   }
219   case TargetOpcode::G_FPTRUNC: {
220     // FIXME: Support other floating point types (half, fp128 etc)
221     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
222     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
223     if (ToSize != 32 || FromSize != 64)
224       return UnableToLegalize;
225     LegalizeResult Status = conversionLibcall(
226         MI, MIRBuilder, Type::getFloatTy(Ctx), Type::getDoubleTy(Ctx));
227     if (Status != Legalized)
228       return Status;
229     break;
230   }
231   case TargetOpcode::G_FPTOSI:
232   case TargetOpcode::G_FPTOUI: {
233     // FIXME: Support other types
234     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
235     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
236     if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
237       return UnableToLegalize;
238     LegalizeResult Status = conversionLibcall(
239         MI, MIRBuilder, Type::getInt32Ty(Ctx),
240         FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
241     if (Status != Legalized)
242       return Status;
243     break;
244   }
245   case TargetOpcode::G_SITOFP:
246   case TargetOpcode::G_UITOFP: {
247     // FIXME: Support other types
248     unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
249     unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
250     if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
251       return UnableToLegalize;
252     LegalizeResult Status = conversionLibcall(
253         MI, MIRBuilder,
254         ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
255         Type::getInt32Ty(Ctx));
256     if (Status != Legalized)
257       return Status;
258     break;
259   }
260   }
261 
262   MI.eraseFromParent();
263   return Legalized;
264 }
265 
266 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
267                                                               unsigned TypeIdx,
268                                                               LLT NarrowTy) {
269   // FIXME: Don't know how to handle secondary types yet.
270   if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
271     return UnableToLegalize;
272 
273   MIRBuilder.setInstr(MI);
274 
275   int64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
276   int64_t NarrowSize = NarrowTy.getSizeInBits();
277 
278   switch (MI.getOpcode()) {
279   default:
280     return UnableToLegalize;
281   case TargetOpcode::G_IMPLICIT_DEF: {
282     // FIXME: add support for when SizeOp0 isn't an exact multiple of
283     // NarrowSize.
284     if (SizeOp0 % NarrowSize != 0)
285       return UnableToLegalize;
286     int NumParts = SizeOp0 / NarrowSize;
287 
288     SmallVector<unsigned, 2> DstRegs;
289     for (int i = 0; i < NumParts; ++i) {
290       unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
291       MIRBuilder.buildUndef(Dst);
292       DstRegs.push_back(Dst);
293     }
294     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
295     MI.eraseFromParent();
296     return Legalized;
297   }
298   case TargetOpcode::G_ADD: {
299     // FIXME: add support for when SizeOp0 isn't an exact multiple of
300     // NarrowSize.
301     if (SizeOp0 % NarrowSize != 0)
302       return UnableToLegalize;
303     // Expand in terms of carry-setting/consuming G_ADDE instructions.
304     int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
305 
306     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
307     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
308     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
309 
310     unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
311     MIRBuilder.buildConstant(CarryIn, 0);
312 
313     for (int i = 0; i < NumParts; ++i) {
314       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
315       unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
316 
317       MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
318                             Src2Regs[i], CarryIn);
319 
320       DstRegs.push_back(DstReg);
321       CarryIn = CarryOut;
322     }
323     unsigned DstReg = MI.getOperand(0).getReg();
324     MIRBuilder.buildMerge(DstReg, DstRegs);
325     MI.eraseFromParent();
326     return Legalized;
327   }
328   case TargetOpcode::G_EXTRACT: {
329     if (TypeIdx != 1)
330       return UnableToLegalize;
331 
332     int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
333     // FIXME: add support for when SizeOp1 isn't an exact multiple of
334     // NarrowSize.
335     if (SizeOp1 % NarrowSize != 0)
336       return UnableToLegalize;
337     int NumParts = SizeOp1 / NarrowSize;
338 
339     SmallVector<unsigned, 2> SrcRegs, DstRegs;
340     SmallVector<uint64_t, 2> Indexes;
341     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
342 
343     unsigned OpReg = MI.getOperand(0).getReg();
344     int64_t OpStart = MI.getOperand(2).getImm();
345     int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
346     for (int i = 0; i < NumParts; ++i) {
347       unsigned SrcStart = i * NarrowSize;
348 
349       if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
350         // No part of the extract uses this subregister, ignore it.
351         continue;
352       } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
353         // The entire subregister is extracted, forward the value.
354         DstRegs.push_back(SrcRegs[i]);
355         continue;
356       }
357 
358       // OpSegStart is where this destination segment would start in OpReg if it
359       // extended infinitely in both directions.
360       int64_t ExtractOffset, SegSize;
361       if (OpStart < SrcStart) {
362         ExtractOffset = 0;
363         SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
364       } else {
365         ExtractOffset = OpStart - SrcStart;
366         SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
367       }
368 
369       unsigned SegReg = SrcRegs[i];
370       if (ExtractOffset != 0 || SegSize != NarrowSize) {
371         // A genuine extract is needed.
372         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
373         MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
374       }
375 
376       DstRegs.push_back(SegReg);
377     }
378 
379     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
380     MI.eraseFromParent();
381     return Legalized;
382   }
383   case TargetOpcode::G_INSERT: {
384     // FIXME: add support for when SizeOp0 isn't an exact multiple of
385     // NarrowSize.
386     if (SizeOp0 % NarrowSize != 0)
387       return UnableToLegalize;
388 
389     int NumParts = SizeOp0 / NarrowSize;
390 
391     SmallVector<unsigned, 2> SrcRegs, DstRegs;
392     SmallVector<uint64_t, 2> Indexes;
393     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
394 
395     unsigned OpReg = MI.getOperand(2).getReg();
396     int64_t OpStart = MI.getOperand(3).getImm();
397     int64_t OpSize = MRI.getType(OpReg).getSizeInBits();
398     for (int i = 0; i < NumParts; ++i) {
399       unsigned DstStart = i * NarrowSize;
400 
401       if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
402         // No part of the insert affects this subregister, forward the original.
403         DstRegs.push_back(SrcRegs[i]);
404         continue;
405       } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
406         // The entire subregister is defined by this insert, forward the new
407         // value.
408         DstRegs.push_back(OpReg);
409         continue;
410       }
411 
412       // OpSegStart is where this destination segment would start in OpReg if it
413       // extended infinitely in both directions.
414       int64_t ExtractOffset, InsertOffset, SegSize;
415       if (OpStart < DstStart) {
416         InsertOffset = 0;
417         ExtractOffset = DstStart - OpStart;
418         SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
419       } else {
420         InsertOffset = OpStart - DstStart;
421         ExtractOffset = 0;
422         SegSize =
423             std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
424       }
425 
426       unsigned SegReg = OpReg;
427       if (ExtractOffset != 0 || SegSize != OpSize) {
428         // A genuine extract is needed.
429         SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
430         MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
431       }
432 
433       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
434       MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
435       DstRegs.push_back(DstReg);
436     }
437 
438     assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
439     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
440     MI.eraseFromParent();
441     return Legalized;
442   }
443   case TargetOpcode::G_LOAD: {
444     // FIXME: add support for when SizeOp0 isn't an exact multiple of
445     // NarrowSize.
446     if (SizeOp0 % NarrowSize != 0)
447       return UnableToLegalize;
448     int NumParts = SizeOp0 / NarrowSize;
449     LLT OffsetTy = LLT::scalar(
450         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
451 
452     SmallVector<unsigned, 2> DstRegs;
453     for (int i = 0; i < NumParts; ++i) {
454       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
455       unsigned SrcReg = 0;
456       unsigned Adjustment = i * NarrowSize / 8;
457 
458       MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
459                                 Adjustment);
460 
461       // TODO: This is conservatively correct, but we probably want to split the
462       // memory operands in the future.
463       MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
464 
465       DstRegs.push_back(DstReg);
466     }
467     unsigned DstReg = MI.getOperand(0).getReg();
468     MIRBuilder.buildMerge(DstReg, DstRegs);
469     MI.eraseFromParent();
470     return Legalized;
471   }
472   case TargetOpcode::G_STORE: {
473     // FIXME: add support for when SizeOp0 isn't an exact multiple of
474     // NarrowSize.
475     if (SizeOp0 % NarrowSize != 0)
476       return UnableToLegalize;
477     int NumParts = SizeOp0 / NarrowSize;
478     LLT OffsetTy = LLT::scalar(
479         MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
480 
481     SmallVector<unsigned, 2> SrcRegs;
482     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
483 
484     for (int i = 0; i < NumParts; ++i) {
485       unsigned DstReg = 0;
486       unsigned Adjustment = i * NarrowSize / 8;
487 
488       MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
489                                 Adjustment);
490 
491       // TODO: This is conservatively correct, but we probably want to split the
492       // memory operands in the future.
493       MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
494     }
495     MI.eraseFromParent();
496     return Legalized;
497   }
498   case TargetOpcode::G_CONSTANT: {
499     // FIXME: add support for when SizeOp0 isn't an exact multiple of
500     // NarrowSize.
501     if (SizeOp0 % NarrowSize != 0)
502       return UnableToLegalize;
503     int NumParts = SizeOp0 / NarrowSize;
504     const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
505     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
506 
507     SmallVector<unsigned, 2> DstRegs;
508     for (int i = 0; i < NumParts; ++i) {
509       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
510       ConstantInt *CI =
511           ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
512       MIRBuilder.buildConstant(DstReg, *CI);
513       DstRegs.push_back(DstReg);
514     }
515     unsigned DstReg = MI.getOperand(0).getReg();
516     MIRBuilder.buildMerge(DstReg, DstRegs);
517     MI.eraseFromParent();
518     return Legalized;
519   }
520   case TargetOpcode::G_OR: {
521     // Legalize bitwise operation:
522     // A = BinOp<Ty> B, C
523     // into:
524     // B1, ..., BN = G_UNMERGE_VALUES B
525     // C1, ..., CN = G_UNMERGE_VALUES C
526     // A1 = BinOp<Ty/N> B1, C2
527     // ...
528     // AN = BinOp<Ty/N> BN, CN
529     // A = G_MERGE_VALUES A1, ..., AN
530 
531     // FIXME: add support for when SizeOp0 isn't an exact multiple of
532     // NarrowSize.
533     if (SizeOp0 % NarrowSize != 0)
534       return UnableToLegalize;
535     int NumParts = SizeOp0 / NarrowSize;
536 
537     // List the registers where the destination will be scattered.
538     SmallVector<unsigned, 2> DstRegs;
539     // List the registers where the first argument will be split.
540     SmallVector<unsigned, 2> SrcsReg1;
541     // List the registers where the second argument will be split.
542     SmallVector<unsigned, 2> SrcsReg2;
543     // Create all the temporary registers.
544     for (int i = 0; i < NumParts; ++i) {
545       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
546       unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
547       unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
548 
549       DstRegs.push_back(DstReg);
550       SrcsReg1.push_back(SrcReg1);
551       SrcsReg2.push_back(SrcReg2);
552     }
553     // Explode the big arguments into smaller chunks.
554     MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
555     MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
556 
557     // Do the operation on each small part.
558     for (int i = 0; i < NumParts; ++i)
559       MIRBuilder.buildOr(DstRegs[i], SrcsReg1[i], SrcsReg2[i]);
560 
561     // Gather the destination registers into the final destination.
562     unsigned DstReg = MI.getOperand(0).getReg();
563     MIRBuilder.buildMerge(DstReg, DstRegs);
564     MI.eraseFromParent();
565     return Legalized;
566   }
567   }
568 }
569 
570 LegalizerHelper::LegalizeResult
571 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
572   MIRBuilder.setInstr(MI);
573 
574   switch (MI.getOpcode()) {
575   default:
576     return UnableToLegalize;
577   case TargetOpcode::G_ADD:
578   case TargetOpcode::G_AND:
579   case TargetOpcode::G_MUL:
580   case TargetOpcode::G_OR:
581   case TargetOpcode::G_XOR:
582   case TargetOpcode::G_SUB:
583   case TargetOpcode::G_SHL: {
584     // Perform operation at larger width (any extension is fine here, high bits
585     // don't affect the result) and then truncate the result back to the
586     // original type.
587     unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
588     unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
589     MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
590     MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
591 
592     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
593     MIRBuilder.buildInstr(MI.getOpcode())
594         .addDef(DstExt)
595         .addUse(Src1Ext)
596         .addUse(Src2Ext);
597 
598     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
599     MI.eraseFromParent();
600     return Legalized;
601   }
602   case TargetOpcode::G_SDIV:
603   case TargetOpcode::G_UDIV:
604   case TargetOpcode::G_SREM:
605   case TargetOpcode::G_UREM:
606   case TargetOpcode::G_ASHR:
607   case TargetOpcode::G_LSHR: {
608     unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
609                              MI.getOpcode() == TargetOpcode::G_SREM ||
610                              MI.getOpcode() == TargetOpcode::G_ASHR
611                          ? TargetOpcode::G_SEXT
612                          : TargetOpcode::G_ZEXT;
613 
614     unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
615     MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
616         MI.getOperand(1).getReg());
617 
618     unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
619     MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
620         MI.getOperand(2).getReg());
621 
622     unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
623     MIRBuilder.buildInstr(MI.getOpcode())
624         .addDef(ResExt)
625         .addUse(LHSExt)
626         .addUse(RHSExt);
627 
628     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
629     MI.eraseFromParent();
630     return Legalized;
631   }
632   case TargetOpcode::G_SELECT: {
633     if (TypeIdx != 0)
634       return UnableToLegalize;
635 
636     // Perform operation at larger width (any extension is fine here, high bits
637     // don't affect the result) and then truncate the result back to the
638     // original type.
639     unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
640     unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
641     MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(2).getReg());
642     MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(3).getReg());
643 
644     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
645     MIRBuilder.buildInstr(TargetOpcode::G_SELECT)
646         .addDef(DstExt)
647         .addReg(MI.getOperand(1).getReg())
648         .addUse(Src1Ext)
649         .addUse(Src2Ext);
650 
651     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
652     MI.eraseFromParent();
653     return Legalized;
654   }
655   case TargetOpcode::G_FPTOSI:
656   case TargetOpcode::G_FPTOUI: {
657     if (TypeIdx != 0)
658       return UnableToLegalize;
659 
660     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
661     MIRBuilder.buildInstr(MI.getOpcode())
662         .addDef(DstExt)
663         .addUse(MI.getOperand(1).getReg());
664 
665     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
666     MI.eraseFromParent();
667     return Legalized;
668   }
669   case TargetOpcode::G_SITOFP:
670   case TargetOpcode::G_UITOFP: {
671     if (TypeIdx != 1)
672       return UnableToLegalize;
673 
674     unsigned Src = MI.getOperand(1).getReg();
675     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
676 
677     if (MI.getOpcode() == TargetOpcode::G_SITOFP) {
678       MIRBuilder.buildSExt(SrcExt, Src);
679     } else {
680       assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
681       MIRBuilder.buildZExt(SrcExt, Src);
682     }
683 
684     MIRBuilder.buildInstr(MI.getOpcode())
685         .addDef(MI.getOperand(0).getReg())
686         .addUse(SrcExt);
687 
688     MI.eraseFromParent();
689     return Legalized;
690   }
691   case TargetOpcode::G_INSERT: {
692     if (TypeIdx != 0)
693       return UnableToLegalize;
694 
695     unsigned Src = MI.getOperand(1).getReg();
696     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
697     MIRBuilder.buildAnyExt(SrcExt, Src);
698 
699     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
700     auto MIB = MIRBuilder.buildInsert(DstExt, SrcExt, MI.getOperand(2).getReg(),
701                                       MI.getOperand(3).getImm());
702     for (unsigned OpNum = 4; OpNum < MI.getNumOperands(); OpNum += 2) {
703       MIB.addReg(MI.getOperand(OpNum).getReg());
704       MIB.addImm(MI.getOperand(OpNum + 1).getImm());
705     }
706 
707     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
708     MI.eraseFromParent();
709     return Legalized;
710   }
711   case TargetOpcode::G_LOAD: {
712     // For some types like i24, we might try to widen to i32. To properly handle
713     // this we should be using a dedicated extending load, until then avoid
714     // trying to legalize.
715     if (alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) !=
716         WideTy.getSizeInBits())
717       return UnableToLegalize;
718 
719     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
720     MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
721                          **MI.memoperands_begin());
722     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
723     MI.eraseFromParent();
724     return Legalized;
725   }
726   case TargetOpcode::G_STORE: {
727     if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
728         WideTy != LLT::scalar(8))
729       return UnableToLegalize;
730 
731     auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
732     auto Content = TLI.getBooleanContents(false, false);
733 
734     unsigned ExtOp = TargetOpcode::G_ANYEXT;
735     if (Content == TargetLoweringBase::ZeroOrOneBooleanContent)
736       ExtOp = TargetOpcode::G_ZEXT;
737     else if (Content == TargetLoweringBase::ZeroOrNegativeOneBooleanContent)
738       ExtOp = TargetOpcode::G_SEXT;
739     else
740       ExtOp = TargetOpcode::G_ANYEXT;
741 
742     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
743     MIRBuilder.buildInstr(ExtOp).addDef(SrcExt).addUse(
744         MI.getOperand(0).getReg());
745     MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
746                           **MI.memoperands_begin());
747     MI.eraseFromParent();
748     return Legalized;
749   }
750   case TargetOpcode::G_CONSTANT: {
751     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
752     MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
753     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
754     MI.eraseFromParent();
755     return Legalized;
756   }
757   case TargetOpcode::G_FCONSTANT: {
758     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
759     const ConstantFP *CFP = MI.getOperand(1).getFPImm();
760     APFloat Val = CFP->getValueAPF();
761     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
762     auto LLT2Sem = [](LLT Ty) {
763       switch (Ty.getSizeInBits()) {
764       case 32:
765         return &APFloat::IEEEsingle();
766         break;
767       case 64:
768         return &APFloat::IEEEdouble();
769         break;
770       default:
771         llvm_unreachable("Unhandled fp widen type");
772       }
773     };
774     bool LosesInfo;
775     Val.convert(*LLT2Sem(WideTy), APFloat::rmTowardZero, &LosesInfo);
776     MIRBuilder.buildFConstant(DstExt, *ConstantFP::get(Ctx, Val));
777     MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
778     MI.eraseFromParent();
779     return Legalized;
780   }
781   case TargetOpcode::G_BRCOND: {
782     unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
783     MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
784     MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
785     MI.eraseFromParent();
786     return Legalized;
787   }
788   case TargetOpcode::G_FCMP: {
789     unsigned Op0Ext, Op1Ext, DstReg;
790     unsigned Cmp1 = MI.getOperand(2).getReg();
791     unsigned Cmp2 = MI.getOperand(3).getReg();
792     if (TypeIdx == 0) {
793       Op0Ext = Cmp1;
794       Op1Ext = Cmp2;
795       DstReg = MRI.createGenericVirtualRegister(WideTy);
796     } else {
797       Op0Ext = MRI.createGenericVirtualRegister(WideTy);
798       Op1Ext = MRI.createGenericVirtualRegister(WideTy);
799       DstReg = MI.getOperand(0).getReg();
800       MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op0Ext, Cmp1);
801       MIRBuilder.buildInstr(TargetOpcode::G_FPEXT, Op1Ext, Cmp2);
802     }
803     MIRBuilder.buildFCmp(
804         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
805         DstReg, Op0Ext, Op1Ext);
806     if (TypeIdx == 0)
807       MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
808                             DstReg);
809     MI.eraseFromParent();
810     return Legalized;
811   }
812   case TargetOpcode::G_ICMP: {
813     bool IsSigned = CmpInst::isSigned(
814         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
815     unsigned Cmp1 = MI.getOperand(2).getReg();
816     unsigned Cmp2 = MI.getOperand(3).getReg();
817     unsigned Op0Ext, Op1Ext, DstReg;
818     if (TypeIdx == 0) {
819       Op0Ext = Cmp1;
820       Op1Ext = Cmp2;
821       DstReg = MRI.createGenericVirtualRegister(WideTy);
822     } else {
823       Op0Ext = MRI.createGenericVirtualRegister(WideTy);
824       Op1Ext = MRI.createGenericVirtualRegister(WideTy);
825       DstReg = MI.getOperand(0).getReg();
826       if (IsSigned) {
827         MIRBuilder.buildSExt(Op0Ext, Cmp1);
828         MIRBuilder.buildSExt(Op1Ext, Cmp2);
829       } else {
830         MIRBuilder.buildZExt(Op0Ext, Cmp1);
831         MIRBuilder.buildZExt(Op1Ext, Cmp2);
832       }
833     }
834     MIRBuilder.buildICmp(
835         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
836         DstReg, Op0Ext, Op1Ext);
837     if (TypeIdx == 0)
838       MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, MI.getOperand(0).getReg(),
839                             DstReg);
840     MI.eraseFromParent();
841     return Legalized;
842   }
843   case TargetOpcode::G_GEP: {
844     assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
845     unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
846     MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
847     MI.getOperand(2).setReg(OffsetExt);
848     return Legalized;
849   }
850   case TargetOpcode::G_PHI: {
851     assert(TypeIdx == 0 && "Expecting only Idx 0");
852     auto getExtendedReg = [&](unsigned Reg, MachineBasicBlock &MBB) {
853       auto FirstTermIt = MBB.getFirstTerminator();
854       MIRBuilder.setInsertPt(MBB, FirstTermIt);
855       MachineInstr *DefMI = MRI.getVRegDef(Reg);
856       MachineInstrBuilder MIB;
857       if (DefMI->getOpcode() == TargetOpcode::G_TRUNC)
858         MIB = MIRBuilder.buildAnyExtOrTrunc(WideTy,
859                                             DefMI->getOperand(1).getReg());
860       else
861         MIB = MIRBuilder.buildAnyExt(WideTy, Reg);
862       return MIB->getOperand(0).getReg();
863     };
864     auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, WideTy);
865     for (auto OpIt = MI.operands_begin() + 1, OpE = MI.operands_end();
866          OpIt != OpE;) {
867       unsigned Reg = OpIt++->getReg();
868       MachineBasicBlock *OpMBB = OpIt++->getMBB();
869       MIB.addReg(getExtendedReg(Reg, *OpMBB));
870       MIB.addMBB(OpMBB);
871     }
872     auto *MBB = MI.getParent();
873     MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
874     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(),
875                           MIB->getOperand(0).getReg());
876     MI.eraseFromParent();
877     return Legalized;
878   }
879   }
880 }
881 
882 LegalizerHelper::LegalizeResult
883 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
884   using namespace TargetOpcode;
885   MIRBuilder.setInstr(MI);
886 
887   switch(MI.getOpcode()) {
888   default:
889     return UnableToLegalize;
890   case TargetOpcode::G_SREM:
891   case TargetOpcode::G_UREM: {
892     unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
893     MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
894         .addDef(QuotReg)
895         .addUse(MI.getOperand(1).getReg())
896         .addUse(MI.getOperand(2).getReg());
897 
898     unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
899     MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
900     MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
901                         ProdReg);
902     MI.eraseFromParent();
903     return Legalized;
904   }
905   case TargetOpcode::G_SMULO:
906   case TargetOpcode::G_UMULO: {
907     // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
908     // result.
909     unsigned Res = MI.getOperand(0).getReg();
910     unsigned Overflow = MI.getOperand(1).getReg();
911     unsigned LHS = MI.getOperand(2).getReg();
912     unsigned RHS = MI.getOperand(3).getReg();
913 
914     MIRBuilder.buildMul(Res, LHS, RHS);
915 
916     unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
917                           ? TargetOpcode::G_SMULH
918                           : TargetOpcode::G_UMULH;
919 
920     unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
921     MIRBuilder.buildInstr(Opcode)
922       .addDef(HiPart)
923       .addUse(LHS)
924       .addUse(RHS);
925 
926     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
927     MIRBuilder.buildConstant(Zero, 0);
928 
929     // For *signed* multiply, overflow is detected by checking:
930     // (hi != (lo >> bitwidth-1))
931     if (Opcode == TargetOpcode::G_SMULH) {
932       unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
933       unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
934       MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
935       MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
936         .addDef(Shifted)
937         .addUse(Res)
938         .addUse(ShiftAmt);
939       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
940     } else {
941       MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
942     }
943     MI.eraseFromParent();
944     return Legalized;
945   }
946   case TargetOpcode::G_FNEG: {
947     // TODO: Handle vector types once we are able to
948     // represent them.
949     if (Ty.isVector())
950       return UnableToLegalize;
951     unsigned Res = MI.getOperand(0).getReg();
952     Type *ZeroTy;
953     LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
954     switch (Ty.getSizeInBits()) {
955     case 16:
956       ZeroTy = Type::getHalfTy(Ctx);
957       break;
958     case 32:
959       ZeroTy = Type::getFloatTy(Ctx);
960       break;
961     case 64:
962       ZeroTy = Type::getDoubleTy(Ctx);
963       break;
964     case 128:
965       ZeroTy = Type::getFP128Ty(Ctx);
966       break;
967     default:
968       llvm_unreachable("unexpected floating-point type");
969     }
970     ConstantFP &ZeroForNegation =
971         *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
972     unsigned Zero = MRI.createGenericVirtualRegister(Ty);
973     MIRBuilder.buildFConstant(Zero, ZeroForNegation);
974     MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
975         .addDef(Res)
976         .addUse(Zero)
977         .addUse(MI.getOperand(1).getReg());
978     MI.eraseFromParent();
979     return Legalized;
980   }
981   case TargetOpcode::G_FSUB: {
982     // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
983     // First, check if G_FNEG is marked as Lower. If so, we may
984     // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
985     if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
986       return UnableToLegalize;
987     unsigned Res = MI.getOperand(0).getReg();
988     unsigned LHS = MI.getOperand(1).getReg();
989     unsigned RHS = MI.getOperand(2).getReg();
990     unsigned Neg = MRI.createGenericVirtualRegister(Ty);
991     MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
992     MIRBuilder.buildInstr(TargetOpcode::G_FADD)
993         .addDef(Res)
994         .addUse(LHS)
995         .addUse(Neg);
996     MI.eraseFromParent();
997     return Legalized;
998   }
999   case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1000     unsigned OldValRes = MI.getOperand(0).getReg();
1001     unsigned SuccessRes = MI.getOperand(1).getReg();
1002     unsigned Addr = MI.getOperand(2).getReg();
1003     unsigned CmpVal = MI.getOperand(3).getReg();
1004     unsigned NewVal = MI.getOperand(4).getReg();
1005     MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1006                                   **MI.memoperands_begin());
1007     MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1008     MI.eraseFromParent();
1009     return Legalized;
1010   }
1011   }
1012 }
1013 
1014 LegalizerHelper::LegalizeResult
1015 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
1016                                      LLT NarrowTy) {
1017   // FIXME: Don't know how to handle secondary types yet.
1018   if (TypeIdx != 0)
1019     return UnableToLegalize;
1020   switch (MI.getOpcode()) {
1021   default:
1022     return UnableToLegalize;
1023   case TargetOpcode::G_ADD: {
1024     unsigned NarrowSize = NarrowTy.getSizeInBits();
1025     unsigned DstReg = MI.getOperand(0).getReg();
1026     unsigned Size = MRI.getType(DstReg).getSizeInBits();
1027     int NumParts = Size / NarrowSize;
1028     // FIXME: Don't know how to handle the situation where the small vectors
1029     // aren't all the same size yet.
1030     if (Size % NarrowSize != 0)
1031       return UnableToLegalize;
1032 
1033     MIRBuilder.setInstr(MI);
1034 
1035     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
1036     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
1037     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
1038 
1039     for (int i = 0; i < NumParts; ++i) {
1040       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
1041       MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
1042       DstRegs.push_back(DstReg);
1043     }
1044 
1045     MIRBuilder.buildMerge(DstReg, DstRegs);
1046     MI.eraseFromParent();
1047     return Legalized;
1048   }
1049   }
1050 }
1051