1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file This file implements the LegalizerHelper class to legalize
11 /// individual instructions and the LegalizeMachineIR wrapper pass for the
12 /// primary legalization.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
17 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/Support/Debug.h"
21 #include "llvm/Support/raw_ostream.h"
22 #include "llvm/Target/TargetLowering.h"
23 #include "llvm/Target/TargetSubtargetInfo.h"
24 
25 #include <sstream>
26 
27 #define DEBUG_TYPE "legalize-mir"
28 
29 using namespace llvm;
30 
31 LegalizerHelper::LegalizerHelper(MachineFunction &MF)
32   : MRI(MF.getRegInfo()) {
33   MIRBuilder.setMF(MF);
34 }
35 
36 LegalizerHelper::LegalizeResult
37 LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
38                                    const LegalizerInfo &LegalizerInfo) {
39   auto Action = LegalizerInfo.getAction(MI, MRI);
40   switch (std::get<0>(Action)) {
41   case LegalizerInfo::Legal:
42     return AlreadyLegal;
43   case LegalizerInfo::Libcall:
44     return libcall(MI);
45   case LegalizerInfo::NarrowScalar:
46     return narrowScalar(MI, std::get<1>(Action), std::get<2>(Action));
47   case LegalizerInfo::WidenScalar:
48     return widenScalar(MI, std::get<1>(Action), std::get<2>(Action));
49   case LegalizerInfo::Lower:
50     return lower(MI, std::get<1>(Action), std::get<2>(Action));
51   case LegalizerInfo::FewerElements:
52     return fewerElementsVector(MI, std::get<1>(Action), std::get<2>(Action));
53   default:
54     return UnableToLegalize;
55   }
56 }
57 
58 LegalizerHelper::LegalizeResult
59 LegalizerHelper::legalizeInstr(MachineInstr &MI,
60                                const LegalizerInfo &LegalizerInfo) {
61   SmallVector<MachineInstr *, 4> WorkList;
62   MIRBuilder.recordInsertions(
63       [&](MachineInstr *MI) { WorkList.push_back(MI); });
64   WorkList.push_back(&MI);
65 
66   bool Changed = false;
67   LegalizeResult Res;
68   unsigned Idx = 0;
69   do {
70     Res = legalizeInstrStep(*WorkList[Idx], LegalizerInfo);
71     if (Res == UnableToLegalize) {
72       MIRBuilder.stopRecordingInsertions();
73       return UnableToLegalize;
74     }
75     Changed |= Res == Legalized;
76     ++Idx;
77   } while (Idx < WorkList.size());
78 
79   MIRBuilder.stopRecordingInsertions();
80 
81   return Changed ? Legalized : AlreadyLegal;
82 }
83 
84 void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
85                                    SmallVectorImpl<unsigned> &VRegs) {
86   unsigned Size = Ty.getSizeInBits();
87   SmallVector<uint64_t, 4> Indexes;
88   for (int i = 0; i < NumParts; ++i) {
89     VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
90     Indexes.push_back(i * Size);
91   }
92   MIRBuilder.buildExtract(VRegs, Indexes, Reg);
93 }
94 
95 LegalizerHelper::LegalizeResult
96 LegalizerHelper::libcall(MachineInstr &MI) {
97   LLT Ty = MRI.getType(MI.getOperand(0).getReg());
98   unsigned Size = Ty.getSizeInBits();
99   MIRBuilder.setInstr(MI);
100 
101   switch (MI.getOpcode()) {
102   default:
103     return UnableToLegalize;
104   case TargetOpcode::G_FREM: {
105     auto &Ctx = MIRBuilder.getMF().getFunction()->getContext();
106     Type *Ty = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
107     auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
108     auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
109     const char *Name =
110         TLI.getLibcallName(Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32);
111 
112     CLI.lowerCall(
113         MIRBuilder, MachineOperand::CreateES(Name),
114         {MI.getOperand(0).getReg(), Ty},
115         {{MI.getOperand(1).getReg(), Ty}, {MI.getOperand(2).getReg(), Ty}});
116     MI.eraseFromParent();
117     return Legalized;
118   }
119   }
120 }
121 
122 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
123                                                               unsigned TypeIdx,
124                                                               LLT NarrowTy) {
125   // FIXME: Don't know how to handle secondary types yet.
126   if (TypeIdx != 0)
127     return UnableToLegalize;
128 
129   MIRBuilder.setInstr(MI);
130 
131   switch (MI.getOpcode()) {
132   default:
133     return UnableToLegalize;
134   case TargetOpcode::G_ADD: {
135     // Expand in terms of carry-setting/consuming G_ADDE instructions.
136     unsigned NarrowSize = NarrowTy.getSizeInBits();
137     int NumParts = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() /
138                    NarrowTy.getSizeInBits();
139 
140     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
141     SmallVector<uint64_t, 2> Indexes;
142     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
143     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
144 
145     unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
146     MIRBuilder.buildConstant(CarryIn, 0);
147 
148     for (int i = 0; i < NumParts; ++i) {
149       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
150       unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
151 
152       MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
153                             Src2Regs[i], CarryIn);
154 
155       DstRegs.push_back(DstReg);
156       Indexes.push_back(i * NarrowSize);
157       CarryIn = CarryOut;
158     }
159     unsigned DstReg = MI.getOperand(0).getReg();
160     MIRBuilder.buildSequence(DstReg, DstRegs, Indexes);
161     MI.eraseFromParent();
162     return Legalized;
163   }
164   case TargetOpcode::G_LOAD: {
165     unsigned NarrowSize = NarrowTy.getSizeInBits();
166     int NumParts =
167         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
168     LLT NarrowPtrTy = LLT::pointer(
169         MRI.getType(MI.getOperand(1).getReg()).getAddressSpace(), NarrowSize);
170 
171     SmallVector<unsigned, 2> DstRegs;
172     SmallVector<uint64_t, 2> Indexes;
173     for (int i = 0; i < NumParts; ++i) {
174       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
175       unsigned SrcReg = MRI.createGenericVirtualRegister(NarrowPtrTy);
176       unsigned Offset = MRI.createGenericVirtualRegister(LLT::scalar(64));
177 
178       MIRBuilder.buildConstant(Offset, i * NarrowSize / 8);
179       MIRBuilder.buildGEP(SrcReg, MI.getOperand(1).getReg(), Offset);
180       // TODO: This is conservatively correct, but we probably want to split the
181       // memory operands in the future.
182       MIRBuilder.buildLoad(DstReg, SrcReg, **MI.memoperands_begin());
183 
184       DstRegs.push_back(DstReg);
185       Indexes.push_back(i * NarrowSize);
186     }
187     unsigned DstReg = MI.getOperand(0).getReg();
188     MIRBuilder.buildSequence(DstReg, DstRegs, Indexes);
189     MI.eraseFromParent();
190     return Legalized;
191   }
192   case TargetOpcode::G_STORE: {
193     unsigned NarrowSize = NarrowTy.getSizeInBits();
194     int NumParts =
195         MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() / NarrowSize;
196     LLT NarrowPtrTy = LLT::pointer(
197         MRI.getType(MI.getOperand(1).getReg()).getAddressSpace(), NarrowSize);
198 
199     SmallVector<unsigned, 2> SrcRegs;
200     extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
201 
202     for (int i = 0; i < NumParts; ++i) {
203       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowPtrTy);
204       unsigned Offset = MRI.createGenericVirtualRegister(LLT::scalar(64));
205       MIRBuilder.buildConstant(Offset, i * NarrowSize / 8);
206       MIRBuilder.buildGEP(DstReg, MI.getOperand(1).getReg(), Offset);
207       // TODO: This is conservatively correct, but we probably want to split the
208       // memory operands in the future.
209       MIRBuilder.buildStore(SrcRegs[i], DstReg, **MI.memoperands_begin());
210     }
211     MI.eraseFromParent();
212     return Legalized;
213   }
214   }
215 }
216 
217 LegalizerHelper::LegalizeResult
218 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
219   MIRBuilder.setInstr(MI);
220 
221   switch (MI.getOpcode()) {
222   default:
223     return UnableToLegalize;
224   case TargetOpcode::G_ADD:
225   case TargetOpcode::G_AND:
226   case TargetOpcode::G_MUL:
227   case TargetOpcode::G_OR:
228   case TargetOpcode::G_XOR:
229   case TargetOpcode::G_SUB:
230   case TargetOpcode::G_SHL: {
231     // Perform operation at larger width (any extension is fine here, high bits
232     // don't affect the result) and then truncate the result back to the
233     // original type.
234     unsigned Src1Ext = MRI.createGenericVirtualRegister(WideTy);
235     unsigned Src2Ext = MRI.createGenericVirtualRegister(WideTy);
236     MIRBuilder.buildAnyExt(Src1Ext, MI.getOperand(1).getReg());
237     MIRBuilder.buildAnyExt(Src2Ext, MI.getOperand(2).getReg());
238 
239     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
240     MIRBuilder.buildInstr(MI.getOpcode())
241         .addDef(DstExt)
242         .addUse(Src1Ext)
243         .addUse(Src2Ext);
244 
245     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
246     MI.eraseFromParent();
247     return Legalized;
248   }
249   case TargetOpcode::G_SDIV:
250   case TargetOpcode::G_UDIV:
251   case TargetOpcode::G_ASHR:
252   case TargetOpcode::G_LSHR: {
253     unsigned ExtOp = MI.getOpcode() == TargetOpcode::G_SDIV ||
254                              MI.getOpcode() == TargetOpcode::G_ASHR
255                          ? TargetOpcode::G_SEXT
256                          : TargetOpcode::G_ZEXT;
257 
258     unsigned LHSExt = MRI.createGenericVirtualRegister(WideTy);
259     MIRBuilder.buildInstr(ExtOp).addDef(LHSExt).addUse(
260         MI.getOperand(1).getReg());
261 
262     unsigned RHSExt = MRI.createGenericVirtualRegister(WideTy);
263     MIRBuilder.buildInstr(ExtOp).addDef(RHSExt).addUse(
264         MI.getOperand(2).getReg());
265 
266     unsigned ResExt = MRI.createGenericVirtualRegister(WideTy);
267     MIRBuilder.buildInstr(MI.getOpcode())
268         .addDef(ResExt)
269         .addUse(LHSExt)
270         .addUse(RHSExt);
271 
272     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), ResExt);
273     MI.eraseFromParent();
274     return Legalized;
275   }
276   case TargetOpcode::G_SITOFP:
277   case TargetOpcode::G_UITOFP: {
278     if (TypeIdx != 1)
279       return UnableToLegalize;
280 
281     unsigned Src = MI.getOperand(1).getReg();
282     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
283 
284     if (MI.getOpcode() == TargetOpcode::G_SITOFP) {
285       MIRBuilder.buildSExt(SrcExt, Src);
286     } else {
287       assert(MI.getOpcode() == TargetOpcode::G_UITOFP && "Unexpected conv op");
288       MIRBuilder.buildZExt(SrcExt, Src);
289     }
290 
291     MIRBuilder.buildInstr(MI.getOpcode())
292         .addDef(MI.getOperand(0).getReg())
293         .addUse(SrcExt);
294 
295     MI.eraseFromParent();
296     return Legalized;
297   }
298   case TargetOpcode::G_LOAD: {
299     assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
300                WideTy.getSizeInBits() &&
301            "illegal to increase number of bytes loaded");
302 
303     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
304     MIRBuilder.buildLoad(DstExt, MI.getOperand(1).getReg(),
305                          **MI.memoperands_begin());
306     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
307     MI.eraseFromParent();
308     return Legalized;
309   }
310   case TargetOpcode::G_STORE: {
311     assert(alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) ==
312                WideTy.getSizeInBits() &&
313            "illegal to increase number of bytes modified by a store");
314 
315     unsigned SrcExt = MRI.createGenericVirtualRegister(WideTy);
316     MIRBuilder.buildAnyExt(SrcExt, MI.getOperand(0).getReg());
317     MIRBuilder.buildStore(SrcExt, MI.getOperand(1).getReg(),
318                           **MI.memoperands_begin());
319     MI.eraseFromParent();
320     return Legalized;
321   }
322   case TargetOpcode::G_CONSTANT: {
323     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
324     MIRBuilder.buildConstant(DstExt, *MI.getOperand(1).getCImm());
325     MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), DstExt);
326     MI.eraseFromParent();
327     return Legalized;
328   }
329   case TargetOpcode::G_FCONSTANT: {
330     unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
331     MIRBuilder.buildFConstant(DstExt, *MI.getOperand(1).getFPImm());
332     MIRBuilder.buildFPTrunc(MI.getOperand(0).getReg(), DstExt);
333     MI.eraseFromParent();
334     return Legalized;
335   }
336   case TargetOpcode::G_BRCOND: {
337     unsigned TstExt = MRI.createGenericVirtualRegister(WideTy);
338     MIRBuilder.buildAnyExt(TstExt, MI.getOperand(0).getReg());
339     MIRBuilder.buildBrCond(TstExt, *MI.getOperand(1).getMBB());
340     MI.eraseFromParent();
341     return Legalized;
342   }
343   case TargetOpcode::G_ICMP: {
344     assert(TypeIdx == 1 && "unable to legalize predicate");
345     bool IsSigned = CmpInst::isSigned(
346         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()));
347     unsigned Op0Ext = MRI.createGenericVirtualRegister(WideTy);
348     unsigned Op1Ext = MRI.createGenericVirtualRegister(WideTy);
349     if (IsSigned) {
350       MIRBuilder.buildSExt(Op0Ext, MI.getOperand(2).getReg());
351       MIRBuilder.buildSExt(Op1Ext, MI.getOperand(3).getReg());
352     } else {
353       MIRBuilder.buildZExt(Op0Ext, MI.getOperand(2).getReg());
354       MIRBuilder.buildZExt(Op1Ext, MI.getOperand(3).getReg());
355     }
356     MIRBuilder.buildICmp(
357         static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()),
358         MI.getOperand(0).getReg(), Op0Ext, Op1Ext);
359     MI.eraseFromParent();
360     return Legalized;
361   }
362   case TargetOpcode::G_GEP: {
363     assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
364     unsigned OffsetExt = MRI.createGenericVirtualRegister(WideTy);
365     MIRBuilder.buildSExt(OffsetExt, MI.getOperand(2).getReg());
366     MI.getOperand(2).setReg(OffsetExt);
367     return Legalized;
368   }
369   }
370 }
371 
372 LegalizerHelper::LegalizeResult
373 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
374   using namespace TargetOpcode;
375   MIRBuilder.setInstr(MI);
376 
377   switch(MI.getOpcode()) {
378   default:
379     return UnableToLegalize;
380   case TargetOpcode::G_SREM:
381   case TargetOpcode::G_UREM: {
382     unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
383     MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
384         .addDef(QuotReg)
385         .addUse(MI.getOperand(1).getReg())
386         .addUse(MI.getOperand(2).getReg());
387 
388     unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
389     MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
390     MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(),
391                         ProdReg);
392     MI.eraseFromParent();
393     return Legalized;
394   }
395   }
396 }
397 
398 LegalizerHelper::LegalizeResult
399 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
400                                      LLT NarrowTy) {
401   // FIXME: Don't know how to handle secondary types yet.
402   if (TypeIdx != 0)
403     return UnableToLegalize;
404   switch (MI.getOpcode()) {
405   default:
406     return UnableToLegalize;
407   case TargetOpcode::G_ADD: {
408     unsigned NarrowSize = NarrowTy.getSizeInBits();
409     unsigned DstReg = MI.getOperand(0).getReg();
410     int NumParts = MRI.getType(DstReg).getSizeInBits() / NarrowSize;
411 
412     MIRBuilder.setInstr(MI);
413 
414     SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
415     SmallVector<uint64_t, 2> Indexes;
416     extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
417     extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
418 
419     for (int i = 0; i < NumParts; ++i) {
420       unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
421       MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
422       DstRegs.push_back(DstReg);
423       Indexes.push_back(i * NarrowSize);
424     }
425 
426     MIRBuilder.buildSequence(DstReg, DstRegs, Indexes);
427     MI.eraseFromParent();
428     return Legalized;
429   }
430   }
431 }
432