1 //===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 /// \file
10 /// This file implements the targeting of the InstructionSelector class for
11 /// AMDGPU.
12 /// \todo This should be generated by TableGen.
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUInstructionSelector.h"
16 #include "AMDGPUInstrInfo.h"
17 #include "AMDGPURegisterBankInfo.h"
18 #include "AMDGPURegisterInfo.h"
19 #include "AMDGPUSubtarget.h"
20 #include "llvm/CodeGen/GlobalISel/Utils.h"
21 #include "llvm/CodeGen/MachineBasicBlock.h"
22 #include "llvm/CodeGen/MachineFunction.h"
23 #include "llvm/CodeGen/MachineInstr.h"
24 #include "llvm/CodeGen/MachineInstrBuilder.h"
25 #include "llvm/CodeGen/MachineRegisterInfo.h"
26 #include "llvm/IR/Type.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Support/raw_ostream.h"
29 
30 #define DEBUG_TYPE "amdgpu-isel"
31 
32 using namespace llvm;
33 
34 AMDGPUInstructionSelector::AMDGPUInstructionSelector(
35     const SISubtarget &STI, const AMDGPURegisterBankInfo &RBI)
36     : InstructionSelector(), TII(*STI.getInstrInfo()),
37       TRI(*STI.getRegisterInfo()), RBI(RBI), AMDGPUASI(STI.getAMDGPUAS()) {}
38 
39 MachineOperand
40 AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
41                                            unsigned SubIdx) const {
42 
43   MachineInstr *MI = MO.getParent();
44   MachineBasicBlock *BB = MO.getParent()->getParent();
45   MachineFunction *MF = BB->getParent();
46   MachineRegisterInfo &MRI = MF->getRegInfo();
47   unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
48 
49   if (MO.isReg()) {
50     unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx);
51     unsigned Reg = MO.getReg();
52     BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg)
53             .addReg(Reg, 0, ComposedSubIdx);
54 
55     return MachineOperand::CreateReg(DstReg, MO.isDef(), MO.isImplicit(),
56                                      MO.isKill(), MO.isDead(), MO.isUndef(),
57                                      MO.isEarlyClobber(), 0, MO.isDebug(),
58                                      MO.isInternalRead());
59   }
60 
61   assert(MO.isImm());
62 
63   APInt Imm(64, MO.getImm());
64 
65   switch (SubIdx) {
66   default:
67     llvm_unreachable("do not know to split immediate with this sub index.");
68   case AMDGPU::sub0:
69     return MachineOperand::CreateImm(Imm.getLoBits(32).getSExtValue());
70   case AMDGPU::sub1:
71     return MachineOperand::CreateImm(Imm.getHiBits(32).getSExtValue());
72   }
73 }
74 
75 bool AMDGPUInstructionSelector::selectG_ADD(MachineInstr &I) const {
76   MachineBasicBlock *BB = I.getParent();
77   MachineFunction *MF = BB->getParent();
78   MachineRegisterInfo &MRI = MF->getRegInfo();
79   unsigned Size = RBI.getSizeInBits(I.getOperand(0).getReg(), MRI, TRI);
80   unsigned DstLo = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
81   unsigned DstHi = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
82 
83   if (Size != 64)
84     return false;
85 
86   DebugLoc DL = I.getDebugLoc();
87 
88   MachineOperand Lo1(getSubOperand64(I.getOperand(1), AMDGPU::sub0));
89   MachineOperand Lo2(getSubOperand64(I.getOperand(2), AMDGPU::sub0));
90 
91   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
92           .add(Lo1)
93           .add(Lo2);
94 
95   MachineOperand Hi1(getSubOperand64(I.getOperand(1), AMDGPU::sub1));
96   MachineOperand Hi2(getSubOperand64(I.getOperand(2), AMDGPU::sub1));
97 
98   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi)
99           .add(Hi1)
100           .add(Hi2);
101 
102   BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), I.getOperand(0).getReg())
103           .addReg(DstLo)
104           .addImm(AMDGPU::sub0)
105           .addReg(DstHi)
106           .addImm(AMDGPU::sub1);
107 
108   for (MachineOperand &MO : I.explicit_operands()) {
109     if (!MO.isReg() || TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
110       continue;
111     RBI.constrainGenericRegister(MO.getReg(), AMDGPU::SReg_64RegClass, MRI);
112   }
113 
114   I.eraseFromParent();
115   return true;
116 }
117 
118 bool AMDGPUInstructionSelector::selectG_GEP(MachineInstr &I) const {
119   return selectG_ADD(I);
120 }
121 
122 bool AMDGPUInstructionSelector::selectG_STORE(MachineInstr &I) const {
123   MachineBasicBlock *BB = I.getParent();
124   DebugLoc DL = I.getDebugLoc();
125 
126   // FIXME: Select store instruction based on address space
127   MachineInstr *Flat = BuildMI(*BB, &I, DL, TII.get(AMDGPU::FLAT_STORE_DWORD))
128           .add(I.getOperand(1))
129           .add(I.getOperand(0))
130           .addImm(0)  // offset
131           .addImm(0)  // glc
132           .addImm(0); // slc
133 
134 
135   // Now that we selected an opcode, we need to constrain the register
136   // operands to use appropriate classes.
137   bool Ret = constrainSelectedInstRegOperands(*Flat, TII, TRI, RBI);
138 
139   I.eraseFromParent();
140   return Ret;
141 }
142 
143 bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
144   MachineBasicBlock *BB = I.getParent();
145   MachineFunction *MF = BB->getParent();
146   MachineRegisterInfo &MRI = MF->getRegInfo();
147   unsigned DstReg = I.getOperand(0).getReg();
148   unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
149 
150   if (Size == 32) {
151     I.setDesc(TII.get(AMDGPU::S_MOV_B32));
152     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
153   }
154 
155   assert(Size == 64);
156 
157   DebugLoc DL = I.getDebugLoc();
158   unsigned LoReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
159   unsigned HiReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
160   const APInt &Imm = I.getOperand(1).getCImm()->getValue();
161 
162   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), LoReg)
163           .addImm(Imm.trunc(32).getZExtValue());
164 
165   BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg)
166           .addImm(Imm.ashr(32).getZExtValue());
167 
168   BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
169           .addReg(LoReg)
170           .addImm(AMDGPU::sub0)
171           .addReg(HiReg)
172           .addImm(AMDGPU::sub1);
173   // We can't call constrainSelectedInstRegOperands here, because it doesn't
174   // work for target independent opcodes
175   I.eraseFromParent();
176   return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, MRI);
177 }
178 
179 static bool isConstant(const MachineInstr &MI) {
180   return MI.getOpcode() == TargetOpcode::G_CONSTANT;
181 }
182 
183 void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load,
184     const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const {
185 
186   const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg());
187 
188   assert(PtrMI);
189 
190   if (PtrMI->getOpcode() != TargetOpcode::G_GEP)
191     return;
192 
193   GEPInfo GEPInfo(*PtrMI);
194 
195   for (unsigned i = 1, e = 3; i < e; ++i) {
196     const MachineOperand &GEPOp = PtrMI->getOperand(i);
197     const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg());
198     assert(OpDef);
199     if (isConstant(*OpDef)) {
200       // FIXME: Is it possible to have multiple Imm parts?  Maybe if we
201       // are lacking other optimizations.
202       assert(GEPInfo.Imm == 0);
203       GEPInfo.Imm = OpDef->getOperand(1).getCImm()->getSExtValue();
204       continue;
205     }
206     const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI);
207     if (OpBank->getID() == AMDGPU::SGPRRegBankID)
208       GEPInfo.SgprParts.push_back(GEPOp.getReg());
209     else
210       GEPInfo.VgprParts.push_back(GEPOp.getReg());
211   }
212 
213   AddrInfo.push_back(GEPInfo);
214   getAddrModeInfo(*PtrMI, MRI, AddrInfo);
215 }
216 
217 static bool isInstrUniform(const MachineInstr &MI) {
218   if (!MI.hasOneMemOperand())
219     return false;
220 
221   const MachineMemOperand *MMO = *MI.memoperands_begin();
222   const Value *Ptr = MMO->getValue();
223 
224   // UndefValue means this is a load of a kernel input.  These are uniform.
225   // Sometimes LDS instructions have constant pointers.
226   // If Ptr is null, then that means this mem operand contains a
227   // PseudoSourceValue like GOT.
228   if (!Ptr || isa<UndefValue>(Ptr) || isa<Argument>(Ptr) ||
229       isa<Constant>(Ptr) || isa<GlobalValue>(Ptr))
230     return true;
231 
232   if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
233     return true;
234 
235   const Instruction *I = dyn_cast<Instruction>(Ptr);
236   return I && I->getMetadata("amdgpu.uniform");
237 }
238 
239 static unsigned getSmrdOpcode(unsigned BaseOpcode, unsigned LoadSize) {
240 
241   if (LoadSize == 32)
242     return BaseOpcode;
243 
244   switch (BaseOpcode) {
245   case AMDGPU::S_LOAD_DWORD_IMM:
246     switch (LoadSize) {
247     case 64:
248       return AMDGPU::S_LOAD_DWORDX2_IMM;
249     case 128:
250       return AMDGPU::S_LOAD_DWORDX4_IMM;
251     case 256:
252       return AMDGPU::S_LOAD_DWORDX8_IMM;
253     case 512:
254       return AMDGPU::S_LOAD_DWORDX16_IMM;
255     }
256     break;
257   case AMDGPU::S_LOAD_DWORD_IMM_ci:
258     switch (LoadSize) {
259     case 64:
260       return AMDGPU::S_LOAD_DWORDX2_IMM_ci;
261     case 128:
262       return AMDGPU::S_LOAD_DWORDX4_IMM_ci;
263     case 256:
264       return AMDGPU::S_LOAD_DWORDX8_IMM_ci;
265     case 512:
266       return AMDGPU::S_LOAD_DWORDX16_IMM_ci;
267     }
268     break;
269   case AMDGPU::S_LOAD_DWORD_SGPR:
270     switch (LoadSize) {
271     case 64:
272       return AMDGPU::S_LOAD_DWORDX2_SGPR;
273     case 128:
274       return AMDGPU::S_LOAD_DWORDX4_SGPR;
275     case 256:
276       return AMDGPU::S_LOAD_DWORDX8_SGPR;
277     case 512:
278       return AMDGPU::S_LOAD_DWORDX16_SGPR;
279     }
280     break;
281   }
282   llvm_unreachable("Invalid base smrd opcode or size");
283 }
284 
285 bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
286   for (const GEPInfo &GEPInfo : AddrInfo) {
287     if (!GEPInfo.VgprParts.empty())
288       return true;
289   }
290   return false;
291 }
292 
293 bool AMDGPUInstructionSelector::selectSMRD(MachineInstr &I,
294                                            ArrayRef<GEPInfo> AddrInfo) const {
295 
296   if (!I.hasOneMemOperand())
297     return false;
298 
299   if ((*I.memoperands_begin())->getAddrSpace() != AMDGPUASI.CONSTANT_ADDRESS &&
300       (*I.memoperands_begin())->getAddrSpace() != AMDGPUASI.CONSTANT_ADDRESS_32BIT)
301     return false;
302 
303   if (!isInstrUniform(I))
304     return false;
305 
306   if (hasVgprParts(AddrInfo))
307     return false;
308 
309   MachineBasicBlock *BB = I.getParent();
310   MachineFunction *MF = BB->getParent();
311   const SISubtarget &Subtarget = MF->getSubtarget<SISubtarget>();
312   MachineRegisterInfo &MRI = MF->getRegInfo();
313   unsigned DstReg = I.getOperand(0).getReg();
314   const DebugLoc &DL = I.getDebugLoc();
315   unsigned Opcode;
316   unsigned LoadSize = RBI.getSizeInBits(DstReg, MRI, TRI);
317 
318   if (!AddrInfo.empty() && AddrInfo[0].SgprParts.size() == 1) {
319 
320     const GEPInfo &GEPInfo = AddrInfo[0];
321 
322     unsigned PtrReg = GEPInfo.SgprParts[0];
323     int64_t EncodedImm = AMDGPU::getSMRDEncodedOffset(Subtarget, GEPInfo.Imm);
324     if (AMDGPU::isLegalSMRDImmOffset(Subtarget, GEPInfo.Imm)) {
325       Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM, LoadSize);
326 
327       MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
328                                  .addReg(PtrReg)
329                                  .addImm(EncodedImm)
330                                  .addImm(0); // glc
331       return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
332     }
333 
334     if (Subtarget.getGeneration() == AMDGPUSubtarget::SEA_ISLANDS &&
335         isUInt<32>(EncodedImm)) {
336       Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM_ci, LoadSize);
337       MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
338                                    .addReg(PtrReg)
339                                    .addImm(EncodedImm)
340                                    .addImm(0); // glc
341       return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
342     }
343 
344     if (isUInt<32>(GEPInfo.Imm)) {
345       Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_SGPR, LoadSize);
346       unsigned OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
347       BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B32), OffsetReg)
348               .addImm(GEPInfo.Imm);
349 
350       MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
351                                    .addReg(PtrReg)
352                                    .addReg(OffsetReg)
353                                    .addImm(0); // glc
354       return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
355     }
356   }
357 
358   unsigned PtrReg = I.getOperand(1).getReg();
359   Opcode = getSmrdOpcode(AMDGPU::S_LOAD_DWORD_IMM, LoadSize);
360   MachineInstr *SMRD = BuildMI(*BB, &I, DL, TII.get(Opcode), DstReg)
361                                .addReg(PtrReg)
362                                .addImm(0)
363                                .addImm(0); // glc
364   return constrainSelectedInstRegOperands(*SMRD, TII, TRI, RBI);
365 }
366 
367 
368 bool AMDGPUInstructionSelector::selectG_LOAD(MachineInstr &I) const {
369   MachineBasicBlock *BB = I.getParent();
370   MachineFunction *MF = BB->getParent();
371   MachineRegisterInfo &MRI = MF->getRegInfo();
372   DebugLoc DL = I.getDebugLoc();
373   unsigned DstReg = I.getOperand(0).getReg();
374   unsigned PtrReg = I.getOperand(1).getReg();
375   unsigned LoadSize = RBI.getSizeInBits(DstReg, MRI, TRI);
376   unsigned Opcode;
377 
378   SmallVector<GEPInfo, 4> AddrInfo;
379 
380   getAddrModeInfo(I, MRI, AddrInfo);
381 
382   if (selectSMRD(I, AddrInfo)) {
383     I.eraseFromParent();
384     return true;
385   }
386 
387   switch (LoadSize) {
388   default:
389     llvm_unreachable("Load size not supported\n");
390   case 32:
391     Opcode = AMDGPU::FLAT_LOAD_DWORD;
392     break;
393   case 64:
394     Opcode = AMDGPU::FLAT_LOAD_DWORDX2;
395     break;
396   }
397 
398   MachineInstr *Flat = BuildMI(*BB, &I, DL, TII.get(Opcode))
399                                .add(I.getOperand(0))
400                                .addReg(PtrReg)
401                                .addImm(0)  // offset
402                                .addImm(0)  // glc
403                                .addImm(0); // slc
404 
405   bool Ret = constrainSelectedInstRegOperands(*Flat, TII, TRI, RBI);
406   I.eraseFromParent();
407   return Ret;
408 }
409 
410 bool AMDGPUInstructionSelector::select(MachineInstr &I,
411                                        CodeGenCoverage &CoverageInfo) const {
412 
413   if (!isPreISelGenericOpcode(I.getOpcode()))
414     return true;
415 
416   switch (I.getOpcode()) {
417   default:
418     break;
419   case TargetOpcode::G_ADD:
420     return selectG_ADD(I);
421   case TargetOpcode::G_CONSTANT:
422     return selectG_CONSTANT(I);
423   case TargetOpcode::G_GEP:
424     return selectG_GEP(I);
425   case TargetOpcode::G_LOAD:
426     return selectG_LOAD(I);
427   case TargetOpcode::G_STORE:
428     return selectG_STORE(I);
429   }
430   return false;
431 }
432