1 //===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements the lowering of LLVM calls to machine code calls for
11 /// GlobalISel.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #include "AMDGPUCallLowering.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUISelLowering.h"
18 #include "AMDGPUSubtarget.h"
19 #include "SIISelLowering.h"
20 #include "SIMachineFunctionInfo.h"
21 #include "SIRegisterInfo.h"
22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23 #include "llvm/CodeGen/Analysis.h"
24 #include "llvm/CodeGen/CallingConvLower.h"
25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/Support/LowLevelTypeImpl.h"
28 
29 using namespace llvm;
30 
31 namespace {
32 
33 struct OutgoingValueHandler : public CallLowering::ValueHandler {
34   OutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
35                        MachineInstrBuilder MIB, CCAssignFn *AssignFn)
36       : ValueHandler(B, MRI, AssignFn), MIB(MIB) {}
37 
38   MachineInstrBuilder MIB;
39 
40   Register getStackAddress(uint64_t Size, int64_t Offset,
41                            MachinePointerInfo &MPO) override {
42     llvm_unreachable("not implemented");
43   }
44 
45   void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
46                             MachinePointerInfo &MPO, CCValAssign &VA) override {
47     llvm_unreachable("not implemented");
48   }
49 
50   void assignValueToReg(Register ValVReg, Register PhysReg,
51                         CCValAssign &VA) override {
52     Register ExtReg;
53     if (VA.getLocVT().getSizeInBits() < 32) {
54       // 16-bit types are reported as legal for 32-bit registers. We need to
55       // extend and do a 32-bit copy to avoid the verifier complaining about it.
56       ExtReg = MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);
57     } else
58       ExtReg = extendRegister(ValVReg, VA);
59 
60     MIRBuilder.buildCopy(PhysReg, ExtReg);
61     MIB.addUse(PhysReg, RegState::Implicit);
62   }
63 
64   bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
65                  CCValAssign::LocInfo LocInfo,
66                  const CallLowering::ArgInfo &Info,
67                  ISD::ArgFlagsTy Flags,
68                  CCState &State) override {
69     return AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
70   }
71 };
72 
73 struct IncomingArgHandler : public CallLowering::ValueHandler {
74   uint64_t StackUsed = 0;
75 
76   IncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
77                      CCAssignFn *AssignFn)
78     : ValueHandler(B, MRI, AssignFn) {}
79 
80   Register getStackAddress(uint64_t Size, int64_t Offset,
81                            MachinePointerInfo &MPO) override {
82     auto &MFI = MIRBuilder.getMF().getFrameInfo();
83     int FI = MFI.CreateFixedObject(Size, Offset, true);
84     MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
85     Register AddrReg = MRI.createGenericVirtualRegister(
86       LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32));
87     MIRBuilder.buildFrameIndex(AddrReg, FI);
88     StackUsed = std::max(StackUsed, Size + Offset);
89     return AddrReg;
90   }
91 
92   void assignValueToReg(Register ValVReg, Register PhysReg,
93                         CCValAssign &VA) override {
94     markPhysRegUsed(PhysReg);
95 
96     if (VA.getLocVT().getSizeInBits() < 32) {
97       // 16-bit types are reported as legal for 32-bit registers. We need to do
98       // a 32-bit copy, and truncate to avoid the verifier complaining about it.
99       auto Copy = MIRBuilder.buildCopy(LLT::scalar(32), PhysReg);
100       MIRBuilder.buildTrunc(ValVReg, Copy);
101       return;
102     }
103 
104     switch (VA.getLocInfo()) {
105     case CCValAssign::LocInfo::SExt:
106     case CCValAssign::LocInfo::ZExt:
107     case CCValAssign::LocInfo::AExt: {
108       auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg);
109       MIRBuilder.buildTrunc(ValVReg, Copy);
110       break;
111     }
112     default:
113       MIRBuilder.buildCopy(ValVReg, PhysReg);
114       break;
115     }
116   }
117 
118   void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
119                             MachinePointerInfo &MPO, CCValAssign &VA) override {
120     // FIXME: Get alignment
121     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
122       MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 1);
123     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
124   }
125 
126   /// How the physical register gets marked varies between formal
127   /// parameters (it's a basic-block live-in), and a call instruction
128   /// (it's an implicit-def of the BL).
129   virtual void markPhysRegUsed(unsigned PhysReg) = 0;
130 
131   // FIXME: What is the point of this being a callback?
132   bool isIncomingArgumentHandler() const override { return true; }
133 };
134 
135 struct FormalArgHandler : public IncomingArgHandler {
136   FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,
137                    CCAssignFn *AssignFn)
138     : IncomingArgHandler(B, MRI, AssignFn) {}
139 
140   void markPhysRegUsed(unsigned PhysReg) override {
141     MIRBuilder.getMBB().addLiveIn(PhysReg);
142   }
143 };
144 
145 }
146 
147 AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
148   : CallLowering(&TLI) {
149 }
150 
151 void AMDGPUCallLowering::splitToValueTypes(
152     const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
153     const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv,
154     SplitArgTy PerformArgSplit) const {
155   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
156   LLVMContext &Ctx = OrigArg.Ty->getContext();
157 
158   if (OrigArg.Ty->isVoidTy())
159     return;
160 
161   SmallVector<EVT, 4> SplitVTs;
162   ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs);
163 
164   assert(OrigArg.Regs.size() == SplitVTs.size());
165 
166   int SplitIdx = 0;
167   for (EVT VT : SplitVTs) {
168     unsigned NumParts = TLI.getNumRegistersForCallingConv(Ctx, CallConv, VT);
169     Type *Ty = VT.getTypeForEVT(Ctx);
170 
171 
172 
173     if (NumParts == 1) {
174       // No splitting to do, but we want to replace the original type (e.g. [1 x
175       // double] -> double).
176       SplitArgs.emplace_back(OrigArg.Regs[SplitIdx], Ty,
177                              OrigArg.Flags, OrigArg.IsFixed);
178 
179       ++SplitIdx;
180       continue;
181     }
182 
183     LLT LLTy = getLLTForType(*Ty, DL);
184 
185     SmallVector<Register, 8> SplitRegs;
186 
187     EVT PartVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);
188     Type *PartTy = PartVT.getTypeForEVT(Ctx);
189     LLT PartLLT = getLLTForType(*PartTy, DL);
190 
191     // FIXME: Should we be reporting all of the part registers for a single
192     // argument, and let handleAssignments take care of the repacking?
193     for (unsigned i = 0; i < NumParts; ++i) {
194       Register PartReg = MRI.createGenericVirtualRegister(PartLLT);
195       SplitRegs.push_back(PartReg);
196       SplitArgs.emplace_back(ArrayRef<Register>(PartReg), PartTy, OrigArg.Flags);
197     }
198 
199     PerformArgSplit(SplitRegs, LLTy, PartLLT, SplitIdx);
200 
201     ++SplitIdx;
202   }
203 }
204 
205 // Get the appropriate type to make \p OrigTy \p Factor times bigger.
206 static LLT getMultipleType(LLT OrigTy, int Factor) {
207   if (OrigTy.isVector()) {
208     return LLT::vector(OrigTy.getNumElements() * Factor,
209                        OrigTy.getElementType());
210   }
211 
212   return LLT::scalar(OrigTy.getSizeInBits() * Factor);
213 }
214 
215 // TODO: Move to generic code
216 static void unpackRegsToOrigType(MachineIRBuilder &B,
217                                  ArrayRef<Register> DstRegs,
218                                  Register SrcReg,
219                                  LLT SrcTy,
220                                  LLT PartTy) {
221   assert(DstRegs.size() > 1 && "Nothing to unpack");
222 
223   MachineFunction &MF = B.getMF();
224   MachineRegisterInfo &MRI = MF.getRegInfo();
225 
226   const unsigned SrcSize = SrcTy.getSizeInBits();
227   const unsigned PartSize = PartTy.getSizeInBits();
228 
229   if (SrcTy.isVector() && !PartTy.isVector() &&
230       PartSize > SrcTy.getElementType().getSizeInBits()) {
231     // Vector was scalarized, and the elements extended.
232     auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(),
233                                                   SrcReg);
234     for (int i = 0, e = DstRegs.size(); i != e; ++i)
235       B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));
236     return;
237   }
238 
239   if (SrcSize % PartSize == 0) {
240     B.buildUnmerge(DstRegs, SrcReg);
241     return;
242   }
243 
244   const int NumRoundedParts = (SrcSize + PartSize - 1) / PartSize;
245 
246   LLT BigTy = getMultipleType(PartTy, NumRoundedParts);
247   auto ImpDef = B.buildUndef(BigTy);
248 
249   Register BigReg = MRI.createGenericVirtualRegister(BigTy);
250   B.buildInsert(BigReg, ImpDef.getReg(0), SrcReg, 0).getReg(0);
251 
252   int64_t Offset = 0;
253   for (unsigned i = 0, e = DstRegs.size(); i != e; ++i, Offset += PartSize)
254     B.buildExtract(DstRegs[i], BigReg, Offset);
255 }
256 
257 /// Lower the return value for the already existing \p Ret. This assumes that
258 /// \p B's insertion point is correct.
259 bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,
260                                         const Value *Val, ArrayRef<Register> VRegs,
261                                         MachineInstrBuilder &Ret) const {
262   if (!Val)
263     return true;
264 
265   auto &MF = B.getMF();
266   const auto &F = MF.getFunction();
267   const DataLayout &DL = MF.getDataLayout();
268 
269   CallingConv::ID CC = F.getCallingConv();
270   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
271   MachineRegisterInfo &MRI = MF.getRegInfo();
272 
273   ArgInfo OrigRetInfo(VRegs, Val->getType());
274   setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F);
275   SmallVector<ArgInfo, 4> SplitRetInfos;
276 
277   splitToValueTypes(
278     OrigRetInfo, SplitRetInfos, DL, MRI, CC,
279     [&](ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, int VTSplitIdx) {
280       unpackRegsToOrigType(B, Regs, VRegs[VTSplitIdx], LLTy, PartLLT);
281     });
282 
283   CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, F.isVarArg());
284 
285   OutgoingValueHandler RetHandler(B, MF.getRegInfo(), Ret, AssignFn);
286   return handleAssignments(B, SplitRetInfos, RetHandler);
287 }
288 
289 bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B,
290                                      const Value *Val,
291                                      ArrayRef<Register> VRegs) const {
292 
293   MachineFunction &MF = B.getMF();
294   MachineRegisterInfo &MRI = MF.getRegInfo();
295   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
296   MFI->setIfReturnsVoid(!Val);
297 
298   assert(!Val == VRegs.empty() && "Return value without a vreg");
299 
300   CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
301   const bool IsShader = AMDGPU::isShader(CC);
302   const bool IsWaveEnd = (IsShader && MFI->returnsVoid()) ||
303                          AMDGPU::isKernel(CC);
304   if (IsWaveEnd) {
305     B.buildInstr(AMDGPU::S_ENDPGM)
306       .addImm(0);
307     return true;
308   }
309 
310   auto const &ST = B.getMF().getSubtarget<GCNSubtarget>();
311 
312   unsigned ReturnOpc =
313       IsShader ? AMDGPU::SI_RETURN_TO_EPILOG : AMDGPU::S_SETPC_B64_return;
314 
315   auto Ret = B.buildInstrNoInsert(ReturnOpc);
316   Register ReturnAddrVReg;
317   if (ReturnOpc == AMDGPU::S_SETPC_B64_return) {
318     ReturnAddrVReg = MRI.createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass);
319     Ret.addUse(ReturnAddrVReg);
320   }
321 
322   if (!lowerReturnVal(B, Val, VRegs, Ret))
323     return false;
324 
325   if (ReturnOpc == AMDGPU::S_SETPC_B64_return) {
326     const SIRegisterInfo *TRI = ST.getRegisterInfo();
327     Register LiveInReturn = MF.addLiveIn(TRI->getReturnAddressReg(MF),
328                                          &AMDGPU::SGPR_64RegClass);
329     B.buildCopy(ReturnAddrVReg, LiveInReturn);
330   }
331 
332   // TODO: Handle CalleeSavedRegsViaCopy.
333 
334   B.insertInstr(Ret);
335   return true;
336 }
337 
338 Register AMDGPUCallLowering::lowerParameterPtr(MachineIRBuilder &B,
339                                                Type *ParamTy,
340                                                uint64_t Offset) const {
341 
342   MachineFunction &MF = B.getMF();
343   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
344   MachineRegisterInfo &MRI = MF.getRegInfo();
345   const Function &F = MF.getFunction();
346   const DataLayout &DL = F.getParent()->getDataLayout();
347   PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS);
348   LLT PtrType = getLLTForType(*PtrTy, DL);
349   Register DstReg = MRI.createGenericVirtualRegister(PtrType);
350   Register KernArgSegmentPtr =
351     MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
352   Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);
353 
354   Register OffsetReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
355   B.buildConstant(OffsetReg, Offset);
356 
357   B.buildGEP(DstReg, KernArgSegmentVReg, OffsetReg);
358 
359   return DstReg;
360 }
361 
362 void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B,
363                                         Type *ParamTy, uint64_t Offset,
364                                         unsigned Align,
365                                         Register DstReg) const {
366   MachineFunction &MF = B.getMF();
367   const Function &F = MF.getFunction();
368   const DataLayout &DL = F.getParent()->getDataLayout();
369   PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS);
370   MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
371   unsigned TypeSize = DL.getTypeStoreSize(ParamTy);
372   Register PtrReg = lowerParameterPtr(B, ParamTy, Offset);
373 
374   MachineMemOperand *MMO =
375       MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad |
376                                        MachineMemOperand::MODereferenceable |
377                                        MachineMemOperand::MOInvariant,
378                                        TypeSize, Align);
379 
380   B.buildLoad(DstReg, PtrReg, *MMO);
381 }
382 
383 // Allocate special inputs passed in user SGPRs.
384 static void allocateHSAUserSGPRs(CCState &CCInfo,
385                                  MachineIRBuilder &B,
386                                  MachineFunction &MF,
387                                  const SIRegisterInfo &TRI,
388                                  SIMachineFunctionInfo &Info) {
389   // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
390   if (Info.hasPrivateSegmentBuffer()) {
391     unsigned PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
392     MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
393     CCInfo.AllocateReg(PrivateSegmentBufferReg);
394   }
395 
396   if (Info.hasDispatchPtr()) {
397     unsigned DispatchPtrReg = Info.addDispatchPtr(TRI);
398     MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
399     CCInfo.AllocateReg(DispatchPtrReg);
400   }
401 
402   if (Info.hasQueuePtr()) {
403     unsigned QueuePtrReg = Info.addQueuePtr(TRI);
404     MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
405     CCInfo.AllocateReg(QueuePtrReg);
406   }
407 
408   if (Info.hasKernargSegmentPtr()) {
409     MachineRegisterInfo &MRI = MF.getRegInfo();
410     Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
411     const LLT P4 = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
412     Register VReg = MRI.createGenericVirtualRegister(P4);
413     MRI.addLiveIn(InputPtrReg, VReg);
414     B.getMBB().addLiveIn(InputPtrReg);
415     B.buildCopy(VReg, InputPtrReg);
416     CCInfo.AllocateReg(InputPtrReg);
417   }
418 
419   if (Info.hasDispatchID()) {
420     unsigned DispatchIDReg = Info.addDispatchID(TRI);
421     MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
422     CCInfo.AllocateReg(DispatchIDReg);
423   }
424 
425   if (Info.hasFlatScratchInit()) {
426     unsigned FlatScratchInitReg = Info.addFlatScratchInit(TRI);
427     MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
428     CCInfo.AllocateReg(FlatScratchInitReg);
429   }
430 
431   // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
432   // these from the dispatch pointer.
433 }
434 
435 bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
436     MachineIRBuilder &B, const Function &F,
437     ArrayRef<ArrayRef<Register>> VRegs) const {
438   MachineFunction &MF = B.getMF();
439   const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();
440   MachineRegisterInfo &MRI = MF.getRegInfo();
441   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
442   const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
443   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
444 
445   const DataLayout &DL = F.getParent()->getDataLayout();
446 
447   SmallVector<CCValAssign, 16> ArgLocs;
448   CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
449 
450   allocateHSAUserSGPRs(CCInfo, B, MF, *TRI, *Info);
451 
452   unsigned i = 0;
453   const unsigned KernArgBaseAlign = 16;
454   const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset(F);
455   uint64_t ExplicitArgOffset = 0;
456 
457   // TODO: Align down to dword alignment and extract bits for extending loads.
458   for (auto &Arg : F.args()) {
459     Type *ArgTy = Arg.getType();
460     unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
461     if (AllocSize == 0)
462       continue;
463 
464     unsigned ABIAlign = DL.getABITypeAlignment(ArgTy);
465 
466     uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
467     ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
468 
469     ArrayRef<Register> OrigArgRegs = VRegs[i];
470     Register ArgReg =
471       OrigArgRegs.size() == 1
472       ? OrigArgRegs[0]
473       : MRI.createGenericVirtualRegister(getLLTForType(*ArgTy, DL));
474     unsigned Align = MinAlign(KernArgBaseAlign, ArgOffset);
475     ArgOffset = alignTo(ArgOffset, DL.getABITypeAlignment(ArgTy));
476     lowerParameter(B, ArgTy, ArgOffset, Align, ArgReg);
477     if (OrigArgRegs.size() > 1)
478       unpackRegs(OrigArgRegs, ArgReg, ArgTy, B);
479     ++i;
480   }
481 
482   TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
483   TLI.allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), false);
484   return true;
485 }
486 
487 // TODO: Move this to generic code
488 static void packSplitRegsToOrigType(MachineIRBuilder &B,
489                                     ArrayRef<Register> OrigRegs,
490                                     ArrayRef<Register> Regs,
491                                     LLT LLTy,
492                                     LLT PartLLT) {
493   if (!LLTy.isVector() && !PartLLT.isVector()) {
494     B.buildMerge(OrigRegs[0], Regs);
495     return;
496   }
497 
498   if (LLTy.isVector() && PartLLT.isVector()) {
499     assert(LLTy.getElementType() == PartLLT.getElementType());
500 
501     int DstElts = LLTy.getNumElements();
502     int PartElts = PartLLT.getNumElements();
503     if (DstElts % PartElts == 0)
504       B.buildConcatVectors(OrigRegs[0], Regs);
505     else {
506       // Deal with v3s16 split into v2s16
507       assert(PartElts == 2 && DstElts % 2 != 0);
508       int RoundedElts = PartElts * ((DstElts + PartElts - 1) / PartElts);
509 
510       LLT RoundedDestTy = LLT::vector(RoundedElts, PartLLT.getElementType());
511       auto RoundedConcat = B.buildConcatVectors(RoundedDestTy, Regs);
512       B.buildExtract(OrigRegs[0], RoundedConcat, 0);
513     }
514 
515     return;
516   }
517 
518   assert(LLTy.isVector() && !PartLLT.isVector());
519 
520   LLT DstEltTy = LLTy.getElementType();
521   if (DstEltTy == PartLLT) {
522     // Vector was trivially scalarized.
523     B.buildBuildVector(OrigRegs[0], Regs);
524   } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {
525     // Deal with vector with 64-bit elements decomposed to 32-bit
526     // registers. Need to create intermediate 64-bit elements.
527     SmallVector<Register, 8> EltMerges;
528     int PartsPerElt = DstEltTy.getSizeInBits() / PartLLT.getSizeInBits();
529 
530     assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0);
531 
532     for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I)  {
533       auto Merge = B.buildMerge(DstEltTy,
534                                          Regs.take_front(PartsPerElt));
535       EltMerges.push_back(Merge.getReg(0));
536       Regs = Regs.drop_front(PartsPerElt);
537     }
538 
539     B.buildBuildVector(OrigRegs[0], EltMerges);
540   } else {
541     // Vector was split, and elements promoted to a wider type.
542     LLT BVType = LLT::vector(LLTy.getNumElements(), PartLLT);
543     auto BV = B.buildBuildVector(BVType, Regs);
544     B.buildTrunc(OrigRegs[0], BV);
545   }
546 }
547 
548 bool AMDGPUCallLowering::lowerFormalArguments(
549     MachineIRBuilder &B, const Function &F,
550     ArrayRef<ArrayRef<Register>> VRegs) const {
551   CallingConv::ID CC = F.getCallingConv();
552 
553   // The infrastructure for normal calling convention lowering is essentially
554   // useless for kernels. We want to avoid any kind of legalization or argument
555   // splitting.
556   if (CC == CallingConv::AMDGPU_KERNEL)
557     return lowerFormalArgumentsKernel(B, F, VRegs);
558 
559   const bool IsShader = AMDGPU::isShader(CC);
560   const bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC);
561 
562   MachineFunction &MF = B.getMF();
563   MachineBasicBlock &MBB = B.getMBB();
564   MachineRegisterInfo &MRI = MF.getRegInfo();
565   SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
566   const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
567   const SIRegisterInfo *TRI = Subtarget.getRegisterInfo();
568   const DataLayout &DL = F.getParent()->getDataLayout();
569 
570 
571   SmallVector<CCValAssign, 16> ArgLocs;
572   CCState CCInfo(CC, F.isVarArg(), MF, ArgLocs, F.getContext());
573 
574   if (!IsEntryFunc) {
575     Register ReturnAddrReg = TRI->getReturnAddressReg(MF);
576     Register LiveInReturn = MF.addLiveIn(ReturnAddrReg,
577                                          &AMDGPU::SGPR_64RegClass);
578     MBB.addLiveIn(ReturnAddrReg);
579     B.buildCopy(LiveInReturn, ReturnAddrReg);
580   }
581 
582   if (Info->hasImplicitBufferPtr()) {
583     Register ImplicitBufferPtrReg = Info->addImplicitBufferPtr(*TRI);
584     MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
585     CCInfo.AllocateReg(ImplicitBufferPtrReg);
586   }
587 
588 
589   SmallVector<ArgInfo, 32> SplitArgs;
590   unsigned Idx = 0;
591   unsigned PSInputNum = 0;
592 
593   for (auto &Arg : F.args()) {
594     if (DL.getTypeStoreSize(Arg.getType()) == 0)
595       continue;
596 
597     const bool InReg = Arg.hasAttribute(Attribute::InReg);
598 
599     // SGPR arguments to functions not implemented.
600     if (!IsShader && InReg)
601       return false;
602 
603     if (Arg.hasAttribute(Attribute::SwiftSelf) ||
604         Arg.hasAttribute(Attribute::SwiftError) ||
605         Arg.hasAttribute(Attribute::Nest))
606       return false;
607 
608     if (CC == CallingConv::AMDGPU_PS && !InReg && PSInputNum <= 15) {
609       const bool ArgUsed = !Arg.use_empty();
610       bool SkipArg = !ArgUsed && !Info->isPSInputAllocated(PSInputNum);
611 
612       if (!SkipArg) {
613         Info->markPSInputAllocated(PSInputNum);
614         if (ArgUsed)
615           Info->markPSInputEnabled(PSInputNum);
616       }
617 
618       ++PSInputNum;
619 
620       if (SkipArg) {
621         for (int I = 0, E = VRegs[Idx].size(); I != E; ++I)
622           B.buildUndef(VRegs[Idx][I]);
623 
624         ++Idx;
625         continue;
626       }
627     }
628 
629     ArgInfo OrigArg(VRegs[Idx], Arg.getType());
630     setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F);
631 
632     splitToValueTypes(
633       OrigArg, SplitArgs, DL, MRI, CC,
634       // FIXME: We should probably be passing multiple registers to
635       // handleAssignments to do this
636       [&](ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT, int VTSplitIdx) {
637         packSplitRegsToOrigType(B, VRegs[Idx][VTSplitIdx], Regs,
638                                 LLTy, PartLLT);
639       });
640 
641     ++Idx;
642   }
643 
644   // At least one interpolation mode must be enabled or else the GPU will
645   // hang.
646   //
647   // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
648   // set PSInputAddr, the user wants to enable some bits after the compilation
649   // based on run-time states. Since we can't know what the final PSInputEna
650   // will look like, so we shouldn't do anything here and the user should take
651   // responsibility for the correct programming.
652   //
653   // Otherwise, the following restrictions apply:
654   // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
655   // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
656   //   enabled too.
657   if (CC == CallingConv::AMDGPU_PS) {
658     if ((Info->getPSInputAddr() & 0x7F) == 0 ||
659         ((Info->getPSInputAddr() & 0xF) == 0 &&
660          Info->isPSInputAllocated(11))) {
661       CCInfo.AllocateReg(AMDGPU::VGPR0);
662       CCInfo.AllocateReg(AMDGPU::VGPR1);
663       Info->markPSInputAllocated(0);
664       Info->markPSInputEnabled(0);
665     }
666 
667     if (Subtarget.isAmdPalOS()) {
668       // For isAmdPalOS, the user does not enable some bits after compilation
669       // based on run-time states; the register values being generated here are
670       // the final ones set in hardware. Therefore we need to apply the
671       // workaround to PSInputAddr and PSInputEnable together.  (The case where
672       // a bit is set in PSInputAddr but not PSInputEnable is where the frontend
673       // set up an input arg for a particular interpolation mode, but nothing
674       // uses that input arg. Really we should have an earlier pass that removes
675       // such an arg.)
676       unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
677       if ((PsInputBits & 0x7F) == 0 ||
678           ((PsInputBits & 0xF) == 0 &&
679            (PsInputBits >> 11 & 1)))
680         Info->markPSInputEnabled(
681           countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined));
682     }
683   }
684 
685   const SITargetLowering &TLI = *getTLI<SITargetLowering>();
686   CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC, F.isVarArg());
687 
688   if (!MBB.empty())
689     B.setInstr(*MBB.begin());
690 
691   FormalArgHandler Handler(B, MRI, AssignFn);
692   if (!handleAssignments(CCInfo, ArgLocs, B, SplitArgs, Handler))
693     return false;
694 
695   if (!IsEntryFunc) {
696     // Special inputs come after user arguments.
697     TLI.allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
698   }
699 
700   // Start adding system SGPRs.
701   if (IsEntryFunc) {
702     TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsShader);
703   } else {
704     CCInfo.AllocateReg(Info->getScratchRSrcReg());
705     CCInfo.AllocateReg(Info->getScratchWaveOffsetReg());
706     CCInfo.AllocateReg(Info->getFrameOffsetReg());
707     TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
708   }
709 
710   // Move back to the end of the basic block.
711   B.setMBB(MBB);
712 
713   return true;
714 }
715