1 //===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// This file implements the lowering of LLVM calls to machine code calls for
12 /// GlobalISel.
13 ///
14 //===----------------------------------------------------------------------===//
15 
16 #include "AArch64CallLowering.h"
17 #include "AArch64ISelLowering.h"
18 #include "AArch64MachineFunctionInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/CodeGen/Analysis.h"
23 #include "llvm/CodeGen/CallingConvLower.h"
24 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25 #include "llvm/CodeGen/GlobalISel/Utils.h"
26 #include "llvm/CodeGen/LowLevelType.h"
27 #include "llvm/CodeGen/MachineBasicBlock.h"
28 #include "llvm/CodeGen/MachineFrameInfo.h"
29 #include "llvm/CodeGen/MachineFunction.h"
30 #include "llvm/CodeGen/MachineInstrBuilder.h"
31 #include "llvm/CodeGen/MachineMemOperand.h"
32 #include "llvm/CodeGen/MachineOperand.h"
33 #include "llvm/CodeGen/MachineRegisterInfo.h"
34 #include "llvm/CodeGen/MachineValueType.h"
35 #include "llvm/CodeGen/ValueTypes.h"
36 #include "llvm/IR/Argument.h"
37 #include "llvm/IR/Attributes.h"
38 #include "llvm/IR/Function.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/Target/TargetRegisterInfo.h"
42 #include "llvm/Target/TargetSubtargetInfo.h"
43 #include <algorithm>
44 #include <cassert>
45 #include <cstdint>
46 #include <iterator>
47 
48 using namespace llvm;
49 
50 #ifndef LLVM_BUILD_GLOBAL_ISEL
51 #error "This shouldn't be built without GISel"
52 #endif
53 
54 AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI)
55   : CallLowering(&TLI) {}
56 
57 struct IncomingArgHandler : public CallLowering::ValueHandler {
58   IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
59                      CCAssignFn *AssignFn)
60       : ValueHandler(MIRBuilder, MRI, AssignFn), StackUsed(0) {}
61 
62   unsigned getStackAddress(uint64_t Size, int64_t Offset,
63                            MachinePointerInfo &MPO) override {
64     auto &MFI = MIRBuilder.getMF().getFrameInfo();
65     int FI = MFI.CreateFixedObject(Size, Offset, true);
66     MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
67     unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64));
68     MIRBuilder.buildFrameIndex(AddrReg, FI);
69     StackUsed = std::max(StackUsed, Size + Offset);
70     return AddrReg;
71   }
72 
73   void assignValueToReg(unsigned ValVReg, unsigned PhysReg,
74                         CCValAssign &VA) override {
75     markPhysRegUsed(PhysReg);
76     MIRBuilder.buildCopy(ValVReg, PhysReg);
77     // FIXME: assert extension
78   }
79 
80   void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size,
81                             MachinePointerInfo &MPO, CCValAssign &VA) override {
82     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
83         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
84         0);
85     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
86   }
87 
88   /// How the physical register gets marked varies between formal
89   /// parameters (it's a basic-block live-in), and a call instruction
90   /// (it's an implicit-def of the BL).
91   virtual void markPhysRegUsed(unsigned PhysReg) = 0;
92 
93   uint64_t StackUsed;
94 };
95 
96 struct FormalArgHandler : public IncomingArgHandler {
97   FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
98                    CCAssignFn *AssignFn)
99     : IncomingArgHandler(MIRBuilder, MRI, AssignFn) {}
100 
101   void markPhysRegUsed(unsigned PhysReg) override {
102     MIRBuilder.getMBB().addLiveIn(PhysReg);
103   }
104 };
105 
106 struct CallReturnHandler : public IncomingArgHandler {
107   CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
108                     MachineInstrBuilder MIB, CCAssignFn *AssignFn)
109     : IncomingArgHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
110 
111   void markPhysRegUsed(unsigned PhysReg) override {
112     MIB.addDef(PhysReg, RegState::Implicit);
113   }
114 
115   MachineInstrBuilder MIB;
116 };
117 
118 struct OutgoingArgHandler : public CallLowering::ValueHandler {
119   OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
120                      MachineInstrBuilder MIB, CCAssignFn *AssignFn,
121                      CCAssignFn *AssignFnVarArg)
122       : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
123         AssignFnVarArg(AssignFnVarArg), StackSize(0) {}
124 
125   unsigned getStackAddress(uint64_t Size, int64_t Offset,
126                            MachinePointerInfo &MPO) override {
127     LLT p0 = LLT::pointer(0, 64);
128     LLT s64 = LLT::scalar(64);
129     unsigned SPReg = MRI.createGenericVirtualRegister(p0);
130     MIRBuilder.buildCopy(SPReg, AArch64::SP);
131 
132     unsigned OffsetReg = MRI.createGenericVirtualRegister(s64);
133     MIRBuilder.buildConstant(OffsetReg, Offset);
134 
135     unsigned AddrReg = MRI.createGenericVirtualRegister(p0);
136     MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg);
137 
138     MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset);
139     return AddrReg;
140   }
141 
142   void assignValueToReg(unsigned ValVReg, unsigned PhysReg,
143                         CCValAssign &VA) override {
144     MIB.addUse(PhysReg, RegState::Implicit);
145     unsigned ExtReg = extendRegister(ValVReg, VA);
146     MIRBuilder.buildCopy(PhysReg, ExtReg);
147   }
148 
149   void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size,
150                             MachinePointerInfo &MPO, CCValAssign &VA) override {
151     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
152         MPO, MachineMemOperand::MOStore, Size, 0);
153     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
154   }
155 
156   bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
157                  CCValAssign::LocInfo LocInfo,
158                  const CallLowering::ArgInfo &Info,
159                  CCState &State) override {
160     bool Res;
161     if (Info.IsFixed)
162       Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
163     else
164       Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
165 
166     StackSize = State.getNextStackOffset();
167     return Res;
168   }
169 
170   MachineInstrBuilder MIB;
171   CCAssignFn *AssignFnVarArg;
172   uint64_t StackSize;
173 };
174 
175 void AArch64CallLowering::splitToValueTypes(
176     const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
177     const DataLayout &DL, MachineRegisterInfo &MRI,
178     const SplitArgTy &PerformArgSplit) const {
179   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
180   LLVMContext &Ctx = OrigArg.Ty->getContext();
181 
182   SmallVector<EVT, 4> SplitVTs;
183   SmallVector<uint64_t, 4> Offsets;
184   ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
185 
186   if (SplitVTs.size() == 1) {
187     // No splitting to do, but we want to replace the original type (e.g. [1 x
188     // double] -> double).
189     SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx),
190                            OrigArg.Flags, OrigArg.IsFixed);
191     return;
192   }
193 
194   unsigned FirstRegIdx = SplitArgs.size();
195   for (auto SplitVT : SplitVTs) {
196     // FIXME: set split flags if they're actually used (e.g. i128 on AAPCS).
197     Type *SplitTy = SplitVT.getTypeForEVT(Ctx);
198     SplitArgs.push_back(
199         ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*SplitTy, DL)),
200                 SplitTy, OrigArg.Flags, OrigArg.IsFixed});
201   }
202 
203   for (unsigned i = 0; i < Offsets.size(); ++i)
204     PerformArgSplit(SplitArgs[FirstRegIdx + i].Reg, Offsets[i] * 8);
205 }
206 
207 bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
208                                       const Value *Val, unsigned VReg) const {
209   MachineFunction &MF = MIRBuilder.getMF();
210   const Function &F = *MF.getFunction();
211 
212   auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR);
213   assert(((Val && VReg) || (!Val && !VReg)) && "Return value without a vreg");
214   bool Success = true;
215   if (VReg) {
216     const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
217     CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
218     MachineRegisterInfo &MRI = MF.getRegInfo();
219     auto &DL = F.getParent()->getDataLayout();
220 
221     ArgInfo OrigArg{VReg, Val->getType()};
222     setArgFlags(OrigArg, AttributeList::ReturnIndex, DL, F);
223 
224     SmallVector<ArgInfo, 8> SplitArgs;
225     splitToValueTypes(OrigArg, SplitArgs, DL, MRI,
226                       [&](unsigned Reg, uint64_t Offset) {
227                         MIRBuilder.buildExtract(Reg, VReg, Offset);
228                       });
229 
230     OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFn, AssignFn);
231     Success = handleAssignments(MIRBuilder, SplitArgs, Handler);
232   }
233 
234   MIRBuilder.insertInstr(MIB);
235   return Success;
236 }
237 
238 bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
239                                                const Function &F,
240                                                ArrayRef<unsigned> VRegs) const {
241   MachineFunction &MF = MIRBuilder.getMF();
242   MachineBasicBlock &MBB = MIRBuilder.getMBB();
243   MachineRegisterInfo &MRI = MF.getRegInfo();
244   auto &DL = F.getParent()->getDataLayout();
245 
246   SmallVector<ArgInfo, 8> SplitArgs;
247   unsigned i = 0;
248   for (auto &Arg : F.args()) {
249     ArgInfo OrigArg{VRegs[i], Arg.getType()};
250     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, F);
251     bool Split = false;
252     LLT Ty = MRI.getType(VRegs[i]);
253     unsigned Dst = VRegs[i];
254 
255     splitToValueTypes(OrigArg, SplitArgs, DL, MRI,
256                       [&](unsigned Reg, uint64_t Offset) {
257                         if (!Split) {
258                           Split = true;
259                           Dst = MRI.createGenericVirtualRegister(Ty);
260                           MIRBuilder.buildUndef(Dst);
261                         }
262                         unsigned Tmp = MRI.createGenericVirtualRegister(Ty);
263                         MIRBuilder.buildInsert(Tmp, Dst, Reg, Offset);
264                         Dst = Tmp;
265                       });
266 
267     if (Dst != VRegs[i])
268       MIRBuilder.buildCopy(VRegs[i], Dst);
269     ++i;
270   }
271 
272   if (!MBB.empty())
273     MIRBuilder.setInstr(*MBB.begin());
274 
275   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
276   CCAssignFn *AssignFn =
277       TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false);
278 
279   FormalArgHandler Handler(MIRBuilder, MRI, AssignFn);
280   if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
281     return false;
282 
283   if (F.isVarArg()) {
284     if (!MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {
285       // FIXME: we need to reimplement saveVarArgsRegisters from
286       // AArch64ISelLowering.
287       return false;
288     }
289 
290     // We currently pass all varargs at 8-byte alignment.
291     uint64_t StackOffset = alignTo(Handler.StackUsed, 8);
292 
293     auto &MFI = MIRBuilder.getMF().getFrameInfo();
294     AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
295     FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
296   }
297 
298   // Move back to the end of the basic block.
299   MIRBuilder.setMBB(MBB);
300 
301   return true;
302 }
303 
304 bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
305                                     CallingConv::ID CallConv,
306                                     const MachineOperand &Callee,
307                                     const ArgInfo &OrigRet,
308                                     ArrayRef<ArgInfo> OrigArgs) const {
309   MachineFunction &MF = MIRBuilder.getMF();
310   const Function &F = *MF.getFunction();
311   MachineRegisterInfo &MRI = MF.getRegInfo();
312   auto &DL = F.getParent()->getDataLayout();
313 
314   SmallVector<ArgInfo, 8> SplitArgs;
315   for (auto &OrigArg : OrigArgs) {
316     splitToValueTypes(OrigArg, SplitArgs, DL, MRI,
317                       [&](unsigned Reg, uint64_t Offset) {
318                         MIRBuilder.buildExtract(Reg, OrigArg.Reg, Offset);
319                       });
320   }
321 
322   // Find out which ABI gets to decide where things go.
323   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
324   CCAssignFn *AssignFnFixed =
325       TLI.CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
326   CCAssignFn *AssignFnVarArg =
327       TLI.CCAssignFnForCall(CallConv, /*IsVarArg=*/true);
328 
329   auto CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
330 
331   // Create a temporarily-floating call instruction so we can add the implicit
332   // uses of arg registers.
333   auto MIB = MIRBuilder.buildInstrNoInsert(Callee.isReg() ? AArch64::BLR
334                                                           : AArch64::BL);
335   MIB.add(Callee);
336 
337   // Tell the call which registers are clobbered.
338   auto TRI = MF.getSubtarget().getRegisterInfo();
339   MIB.addRegMask(TRI->getCallPreservedMask(MF, F.getCallingConv()));
340 
341   // Do the actual argument marshalling.
342   SmallVector<unsigned, 8> PhysRegs;
343   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
344                              AssignFnVarArg);
345   if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
346     return false;
347 
348   // Now we can add the actual call instruction to the correct basic block.
349   MIRBuilder.insertInstr(MIB);
350 
351   // If Callee is a reg, since it is used by a target specific
352   // instruction, it must have a register class matching the
353   // constraint of that instruction.
354   if (Callee.isReg())
355     MIB->getOperand(0).setReg(constrainOperandRegClass(
356         MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
357         *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(),
358         Callee.getReg(), 0));
359 
360   // Finally we can copy the returned value back into its virtual-register. In
361   // symmetry with the arugments, the physical register must be an
362   // implicit-define of the call instruction.
363   CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
364   if (OrigRet.Reg) {
365     SplitArgs.clear();
366 
367     SmallVector<uint64_t, 8> RegOffsets;
368     SmallVector<unsigned, 8> SplitRegs;
369     splitToValueTypes(OrigRet, SplitArgs, DL, MRI,
370                       [&](unsigned Reg, uint64_t Offset) {
371                         RegOffsets.push_back(Offset);
372                         SplitRegs.push_back(Reg);
373                       });
374 
375     CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn);
376     if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
377       return false;
378 
379     if (!RegOffsets.empty())
380       MIRBuilder.buildSequence(OrigRet.Reg, SplitRegs, RegOffsets);
381   }
382 
383   CallSeqStart.addImm(Handler.StackSize).addImm(0);
384   MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
385       .addImm(Handler.StackSize)
386       .addImm(0);
387 
388   return true;
389 }
390