1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/GlobalISel/Utils.h" 27 #include "llvm/CodeGen/LowLevelType.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFrameInfo.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineMemOperand.h" 33 #include "llvm/CodeGen/MachineOperand.h" 34 #include "llvm/CodeGen/MachineRegisterInfo.h" 35 #include "llvm/CodeGen/TargetInstrInfo.h" 36 #include "llvm/CodeGen/TargetSubtargetInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/IR/Value.h" 42 #include "llvm/MC/MCRegisterInfo.h" 43 #include "llvm/Support/LowLevelTypeImpl.h" 44 #include "llvm/Support/MachineValueType.h" 45 #include <cassert> 46 #include <cstdint> 47 48 using namespace llvm; 49 50 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 51 : CallLowering(&TLI) {} 52 53 bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 54 SmallVectorImpl<ArgInfo> &SplitArgs, 55 const DataLayout &DL, 56 MachineRegisterInfo &MRI, 57 SplitArgTy PerformArgSplit) const { 58 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 59 LLVMContext &Context = OrigArg.Ty->getContext(); 60 61 SmallVector<EVT, 4> SplitVTs; 62 SmallVector<uint64_t, 4> Offsets; 63 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 64 assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); 65 66 if (OrigArg.Ty->isVoidTy()) 67 return true; 68 69 EVT VT = SplitVTs[0]; 70 unsigned NumParts = TLI.getNumRegisters(Context, VT); 71 72 if (NumParts == 1) { 73 // replace the original type ( pointer -> GPR ). 74 SplitArgs.emplace_back(OrigArg.Regs[0], VT.getTypeForEVT(Context), 75 OrigArg.Flags, OrigArg.IsFixed); 76 return true; 77 } 78 79 SmallVector<Register, 8> SplitRegs; 80 81 EVT PartVT = TLI.getRegisterType(Context, VT); 82 Type *PartTy = PartVT.getTypeForEVT(Context); 83 84 for (unsigned i = 0; i < NumParts; ++i) { 85 ArgInfo Info = 86 ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), 87 PartTy, OrigArg.Flags}; 88 SplitArgs.push_back(Info); 89 SplitRegs.push_back(Info.Regs[0]); 90 } 91 92 PerformArgSplit(SplitRegs); 93 return true; 94 } 95 96 namespace { 97 98 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 99 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 100 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB, 101 CCAssignFn *AssignFn) 102 : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), 103 DL(MIRBuilder.getMF().getDataLayout()), 104 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 105 106 Register getStackAddress(uint64_t Size, int64_t Offset, 107 MachinePointerInfo &MPO) override { 108 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 109 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 110 auto SPReg = 111 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 112 113 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 114 115 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 116 117 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 118 return AddrReg.getReg(0); 119 } 120 121 void assignValueToReg(Register ValVReg, Register PhysReg, 122 CCValAssign &VA) override { 123 MIB.addUse(PhysReg, RegState::Implicit); 124 125 Register ExtReg; 126 // If we are copying the value to a physical register with the 127 // size larger than the size of the value itself - build AnyExt 128 // to the size of the register first and only then do the copy. 129 // The example of that would be copying from s32 to xmm0, for which 130 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 131 // we expect normal extendRegister mechanism to work. 132 unsigned PhysRegSize = 133 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 134 unsigned ValSize = VA.getValVT().getSizeInBits(); 135 unsigned LocSize = VA.getLocVT().getSizeInBits(); 136 if (PhysRegSize > ValSize && LocSize == ValSize) { 137 assert((PhysRegSize == 128 || PhysRegSize == 80) && 138 "We expect that to be 128 bit"); 139 ExtReg = 140 MIRBuilder.buildAnyExt(LLT::scalar(PhysRegSize), ValVReg).getReg(0); 141 } else 142 ExtReg = extendRegister(ValVReg, VA); 143 144 MIRBuilder.buildCopy(PhysReg, ExtReg); 145 } 146 147 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 148 MachinePointerInfo &MPO, CCValAssign &VA) override { 149 MachineFunction &MF = MIRBuilder.getMF(); 150 Register ExtReg = extendRegister(ValVReg, VA); 151 152 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, 153 VA.getLocVT().getStoreSize(), 154 inferAlignFromPtrInfo(MF, MPO)); 155 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 156 } 157 158 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, 159 CCValAssign::LocInfo LocInfo, 160 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 161 CCState &State) override { 162 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 163 StackSize = State.getNextStackOffset(); 164 165 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 166 X86::XMM3, X86::XMM4, X86::XMM5, 167 X86::XMM6, X86::XMM7}; 168 if (!Info.IsFixed) 169 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 170 171 return Res; 172 } 173 174 uint64_t getStackSize() { return StackSize; } 175 uint64_t getNumXmmRegs() { return NumXMMRegs; } 176 177 protected: 178 MachineInstrBuilder &MIB; 179 uint64_t StackSize = 0; 180 const DataLayout &DL; 181 const X86Subtarget &STI; 182 unsigned NumXMMRegs = 0; 183 }; 184 185 } // end anonymous namespace 186 187 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 188 const Value *Val, ArrayRef<Register> VRegs, 189 FunctionLoweringInfo &FLI) const { 190 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 191 "Return value without a vreg"); 192 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 193 194 if (!VRegs.empty()) { 195 MachineFunction &MF = MIRBuilder.getMF(); 196 const Function &F = MF.getFunction(); 197 MachineRegisterInfo &MRI = MF.getRegInfo(); 198 const DataLayout &DL = MF.getDataLayout(); 199 LLVMContext &Ctx = Val->getType()->getContext(); 200 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 201 202 SmallVector<EVT, 4> SplitEVTs; 203 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs); 204 assert(VRegs.size() == SplitEVTs.size() && 205 "For each split Type there should be exactly one VReg."); 206 207 SmallVector<ArgInfo, 8> SplitArgs; 208 for (unsigned i = 0; i < SplitEVTs.size(); ++i) { 209 ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)}; 210 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); 211 if (!splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, 212 [&](ArrayRef<Register> Regs) { 213 MIRBuilder.buildUnmerge(Regs, VRegs[i]); 214 })) 215 return false; 216 } 217 218 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); 219 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 220 F.isVarArg())) 221 return false; 222 } 223 224 MIRBuilder.insertInstr(MIB); 225 return true; 226 } 227 228 namespace { 229 230 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 231 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 232 MachineRegisterInfo &MRI, CCAssignFn *AssignFn) 233 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), 234 DL(MIRBuilder.getMF().getDataLayout()) {} 235 236 Register getStackAddress(uint64_t Size, int64_t Offset, 237 MachinePointerInfo &MPO) override { 238 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 239 int FI = MFI.CreateFixedObject(Size, Offset, true); 240 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 241 242 return MIRBuilder 243 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 244 .getReg(0); 245 } 246 247 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 248 MachinePointerInfo &MPO, CCValAssign &VA) override { 249 MachineFunction &MF = MIRBuilder.getMF(); 250 auto *MMO = MF.getMachineMemOperand( 251 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 252 inferAlignFromPtrInfo(MF, MPO)); 253 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 254 } 255 256 void assignValueToReg(Register ValVReg, Register PhysReg, 257 CCValAssign &VA) override { 258 markPhysRegUsed(PhysReg); 259 260 switch (VA.getLocInfo()) { 261 default: { 262 // If we are copying the value from a physical register with the 263 // size larger than the size of the value itself - build the copy 264 // of the phys reg first and then build the truncation of that copy. 265 // The example of that would be copying from xmm0 to s32, for which 266 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 267 // we expect this to be handled in SExt/ZExt/AExt case. 268 unsigned PhysRegSize = 269 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 270 unsigned ValSize = VA.getValVT().getSizeInBits(); 271 unsigned LocSize = VA.getLocVT().getSizeInBits(); 272 if (PhysRegSize > ValSize && LocSize == ValSize) { 273 auto Copy = MIRBuilder.buildCopy(LLT::scalar(PhysRegSize), PhysReg); 274 MIRBuilder.buildTrunc(ValVReg, Copy); 275 return; 276 } 277 278 MIRBuilder.buildCopy(ValVReg, PhysReg); 279 break; 280 } 281 case CCValAssign::LocInfo::SExt: 282 case CCValAssign::LocInfo::ZExt: 283 case CCValAssign::LocInfo::AExt: { 284 auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg); 285 MIRBuilder.buildTrunc(ValVReg, Copy); 286 break; 287 } 288 } 289 } 290 291 /// How the physical register gets marked varies between formal 292 /// parameters (it's a basic-block live-in), and a call instruction 293 /// (it's an implicit-def of the BL). 294 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 295 296 protected: 297 const DataLayout &DL; 298 }; 299 300 struct FormalArgHandler : public X86IncomingValueHandler { 301 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 302 CCAssignFn *AssignFn) 303 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn) {} 304 305 void markPhysRegUsed(unsigned PhysReg) override { 306 MIRBuilder.getMRI()->addLiveIn(PhysReg); 307 MIRBuilder.getMBB().addLiveIn(PhysReg); 308 } 309 }; 310 311 struct CallReturnHandler : public X86IncomingValueHandler { 312 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 313 CCAssignFn *AssignFn, MachineInstrBuilder &MIB) 314 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {} 315 316 void markPhysRegUsed(unsigned PhysReg) override { 317 MIB.addDef(PhysReg, RegState::Implicit); 318 } 319 320 protected: 321 MachineInstrBuilder &MIB; 322 }; 323 324 } // end anonymous namespace 325 326 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 327 const Function &F, 328 ArrayRef<ArrayRef<Register>> VRegs, 329 FunctionLoweringInfo &FLI) const { 330 if (F.arg_empty()) 331 return true; 332 333 // TODO: handle variadic function 334 if (F.isVarArg()) 335 return false; 336 337 MachineFunction &MF = MIRBuilder.getMF(); 338 MachineRegisterInfo &MRI = MF.getRegInfo(); 339 auto DL = MF.getDataLayout(); 340 341 SmallVector<ArgInfo, 8> SplitArgs; 342 unsigned Idx = 0; 343 for (const auto &Arg : F.args()) { 344 // TODO: handle not simple cases. 345 if (Arg.hasAttribute(Attribute::ByVal) || 346 Arg.hasAttribute(Attribute::InReg) || 347 Arg.hasAttribute(Attribute::StructRet) || 348 Arg.hasAttribute(Attribute::SwiftSelf) || 349 Arg.hasAttribute(Attribute::SwiftError) || 350 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 351 return false; 352 353 ArgInfo OrigArg(VRegs[Idx], Arg.getType()); 354 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 355 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 356 [&](ArrayRef<Register> Regs) { 357 MIRBuilder.buildMerge(VRegs[Idx][0], Regs); 358 })) 359 return false; 360 Idx++; 361 } 362 363 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 364 if (!MBB.empty()) 365 MIRBuilder.setInstr(*MBB.begin()); 366 367 FormalArgHandler Handler(MIRBuilder, MRI, CC_X86); 368 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 369 F.isVarArg())) 370 return false; 371 372 // Move back to the end of the basic block. 373 MIRBuilder.setMBB(MBB); 374 375 return true; 376 } 377 378 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 379 CallLoweringInfo &Info) const { 380 MachineFunction &MF = MIRBuilder.getMF(); 381 const Function &F = MF.getFunction(); 382 MachineRegisterInfo &MRI = MF.getRegInfo(); 383 const DataLayout &DL = F.getParent()->getDataLayout(); 384 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 385 const TargetInstrInfo &TII = *STI.getInstrInfo(); 386 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 387 388 // Handle only Linux C, X86_64_SysV calling conventions for now. 389 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 390 Info.CallConv == CallingConv::X86_64_SysV)) 391 return false; 392 393 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 394 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 395 396 // Create a temporarily-floating call instruction so we can add the implicit 397 // uses of arg registers. 398 bool Is64Bit = STI.is64Bit(); 399 unsigned CallOpc = Info.Callee.isReg() 400 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 401 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 402 403 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 404 .add(Info.Callee) 405 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 406 407 SmallVector<ArgInfo, 8> SplitArgs; 408 for (const auto &OrigArg : Info.OrigArgs) { 409 410 // TODO: handle not simple cases. 411 if (OrigArg.Flags[0].isByVal()) 412 return false; 413 414 if (OrigArg.Regs.size() > 1) 415 return false; 416 417 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 418 [&](ArrayRef<Register> Regs) { 419 MIRBuilder.buildUnmerge(Regs, OrigArg.Regs[0]); 420 })) 421 return false; 422 } 423 // Do the actual argument marshalling. 424 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86); 425 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 426 Info.IsVarArg)) 427 return false; 428 429 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 430 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 431 // From AMD64 ABI document: 432 // For calls that may call functions that use varargs or stdargs 433 // (prototype-less calls or calls to functions containing ellipsis (...) in 434 // the declaration) %al is used as hidden argument to specify the number 435 // of SSE registers used. The contents of %al do not need to match exactly 436 // the number of registers, but must be an ubound on the number of SSE 437 // registers used and is in the range 0 - 8 inclusive. 438 439 MIRBuilder.buildInstr(X86::MOV8ri) 440 .addDef(X86::AL) 441 .addImm(Handler.getNumXmmRegs()); 442 MIB.addUse(X86::AL, RegState::Implicit); 443 } 444 445 // Now we can add the actual call instruction to the correct basic block. 446 MIRBuilder.insertInstr(MIB); 447 448 // If Callee is a reg, since it is used by a target specific 449 // instruction, it must have a register class matching the 450 // constraint of that instruction. 451 if (Info.Callee.isReg()) 452 MIB->getOperand(0).setReg(constrainOperandRegClass( 453 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 454 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 455 0)); 456 457 // Finally we can copy the returned value back into its virtual-register. In 458 // symmetry with the arguments, the physical register must be an 459 // implicit-define of the call instruction. 460 461 if (!Info.OrigRet.Ty->isVoidTy()) { 462 if (Info.OrigRet.Regs.size() > 1) 463 return false; 464 465 SplitArgs.clear(); 466 SmallVector<Register, 8> NewRegs; 467 468 if (!splitToValueTypes(Info.OrigRet, SplitArgs, DL, MRI, 469 [&](ArrayRef<Register> Regs) { 470 NewRegs.assign(Regs.begin(), Regs.end()); 471 })) 472 return false; 473 474 CallReturnHandler Handler(MIRBuilder, MRI, RetCC_X86, MIB); 475 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 476 Info.IsVarArg)) 477 return false; 478 479 if (!NewRegs.empty()) 480 MIRBuilder.buildMerge(Info.OrigRet.Regs[0], NewRegs); 481 } 482 483 CallSeqStart.addImm(Handler.getStackSize()) 484 .addImm(0 /* see getFrameTotalSize */) 485 .addImm(0 /* see getFrameAdjustment */); 486 487 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 488 MIRBuilder.buildInstr(AdjStackUp) 489 .addImm(Handler.getStackSize()) 490 .addImm(0 /* NumBytesForCalleeToPop */); 491 492 return true; 493 } 494