1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/GlobalISel/Utils.h" 27 #include "llvm/CodeGen/LowLevelType.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFrameInfo.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineMemOperand.h" 33 #include "llvm/CodeGen/MachineOperand.h" 34 #include "llvm/CodeGen/MachineRegisterInfo.h" 35 #include "llvm/CodeGen/TargetInstrInfo.h" 36 #include "llvm/CodeGen/TargetSubtargetInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/IR/Value.h" 42 #include "llvm/MC/MCRegisterInfo.h" 43 #include "llvm/Support/LowLevelTypeImpl.h" 44 #include "llvm/Support/MachineValueType.h" 45 #include <cassert> 46 #include <cstdint> 47 48 using namespace llvm; 49 50 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 51 : CallLowering(&TLI) {} 52 53 // FIXME: This should be removed and the generic version used 54 bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 55 SmallVectorImpl<ArgInfo> &SplitArgs, 56 const DataLayout &DL, 57 MachineRegisterInfo &MRI, 58 SplitArgTy PerformArgSplit) const { 59 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 60 LLVMContext &Context = OrigArg.Ty->getContext(); 61 62 SmallVector<EVT, 4> SplitVTs; 63 SmallVector<uint64_t, 4> Offsets; 64 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 65 assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); 66 67 if (OrigArg.Ty->isVoidTy()) 68 return true; 69 70 EVT VT = SplitVTs[0]; 71 unsigned NumParts = TLI.getNumRegisters(Context, VT); 72 73 if (NumParts == 1) { 74 // replace the original type ( pointer -> GPR ). 75 SplitArgs.emplace_back(OrigArg.Regs[0], VT.getTypeForEVT(Context), 76 OrigArg.Flags, OrigArg.IsFixed); 77 return true; 78 } 79 80 SmallVector<Register, 8> SplitRegs; 81 82 EVT PartVT = TLI.getRegisterType(Context, VT); 83 Type *PartTy = PartVT.getTypeForEVT(Context); 84 85 for (unsigned i = 0; i < NumParts; ++i) { 86 ArgInfo Info = 87 ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), 88 PartTy, OrigArg.Flags}; 89 SplitArgs.push_back(Info); 90 SplitRegs.push_back(Info.Regs[0]); 91 } 92 93 PerformArgSplit(SplitRegs); 94 return true; 95 } 96 97 namespace { 98 99 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 100 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 101 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB, 102 CCAssignFn *AssignFn) 103 : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), 104 DL(MIRBuilder.getMF().getDataLayout()), 105 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 106 107 Register getStackAddress(uint64_t Size, int64_t Offset, 108 MachinePointerInfo &MPO, 109 ISD::ArgFlagsTy Flags) override { 110 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 111 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 112 auto SPReg = 113 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 114 115 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 116 117 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 118 119 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 120 return AddrReg.getReg(0); 121 } 122 123 void assignValueToReg(Register ValVReg, Register PhysReg, 124 CCValAssign &VA) override { 125 MIB.addUse(PhysReg, RegState::Implicit); 126 127 Register ExtReg; 128 // If we are copying the value to a physical register with the 129 // size larger than the size of the value itself - build AnyExt 130 // to the size of the register first and only then do the copy. 131 // The example of that would be copying from s32 to xmm0, for which 132 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 133 // we expect normal extendRegister mechanism to work. 134 unsigned PhysRegSize = 135 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 136 unsigned ValSize = VA.getValVT().getSizeInBits(); 137 unsigned LocSize = VA.getLocVT().getSizeInBits(); 138 if (PhysRegSize > ValSize && LocSize == ValSize) { 139 assert((PhysRegSize == 128 || PhysRegSize == 80) && 140 "We expect that to be 128 bit"); 141 ExtReg = 142 MIRBuilder.buildAnyExt(LLT::scalar(PhysRegSize), ValVReg).getReg(0); 143 } else 144 ExtReg = extendRegister(ValVReg, VA); 145 146 MIRBuilder.buildCopy(PhysReg, ExtReg); 147 } 148 149 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 150 MachinePointerInfo &MPO, CCValAssign &VA) override { 151 MachineFunction &MF = MIRBuilder.getMF(); 152 Register ExtReg = extendRegister(ValVReg, VA); 153 154 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, 155 VA.getLocVT().getStoreSize(), 156 inferAlignFromPtrInfo(MF, MPO)); 157 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 158 } 159 160 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, 161 CCValAssign::LocInfo LocInfo, 162 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 163 CCState &State) override { 164 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 165 StackSize = State.getNextStackOffset(); 166 167 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 168 X86::XMM3, X86::XMM4, X86::XMM5, 169 X86::XMM6, X86::XMM7}; 170 if (!Info.IsFixed) 171 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 172 173 return Res; 174 } 175 176 uint64_t getStackSize() { return StackSize; } 177 uint64_t getNumXmmRegs() { return NumXMMRegs; } 178 179 protected: 180 MachineInstrBuilder &MIB; 181 uint64_t StackSize = 0; 182 const DataLayout &DL; 183 const X86Subtarget &STI; 184 unsigned NumXMMRegs = 0; 185 }; 186 187 } // end anonymous namespace 188 189 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 190 const Value *Val, ArrayRef<Register> VRegs, 191 FunctionLoweringInfo &FLI) const { 192 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 193 "Return value without a vreg"); 194 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 195 196 if (!VRegs.empty()) { 197 MachineFunction &MF = MIRBuilder.getMF(); 198 const Function &F = MF.getFunction(); 199 MachineRegisterInfo &MRI = MF.getRegInfo(); 200 const DataLayout &DL = MF.getDataLayout(); 201 LLVMContext &Ctx = Val->getType()->getContext(); 202 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 203 204 SmallVector<EVT, 4> SplitEVTs; 205 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs); 206 assert(VRegs.size() == SplitEVTs.size() && 207 "For each split Type there should be exactly one VReg."); 208 209 SmallVector<ArgInfo, 8> SplitArgs; 210 for (unsigned i = 0; i < SplitEVTs.size(); ++i) { 211 ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)}; 212 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); 213 if (!splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, 214 [&](ArrayRef<Register> Regs) { 215 MIRBuilder.buildUnmerge(Regs, VRegs[i]); 216 })) 217 return false; 218 } 219 220 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); 221 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 222 F.isVarArg())) 223 return false; 224 } 225 226 MIRBuilder.insertInstr(MIB); 227 return true; 228 } 229 230 namespace { 231 232 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 233 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 234 MachineRegisterInfo &MRI, CCAssignFn *AssignFn) 235 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), 236 DL(MIRBuilder.getMF().getDataLayout()) {} 237 238 Register getStackAddress(uint64_t Size, int64_t Offset, 239 MachinePointerInfo &MPO, 240 ISD::ArgFlagsTy Flags) override { 241 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 242 243 // Byval is assumed to be writable memory, but other stack passed arguments 244 // are not. 245 const bool IsImmutable = !Flags.isByVal(); 246 247 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable); 248 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 249 250 return MIRBuilder 251 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 252 .getReg(0); 253 } 254 255 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 256 MachinePointerInfo &MPO, CCValAssign &VA) override { 257 MachineFunction &MF = MIRBuilder.getMF(); 258 auto *MMO = MF.getMachineMemOperand( 259 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 260 inferAlignFromPtrInfo(MF, MPO)); 261 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 262 } 263 264 void assignValueToReg(Register ValVReg, Register PhysReg, 265 CCValAssign &VA) override { 266 markPhysRegUsed(PhysReg); 267 268 switch (VA.getLocInfo()) { 269 default: { 270 // If we are copying the value from a physical register with the 271 // size larger than the size of the value itself - build the copy 272 // of the phys reg first and then build the truncation of that copy. 273 // The example of that would be copying from xmm0 to s32, for which 274 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 275 // we expect this to be handled in SExt/ZExt/AExt case. 276 unsigned PhysRegSize = 277 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 278 unsigned ValSize = VA.getValVT().getSizeInBits(); 279 unsigned LocSize = VA.getLocVT().getSizeInBits(); 280 if (PhysRegSize > ValSize && LocSize == ValSize) { 281 auto Copy = MIRBuilder.buildCopy(LLT::scalar(PhysRegSize), PhysReg); 282 MIRBuilder.buildTrunc(ValVReg, Copy); 283 return; 284 } 285 286 MIRBuilder.buildCopy(ValVReg, PhysReg); 287 break; 288 } 289 case CCValAssign::LocInfo::SExt: 290 case CCValAssign::LocInfo::ZExt: 291 case CCValAssign::LocInfo::AExt: { 292 auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg); 293 MIRBuilder.buildTrunc(ValVReg, Copy); 294 break; 295 } 296 } 297 } 298 299 /// How the physical register gets marked varies between formal 300 /// parameters (it's a basic-block live-in), and a call instruction 301 /// (it's an implicit-def of the BL). 302 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 303 304 protected: 305 const DataLayout &DL; 306 }; 307 308 struct FormalArgHandler : public X86IncomingValueHandler { 309 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 310 CCAssignFn *AssignFn) 311 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn) {} 312 313 void markPhysRegUsed(unsigned PhysReg) override { 314 MIRBuilder.getMRI()->addLiveIn(PhysReg); 315 MIRBuilder.getMBB().addLiveIn(PhysReg); 316 } 317 }; 318 319 struct CallReturnHandler : public X86IncomingValueHandler { 320 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 321 CCAssignFn *AssignFn, MachineInstrBuilder &MIB) 322 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {} 323 324 void markPhysRegUsed(unsigned PhysReg) override { 325 MIB.addDef(PhysReg, RegState::Implicit); 326 } 327 328 protected: 329 MachineInstrBuilder &MIB; 330 }; 331 332 } // end anonymous namespace 333 334 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 335 const Function &F, 336 ArrayRef<ArrayRef<Register>> VRegs, 337 FunctionLoweringInfo &FLI) const { 338 if (F.arg_empty()) 339 return true; 340 341 // TODO: handle variadic function 342 if (F.isVarArg()) 343 return false; 344 345 MachineFunction &MF = MIRBuilder.getMF(); 346 MachineRegisterInfo &MRI = MF.getRegInfo(); 347 auto DL = MF.getDataLayout(); 348 349 SmallVector<ArgInfo, 8> SplitArgs; 350 unsigned Idx = 0; 351 for (const auto &Arg : F.args()) { 352 // TODO: handle not simple cases. 353 if (Arg.hasAttribute(Attribute::ByVal) || 354 Arg.hasAttribute(Attribute::InReg) || 355 Arg.hasAttribute(Attribute::StructRet) || 356 Arg.hasAttribute(Attribute::SwiftSelf) || 357 Arg.hasAttribute(Attribute::SwiftError) || 358 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 359 return false; 360 361 ArgInfo OrigArg(VRegs[Idx], Arg.getType()); 362 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 363 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 364 [&](ArrayRef<Register> Regs) { 365 MIRBuilder.buildMerge(VRegs[Idx][0], Regs); 366 })) 367 return false; 368 Idx++; 369 } 370 371 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 372 if (!MBB.empty()) 373 MIRBuilder.setInstr(*MBB.begin()); 374 375 FormalArgHandler Handler(MIRBuilder, MRI, CC_X86); 376 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 377 F.isVarArg())) 378 return false; 379 380 // Move back to the end of the basic block. 381 MIRBuilder.setMBB(MBB); 382 383 return true; 384 } 385 386 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 387 CallLoweringInfo &Info) const { 388 MachineFunction &MF = MIRBuilder.getMF(); 389 const Function &F = MF.getFunction(); 390 MachineRegisterInfo &MRI = MF.getRegInfo(); 391 const DataLayout &DL = F.getParent()->getDataLayout(); 392 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 393 const TargetInstrInfo &TII = *STI.getInstrInfo(); 394 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 395 396 // Handle only Linux C, X86_64_SysV calling conventions for now. 397 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 398 Info.CallConv == CallingConv::X86_64_SysV)) 399 return false; 400 401 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 402 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 403 404 // Create a temporarily-floating call instruction so we can add the implicit 405 // uses of arg registers. 406 bool Is64Bit = STI.is64Bit(); 407 unsigned CallOpc = Info.Callee.isReg() 408 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 409 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 410 411 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 412 .add(Info.Callee) 413 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 414 415 SmallVector<ArgInfo, 8> SplitArgs; 416 for (const auto &OrigArg : Info.OrigArgs) { 417 418 // TODO: handle not simple cases. 419 if (OrigArg.Flags[0].isByVal()) 420 return false; 421 422 if (OrigArg.Regs.size() > 1) 423 return false; 424 425 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 426 [&](ArrayRef<Register> Regs) { 427 MIRBuilder.buildUnmerge(Regs, OrigArg.Regs[0]); 428 })) 429 return false; 430 } 431 // Do the actual argument marshalling. 432 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86); 433 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 434 Info.IsVarArg)) 435 return false; 436 437 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 438 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 439 // From AMD64 ABI document: 440 // For calls that may call functions that use varargs or stdargs 441 // (prototype-less calls or calls to functions containing ellipsis (...) in 442 // the declaration) %al is used as hidden argument to specify the number 443 // of SSE registers used. The contents of %al do not need to match exactly 444 // the number of registers, but must be an ubound on the number of SSE 445 // registers used and is in the range 0 - 8 inclusive. 446 447 MIRBuilder.buildInstr(X86::MOV8ri) 448 .addDef(X86::AL) 449 .addImm(Handler.getNumXmmRegs()); 450 MIB.addUse(X86::AL, RegState::Implicit); 451 } 452 453 // Now we can add the actual call instruction to the correct basic block. 454 MIRBuilder.insertInstr(MIB); 455 456 // If Callee is a reg, since it is used by a target specific 457 // instruction, it must have a register class matching the 458 // constraint of that instruction. 459 if (Info.Callee.isReg()) 460 MIB->getOperand(0).setReg(constrainOperandRegClass( 461 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 462 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 463 0)); 464 465 // Finally we can copy the returned value back into its virtual-register. In 466 // symmetry with the arguments, the physical register must be an 467 // implicit-define of the call instruction. 468 469 if (!Info.OrigRet.Ty->isVoidTy()) { 470 if (Info.OrigRet.Regs.size() > 1) 471 return false; 472 473 SplitArgs.clear(); 474 SmallVector<Register, 8> NewRegs; 475 476 if (!splitToValueTypes(Info.OrigRet, SplitArgs, DL, MRI, 477 [&](ArrayRef<Register> Regs) { 478 NewRegs.assign(Regs.begin(), Regs.end()); 479 })) 480 return false; 481 482 CallReturnHandler Handler(MIRBuilder, MRI, RetCC_X86, MIB); 483 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 484 Info.IsVarArg)) 485 return false; 486 487 if (!NewRegs.empty()) 488 MIRBuilder.buildMerge(Info.OrigRet.Regs[0], NewRegs); 489 } 490 491 CallSeqStart.addImm(Handler.getStackSize()) 492 .addImm(0 /* see getFrameTotalSize */) 493 .addImm(0 /* see getFrameAdjustment */); 494 495 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 496 MIRBuilder.buildInstr(AdjStackUp) 497 .addImm(Handler.getStackSize()) 498 .addImm(0 /* NumBytesForCalleeToPop */); 499 500 return true; 501 } 502