1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/GlobalISel/Utils.h" 27 #include "llvm/CodeGen/LowLevelType.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFrameInfo.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineMemOperand.h" 33 #include "llvm/CodeGen/MachineOperand.h" 34 #include "llvm/CodeGen/MachineRegisterInfo.h" 35 #include "llvm/CodeGen/TargetInstrInfo.h" 36 #include "llvm/CodeGen/TargetSubtargetInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/IR/Value.h" 42 #include "llvm/MC/MCRegisterInfo.h" 43 #include "llvm/Support/LowLevelTypeImpl.h" 44 #include "llvm/Support/MachineValueType.h" 45 #include <cassert> 46 #include <cstdint> 47 48 using namespace llvm; 49 50 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 51 : CallLowering(&TLI) {} 52 53 // FIXME: This should be removed and the generic version used 54 bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 55 SmallVectorImpl<ArgInfo> &SplitArgs, 56 const DataLayout &DL, 57 MachineRegisterInfo &MRI, 58 SplitArgTy PerformArgSplit) const { 59 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 60 LLVMContext &Context = OrigArg.Ty->getContext(); 61 62 SmallVector<EVT, 4> SplitVTs; 63 SmallVector<uint64_t, 4> Offsets; 64 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 65 assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); 66 67 if (OrigArg.Ty->isVoidTy()) 68 return true; 69 70 EVT VT = SplitVTs[0]; 71 unsigned NumParts = TLI.getNumRegisters(Context, VT); 72 73 if (NumParts == 1) { 74 // replace the original type ( pointer -> GPR ). 75 SplitArgs.emplace_back(OrigArg.Regs[0], VT.getTypeForEVT(Context), 76 OrigArg.Flags, OrigArg.IsFixed); 77 return true; 78 } 79 80 SmallVector<Register, 8> SplitRegs; 81 82 EVT PartVT = TLI.getRegisterType(Context, VT); 83 Type *PartTy = PartVT.getTypeForEVT(Context); 84 85 for (unsigned i = 0; i < NumParts; ++i) { 86 ArgInfo Info = 87 ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), 88 PartTy, OrigArg.Flags}; 89 SplitArgs.push_back(Info); 90 SplitRegs.push_back(Info.Regs[0]); 91 } 92 93 PerformArgSplit(SplitRegs); 94 return true; 95 } 96 97 namespace { 98 99 struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler { 100 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 101 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB, 102 CCAssignFn *AssignFn) 103 : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), 104 DL(MIRBuilder.getMF().getDataLayout()), 105 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 106 107 Register getStackAddress(uint64_t Size, int64_t Offset, 108 MachinePointerInfo &MPO) override { 109 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 110 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 111 auto SPReg = 112 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 113 114 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 115 116 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 117 118 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 119 return AddrReg.getReg(0); 120 } 121 122 void assignValueToReg(Register ValVReg, Register PhysReg, 123 CCValAssign &VA) override { 124 MIB.addUse(PhysReg, RegState::Implicit); 125 126 Register ExtReg; 127 // If we are copying the value to a physical register with the 128 // size larger than the size of the value itself - build AnyExt 129 // to the size of the register first and only then do the copy. 130 // The example of that would be copying from s32 to xmm0, for which 131 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 132 // we expect normal extendRegister mechanism to work. 133 unsigned PhysRegSize = 134 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 135 unsigned ValSize = VA.getValVT().getSizeInBits(); 136 unsigned LocSize = VA.getLocVT().getSizeInBits(); 137 if (PhysRegSize > ValSize && LocSize == ValSize) { 138 assert((PhysRegSize == 128 || PhysRegSize == 80) && 139 "We expect that to be 128 bit"); 140 ExtReg = 141 MIRBuilder.buildAnyExt(LLT::scalar(PhysRegSize), ValVReg).getReg(0); 142 } else 143 ExtReg = extendRegister(ValVReg, VA); 144 145 MIRBuilder.buildCopy(PhysReg, ExtReg); 146 } 147 148 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 149 MachinePointerInfo &MPO, CCValAssign &VA) override { 150 MachineFunction &MF = MIRBuilder.getMF(); 151 Register ExtReg = extendRegister(ValVReg, VA); 152 153 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, 154 VA.getLocVT().getStoreSize(), 155 inferAlignFromPtrInfo(MF, MPO)); 156 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 157 } 158 159 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, 160 CCValAssign::LocInfo LocInfo, 161 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 162 CCState &State) override { 163 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 164 StackSize = State.getNextStackOffset(); 165 166 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 167 X86::XMM3, X86::XMM4, X86::XMM5, 168 X86::XMM6, X86::XMM7}; 169 if (!Info.IsFixed) 170 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 171 172 return Res; 173 } 174 175 uint64_t getStackSize() { return StackSize; } 176 uint64_t getNumXmmRegs() { return NumXMMRegs; } 177 178 protected: 179 MachineInstrBuilder &MIB; 180 uint64_t StackSize = 0; 181 const DataLayout &DL; 182 const X86Subtarget &STI; 183 unsigned NumXMMRegs = 0; 184 }; 185 186 } // end anonymous namespace 187 188 bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, 189 const Value *Val, ArrayRef<Register> VRegs, 190 FunctionLoweringInfo &FLI) const { 191 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 192 "Return value without a vreg"); 193 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 194 195 if (!VRegs.empty()) { 196 MachineFunction &MF = MIRBuilder.getMF(); 197 const Function &F = MF.getFunction(); 198 MachineRegisterInfo &MRI = MF.getRegInfo(); 199 const DataLayout &DL = MF.getDataLayout(); 200 LLVMContext &Ctx = Val->getType()->getContext(); 201 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 202 203 SmallVector<EVT, 4> SplitEVTs; 204 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs); 205 assert(VRegs.size() == SplitEVTs.size() && 206 "For each split Type there should be exactly one VReg."); 207 208 SmallVector<ArgInfo, 8> SplitArgs; 209 for (unsigned i = 0; i < SplitEVTs.size(); ++i) { 210 ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)}; 211 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); 212 if (!splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, 213 [&](ArrayRef<Register> Regs) { 214 MIRBuilder.buildUnmerge(Regs, VRegs[i]); 215 })) 216 return false; 217 } 218 219 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); 220 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 221 F.isVarArg())) 222 return false; 223 } 224 225 MIRBuilder.insertInstr(MIB); 226 return true; 227 } 228 229 namespace { 230 231 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 232 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 233 MachineRegisterInfo &MRI, CCAssignFn *AssignFn) 234 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), 235 DL(MIRBuilder.getMF().getDataLayout()) {} 236 237 Register getStackAddress(uint64_t Size, int64_t Offset, 238 MachinePointerInfo &MPO) override { 239 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 240 int FI = MFI.CreateFixedObject(Size, Offset, true); 241 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 242 243 return MIRBuilder 244 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 245 .getReg(0); 246 } 247 248 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 249 MachinePointerInfo &MPO, CCValAssign &VA) override { 250 MachineFunction &MF = MIRBuilder.getMF(); 251 auto *MMO = MF.getMachineMemOperand( 252 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 253 inferAlignFromPtrInfo(MF, MPO)); 254 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 255 } 256 257 void assignValueToReg(Register ValVReg, Register PhysReg, 258 CCValAssign &VA) override { 259 markPhysRegUsed(PhysReg); 260 261 switch (VA.getLocInfo()) { 262 default: { 263 // If we are copying the value from a physical register with the 264 // size larger than the size of the value itself - build the copy 265 // of the phys reg first and then build the truncation of that copy. 266 // The example of that would be copying from xmm0 to s32, for which 267 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 268 // we expect this to be handled in SExt/ZExt/AExt case. 269 unsigned PhysRegSize = 270 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 271 unsigned ValSize = VA.getValVT().getSizeInBits(); 272 unsigned LocSize = VA.getLocVT().getSizeInBits(); 273 if (PhysRegSize > ValSize && LocSize == ValSize) { 274 auto Copy = MIRBuilder.buildCopy(LLT::scalar(PhysRegSize), PhysReg); 275 MIRBuilder.buildTrunc(ValVReg, Copy); 276 return; 277 } 278 279 MIRBuilder.buildCopy(ValVReg, PhysReg); 280 break; 281 } 282 case CCValAssign::LocInfo::SExt: 283 case CCValAssign::LocInfo::ZExt: 284 case CCValAssign::LocInfo::AExt: { 285 auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg); 286 MIRBuilder.buildTrunc(ValVReg, Copy); 287 break; 288 } 289 } 290 } 291 292 /// How the physical register gets marked varies between formal 293 /// parameters (it's a basic-block live-in), and a call instruction 294 /// (it's an implicit-def of the BL). 295 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 296 297 protected: 298 const DataLayout &DL; 299 }; 300 301 struct FormalArgHandler : public X86IncomingValueHandler { 302 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 303 CCAssignFn *AssignFn) 304 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn) {} 305 306 void markPhysRegUsed(unsigned PhysReg) override { 307 MIRBuilder.getMRI()->addLiveIn(PhysReg); 308 MIRBuilder.getMBB().addLiveIn(PhysReg); 309 } 310 }; 311 312 struct CallReturnHandler : public X86IncomingValueHandler { 313 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 314 CCAssignFn *AssignFn, MachineInstrBuilder &MIB) 315 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {} 316 317 void markPhysRegUsed(unsigned PhysReg) override { 318 MIB.addDef(PhysReg, RegState::Implicit); 319 } 320 321 protected: 322 MachineInstrBuilder &MIB; 323 }; 324 325 } // end anonymous namespace 326 327 bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, 328 const Function &F, 329 ArrayRef<ArrayRef<Register>> VRegs, 330 FunctionLoweringInfo &FLI) const { 331 if (F.arg_empty()) 332 return true; 333 334 // TODO: handle variadic function 335 if (F.isVarArg()) 336 return false; 337 338 MachineFunction &MF = MIRBuilder.getMF(); 339 MachineRegisterInfo &MRI = MF.getRegInfo(); 340 auto DL = MF.getDataLayout(); 341 342 SmallVector<ArgInfo, 8> SplitArgs; 343 unsigned Idx = 0; 344 for (const auto &Arg : F.args()) { 345 // TODO: handle not simple cases. 346 if (Arg.hasAttribute(Attribute::ByVal) || 347 Arg.hasAttribute(Attribute::InReg) || 348 Arg.hasAttribute(Attribute::StructRet) || 349 Arg.hasAttribute(Attribute::SwiftSelf) || 350 Arg.hasAttribute(Attribute::SwiftError) || 351 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 352 return false; 353 354 ArgInfo OrigArg(VRegs[Idx], Arg.getType()); 355 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 356 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 357 [&](ArrayRef<Register> Regs) { 358 MIRBuilder.buildMerge(VRegs[Idx][0], Regs); 359 })) 360 return false; 361 Idx++; 362 } 363 364 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 365 if (!MBB.empty()) 366 MIRBuilder.setInstr(*MBB.begin()); 367 368 FormalArgHandler Handler(MIRBuilder, MRI, CC_X86); 369 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, F.getCallingConv(), 370 F.isVarArg())) 371 return false; 372 373 // Move back to the end of the basic block. 374 MIRBuilder.setMBB(MBB); 375 376 return true; 377 } 378 379 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 380 CallLoweringInfo &Info) const { 381 MachineFunction &MF = MIRBuilder.getMF(); 382 const Function &F = MF.getFunction(); 383 MachineRegisterInfo &MRI = MF.getRegInfo(); 384 const DataLayout &DL = F.getParent()->getDataLayout(); 385 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 386 const TargetInstrInfo &TII = *STI.getInstrInfo(); 387 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 388 389 // Handle only Linux C, X86_64_SysV calling conventions for now. 390 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 391 Info.CallConv == CallingConv::X86_64_SysV)) 392 return false; 393 394 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 395 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 396 397 // Create a temporarily-floating call instruction so we can add the implicit 398 // uses of arg registers. 399 bool Is64Bit = STI.is64Bit(); 400 unsigned CallOpc = Info.Callee.isReg() 401 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 402 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 403 404 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 405 .add(Info.Callee) 406 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 407 408 SmallVector<ArgInfo, 8> SplitArgs; 409 for (const auto &OrigArg : Info.OrigArgs) { 410 411 // TODO: handle not simple cases. 412 if (OrigArg.Flags[0].isByVal()) 413 return false; 414 415 if (OrigArg.Regs.size() > 1) 416 return false; 417 418 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 419 [&](ArrayRef<Register> Regs) { 420 MIRBuilder.buildUnmerge(Regs, OrigArg.Regs[0]); 421 })) 422 return false; 423 } 424 // Do the actual argument marshalling. 425 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86); 426 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 427 Info.IsVarArg)) 428 return false; 429 430 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 431 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 432 // From AMD64 ABI document: 433 // For calls that may call functions that use varargs or stdargs 434 // (prototype-less calls or calls to functions containing ellipsis (...) in 435 // the declaration) %al is used as hidden argument to specify the number 436 // of SSE registers used. The contents of %al do not need to match exactly 437 // the number of registers, but must be an ubound on the number of SSE 438 // registers used and is in the range 0 - 8 inclusive. 439 440 MIRBuilder.buildInstr(X86::MOV8ri) 441 .addDef(X86::AL) 442 .addImm(Handler.getNumXmmRegs()); 443 MIB.addUse(X86::AL, RegState::Implicit); 444 } 445 446 // Now we can add the actual call instruction to the correct basic block. 447 MIRBuilder.insertInstr(MIB); 448 449 // If Callee is a reg, since it is used by a target specific 450 // instruction, it must have a register class matching the 451 // constraint of that instruction. 452 if (Info.Callee.isReg()) 453 MIB->getOperand(0).setReg(constrainOperandRegClass( 454 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 455 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 456 0)); 457 458 // Finally we can copy the returned value back into its virtual-register. In 459 // symmetry with the arguments, the physical register must be an 460 // implicit-define of the call instruction. 461 462 if (!Info.OrigRet.Ty->isVoidTy()) { 463 if (Info.OrigRet.Regs.size() > 1) 464 return false; 465 466 SplitArgs.clear(); 467 SmallVector<Register, 8> NewRegs; 468 469 if (!splitToValueTypes(Info.OrigRet, SplitArgs, DL, MRI, 470 [&](ArrayRef<Register> Regs) { 471 NewRegs.assign(Regs.begin(), Regs.end()); 472 })) 473 return false; 474 475 CallReturnHandler Handler(MIRBuilder, MRI, RetCC_X86, MIB); 476 if (!handleAssignments(MIRBuilder, SplitArgs, Handler, Info.CallConv, 477 Info.IsVarArg)) 478 return false; 479 480 if (!NewRegs.empty()) 481 MIRBuilder.buildMerge(Info.OrigRet.Regs[0], NewRegs); 482 } 483 484 CallSeqStart.addImm(Handler.getStackSize()) 485 .addImm(0 /* see getFrameTotalSize */) 486 .addImm(0 /* see getFrameAdjustment */); 487 488 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 489 MIRBuilder.buildInstr(AdjStackUp) 490 .addImm(Handler.getStackSize()) 491 .addImm(0 /* NumBytesForCalleeToPop */); 492 493 return true; 494 } 495