1 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file implements the lowering of LLVM calls to machine code calls for 11 /// GlobalISel. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86CallLowering.h" 16 #include "X86CallingConv.h" 17 #include "X86ISelLowering.h" 18 #include "X86InstrInfo.h" 19 #include "X86RegisterInfo.h" 20 #include "X86Subtarget.h" 21 #include "llvm/ADT/ArrayRef.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/CodeGen/Analysis.h" 24 #include "llvm/CodeGen/CallingConvLower.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/GlobalISel/Utils.h" 27 #include "llvm/CodeGen/LowLevelType.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFrameInfo.h" 30 #include "llvm/CodeGen/MachineFunction.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineMemOperand.h" 33 #include "llvm/CodeGen/MachineOperand.h" 34 #include "llvm/CodeGen/MachineRegisterInfo.h" 35 #include "llvm/CodeGen/TargetInstrInfo.h" 36 #include "llvm/CodeGen/TargetSubtargetInfo.h" 37 #include "llvm/CodeGen/ValueTypes.h" 38 #include "llvm/IR/Attributes.h" 39 #include "llvm/IR/DataLayout.h" 40 #include "llvm/IR/Function.h" 41 #include "llvm/IR/Value.h" 42 #include "llvm/MC/MCRegisterInfo.h" 43 #include "llvm/Support/LowLevelTypeImpl.h" 44 #include "llvm/Support/MachineValueType.h" 45 #include <cassert> 46 #include <cstdint> 47 48 using namespace llvm; 49 50 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI) 51 : CallLowering(&TLI) {} 52 53 bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg, 54 SmallVectorImpl<ArgInfo> &SplitArgs, 55 const DataLayout &DL, 56 MachineRegisterInfo &MRI, 57 SplitArgTy PerformArgSplit) const { 58 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 59 LLVMContext &Context = OrigArg.Ty->getContext(); 60 61 SmallVector<EVT, 4> SplitVTs; 62 SmallVector<uint64_t, 4> Offsets; 63 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); 64 assert(OrigArg.Regs.size() == 1 && "Can't handle multple regs yet"); 65 66 if (OrigArg.Ty->isVoidTy()) 67 return true; 68 69 EVT VT = SplitVTs[0]; 70 unsigned NumParts = TLI.getNumRegisters(Context, VT); 71 72 if (NumParts == 1) { 73 // replace the original type ( pointer -> GPR ). 74 SplitArgs.emplace_back(OrigArg.Regs[0], VT.getTypeForEVT(Context), 75 OrigArg.Flags, OrigArg.IsFixed); 76 return true; 77 } 78 79 SmallVector<Register, 8> SplitRegs; 80 81 EVT PartVT = TLI.getRegisterType(Context, VT); 82 Type *PartTy = PartVT.getTypeForEVT(Context); 83 84 for (unsigned i = 0; i < NumParts; ++i) { 85 ArgInfo Info = 86 ArgInfo{MRI.createGenericVirtualRegister(getLLTForType(*PartTy, DL)), 87 PartTy, OrigArg.Flags}; 88 SplitArgs.push_back(Info); 89 SplitRegs.push_back(Info.Regs[0]); 90 } 91 92 PerformArgSplit(SplitRegs); 93 return true; 94 } 95 96 namespace { 97 98 struct X86OutgoingValueHandler : public CallLowering::IncomingValueHandler { 99 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder, 100 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB, 101 CCAssignFn *AssignFn) 102 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB), 103 DL(MIRBuilder.getMF().getDataLayout()), 104 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {} 105 106 Register getStackAddress(uint64_t Size, int64_t Offset, 107 MachinePointerInfo &MPO) override { 108 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0)); 109 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0)); 110 auto SPReg = 111 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister()); 112 113 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset); 114 115 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg); 116 117 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); 118 return AddrReg.getReg(0); 119 } 120 121 void assignValueToReg(Register ValVReg, Register PhysReg, 122 CCValAssign &VA) override { 123 MIB.addUse(PhysReg, RegState::Implicit); 124 125 Register ExtReg; 126 // If we are copying the value to a physical register with the 127 // size larger than the size of the value itself - build AnyExt 128 // to the size of the register first and only then do the copy. 129 // The example of that would be copying from s32 to xmm0, for which 130 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 131 // we expect normal extendRegister mechanism to work. 132 unsigned PhysRegSize = 133 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 134 unsigned ValSize = VA.getValVT().getSizeInBits(); 135 unsigned LocSize = VA.getLocVT().getSizeInBits(); 136 if (PhysRegSize > ValSize && LocSize == ValSize) { 137 assert((PhysRegSize == 128 || PhysRegSize == 80) && 138 "We expect that to be 128 bit"); 139 ExtReg = 140 MIRBuilder.buildAnyExt(LLT::scalar(PhysRegSize), ValVReg).getReg(0); 141 } else 142 ExtReg = extendRegister(ValVReg, VA); 143 144 MIRBuilder.buildCopy(PhysReg, ExtReg); 145 } 146 147 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 148 MachinePointerInfo &MPO, CCValAssign &VA) override { 149 MachineFunction &MF = MIRBuilder.getMF(); 150 Register ExtReg = extendRegister(ValVReg, VA); 151 152 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, 153 VA.getLocVT().getStoreSize(), 154 inferAlignFromPtrInfo(MF, MPO)); 155 MIRBuilder.buildStore(ExtReg, Addr, *MMO); 156 } 157 158 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, 159 CCValAssign::LocInfo LocInfo, 160 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags, 161 CCState &State) override { 162 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State); 163 StackSize = State.getNextStackOffset(); 164 165 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2, 166 X86::XMM3, X86::XMM4, X86::XMM5, 167 X86::XMM6, X86::XMM7}; 168 if (!Info.IsFixed) 169 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs); 170 171 return Res; 172 } 173 174 uint64_t getStackSize() { return StackSize; } 175 uint64_t getNumXmmRegs() { return NumXMMRegs; } 176 177 protected: 178 MachineInstrBuilder &MIB; 179 uint64_t StackSize = 0; 180 const DataLayout &DL; 181 const X86Subtarget &STI; 182 unsigned NumXMMRegs = 0; 183 }; 184 185 } // end anonymous namespace 186 187 bool X86CallLowering::lowerReturn( 188 MachineIRBuilder &MIRBuilder, const Value *Val, 189 ArrayRef<Register> VRegs) const { 190 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) && 191 "Return value without a vreg"); 192 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0); 193 194 if (!VRegs.empty()) { 195 MachineFunction &MF = MIRBuilder.getMF(); 196 const Function &F = MF.getFunction(); 197 MachineRegisterInfo &MRI = MF.getRegInfo(); 198 const DataLayout &DL = MF.getDataLayout(); 199 LLVMContext &Ctx = Val->getType()->getContext(); 200 const X86TargetLowering &TLI = *getTLI<X86TargetLowering>(); 201 202 SmallVector<EVT, 4> SplitEVTs; 203 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs); 204 assert(VRegs.size() == SplitEVTs.size() && 205 "For each split Type there should be exactly one VReg."); 206 207 SmallVector<ArgInfo, 8> SplitArgs; 208 for (unsigned i = 0; i < SplitEVTs.size(); ++i) { 209 ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)}; 210 setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F); 211 if (!splitToValueTypes(CurArgInfo, SplitArgs, DL, MRI, 212 [&](ArrayRef<Register> Regs) { 213 MIRBuilder.buildUnmerge(Regs, VRegs[i]); 214 })) 215 return false; 216 } 217 218 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, RetCC_X86); 219 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 220 return false; 221 } 222 223 MIRBuilder.insertInstr(MIB); 224 return true; 225 } 226 227 namespace { 228 229 struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler { 230 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder, 231 MachineRegisterInfo &MRI, CCAssignFn *AssignFn) 232 : IncomingValueHandler(MIRBuilder, MRI, AssignFn), 233 DL(MIRBuilder.getMF().getDataLayout()) {} 234 235 Register getStackAddress(uint64_t Size, int64_t Offset, 236 MachinePointerInfo &MPO) override { 237 auto &MFI = MIRBuilder.getMF().getFrameInfo(); 238 int FI = MFI.CreateFixedObject(Size, Offset, true); 239 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); 240 241 return MIRBuilder 242 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI) 243 .getReg(0); 244 } 245 246 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, 247 MachinePointerInfo &MPO, CCValAssign &VA) override { 248 MachineFunction &MF = MIRBuilder.getMF(); 249 auto *MMO = MF.getMachineMemOperand( 250 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, 251 inferAlignFromPtrInfo(MF, MPO)); 252 MIRBuilder.buildLoad(ValVReg, Addr, *MMO); 253 } 254 255 void assignValueToReg(Register ValVReg, Register PhysReg, 256 CCValAssign &VA) override { 257 markPhysRegUsed(PhysReg); 258 259 switch (VA.getLocInfo()) { 260 default: { 261 // If we are copying the value from a physical register with the 262 // size larger than the size of the value itself - build the copy 263 // of the phys reg first and then build the truncation of that copy. 264 // The example of that would be copying from xmm0 to s32, for which 265 // case ValVT == LocVT == MVT::f32. If LocSize and ValSize are not equal 266 // we expect this to be handled in SExt/ZExt/AExt case. 267 unsigned PhysRegSize = 268 MRI.getTargetRegisterInfo()->getRegSizeInBits(PhysReg, MRI); 269 unsigned ValSize = VA.getValVT().getSizeInBits(); 270 unsigned LocSize = VA.getLocVT().getSizeInBits(); 271 if (PhysRegSize > ValSize && LocSize == ValSize) { 272 auto Copy = MIRBuilder.buildCopy(LLT::scalar(PhysRegSize), PhysReg); 273 MIRBuilder.buildTrunc(ValVReg, Copy); 274 return; 275 } 276 277 MIRBuilder.buildCopy(ValVReg, PhysReg); 278 break; 279 } 280 case CCValAssign::LocInfo::SExt: 281 case CCValAssign::LocInfo::ZExt: 282 case CCValAssign::LocInfo::AExt: { 283 auto Copy = MIRBuilder.buildCopy(LLT{VA.getLocVT()}, PhysReg); 284 MIRBuilder.buildTrunc(ValVReg, Copy); 285 break; 286 } 287 } 288 } 289 290 /// How the physical register gets marked varies between formal 291 /// parameters (it's a basic-block live-in), and a call instruction 292 /// (it's an implicit-def of the BL). 293 virtual void markPhysRegUsed(unsigned PhysReg) = 0; 294 295 protected: 296 const DataLayout &DL; 297 }; 298 299 struct FormalArgHandler : public X86IncomingValueHandler { 300 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 301 CCAssignFn *AssignFn) 302 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn) {} 303 304 void markPhysRegUsed(unsigned PhysReg) override { 305 MIRBuilder.getMRI()->addLiveIn(PhysReg); 306 MIRBuilder.getMBB().addLiveIn(PhysReg); 307 } 308 }; 309 310 struct CallReturnHandler : public X86IncomingValueHandler { 311 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, 312 CCAssignFn *AssignFn, MachineInstrBuilder &MIB) 313 : X86IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {} 314 315 void markPhysRegUsed(unsigned PhysReg) override { 316 MIB.addDef(PhysReg, RegState::Implicit); 317 } 318 319 protected: 320 MachineInstrBuilder &MIB; 321 }; 322 323 } // end anonymous namespace 324 325 bool X86CallLowering::lowerFormalArguments( 326 MachineIRBuilder &MIRBuilder, const Function &F, 327 ArrayRef<ArrayRef<Register>> VRegs) const { 328 if (F.arg_empty()) 329 return true; 330 331 // TODO: handle variadic function 332 if (F.isVarArg()) 333 return false; 334 335 MachineFunction &MF = MIRBuilder.getMF(); 336 MachineRegisterInfo &MRI = MF.getRegInfo(); 337 auto DL = MF.getDataLayout(); 338 339 SmallVector<ArgInfo, 8> SplitArgs; 340 unsigned Idx = 0; 341 for (const auto &Arg : F.args()) { 342 // TODO: handle not simple cases. 343 if (Arg.hasAttribute(Attribute::ByVal) || 344 Arg.hasAttribute(Attribute::InReg) || 345 Arg.hasAttribute(Attribute::StructRet) || 346 Arg.hasAttribute(Attribute::SwiftSelf) || 347 Arg.hasAttribute(Attribute::SwiftError) || 348 Arg.hasAttribute(Attribute::Nest) || VRegs[Idx].size() > 1) 349 return false; 350 351 ArgInfo OrigArg(VRegs[Idx], Arg.getType()); 352 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F); 353 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 354 [&](ArrayRef<Register> Regs) { 355 MIRBuilder.buildMerge(VRegs[Idx][0], Regs); 356 })) 357 return false; 358 Idx++; 359 } 360 361 MachineBasicBlock &MBB = MIRBuilder.getMBB(); 362 if (!MBB.empty()) 363 MIRBuilder.setInstr(*MBB.begin()); 364 365 FormalArgHandler Handler(MIRBuilder, MRI, CC_X86); 366 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 367 return false; 368 369 // Move back to the end of the basic block. 370 MIRBuilder.setMBB(MBB); 371 372 return true; 373 } 374 375 bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, 376 CallLoweringInfo &Info) const { 377 MachineFunction &MF = MIRBuilder.getMF(); 378 const Function &F = MF.getFunction(); 379 MachineRegisterInfo &MRI = MF.getRegInfo(); 380 const DataLayout &DL = F.getParent()->getDataLayout(); 381 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); 382 const TargetInstrInfo &TII = *STI.getInstrInfo(); 383 const X86RegisterInfo *TRI = STI.getRegisterInfo(); 384 385 // Handle only Linux C, X86_64_SysV calling conventions for now. 386 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C || 387 Info.CallConv == CallingConv::X86_64_SysV)) 388 return false; 389 390 unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); 391 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown); 392 393 // Create a temporarily-floating call instruction so we can add the implicit 394 // uses of arg registers. 395 bool Is64Bit = STI.is64Bit(); 396 unsigned CallOpc = Info.Callee.isReg() 397 ? (Is64Bit ? X86::CALL64r : X86::CALL32r) 398 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32); 399 400 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc) 401 .add(Info.Callee) 402 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv)); 403 404 SmallVector<ArgInfo, 8> SplitArgs; 405 for (const auto &OrigArg : Info.OrigArgs) { 406 407 // TODO: handle not simple cases. 408 if (OrigArg.Flags[0].isByVal()) 409 return false; 410 411 if (OrigArg.Regs.size() > 1) 412 return false; 413 414 if (!splitToValueTypes(OrigArg, SplitArgs, DL, MRI, 415 [&](ArrayRef<Register> Regs) { 416 MIRBuilder.buildUnmerge(Regs, OrigArg.Regs[0]); 417 })) 418 return false; 419 } 420 // Do the actual argument marshalling. 421 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB, CC_X86); 422 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 423 return false; 424 425 bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed; 426 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) { 427 // From AMD64 ABI document: 428 // For calls that may call functions that use varargs or stdargs 429 // (prototype-less calls or calls to functions containing ellipsis (...) in 430 // the declaration) %al is used as hidden argument to specify the number 431 // of SSE registers used. The contents of %al do not need to match exactly 432 // the number of registers, but must be an ubound on the number of SSE 433 // registers used and is in the range 0 - 8 inclusive. 434 435 MIRBuilder.buildInstr(X86::MOV8ri) 436 .addDef(X86::AL) 437 .addImm(Handler.getNumXmmRegs()); 438 MIB.addUse(X86::AL, RegState::Implicit); 439 } 440 441 // Now we can add the actual call instruction to the correct basic block. 442 MIRBuilder.insertInstr(MIB); 443 444 // If Callee is a reg, since it is used by a target specific 445 // instruction, it must have a register class matching the 446 // constraint of that instruction. 447 if (Info.Callee.isReg()) 448 MIB->getOperand(0).setReg(constrainOperandRegClass( 449 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), 450 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee, 451 0)); 452 453 // Finally we can copy the returned value back into its virtual-register. In 454 // symmetry with the arguments, the physical register must be an 455 // implicit-define of the call instruction. 456 457 if (!Info.OrigRet.Ty->isVoidTy()) { 458 if (Info.OrigRet.Regs.size() > 1) 459 return false; 460 461 SplitArgs.clear(); 462 SmallVector<Register, 8> NewRegs; 463 464 if (!splitToValueTypes(Info.OrigRet, SplitArgs, DL, MRI, 465 [&](ArrayRef<Register> Regs) { 466 NewRegs.assign(Regs.begin(), Regs.end()); 467 })) 468 return false; 469 470 CallReturnHandler Handler(MIRBuilder, MRI, RetCC_X86, MIB); 471 if (!handleAssignments(MIRBuilder, SplitArgs, Handler)) 472 return false; 473 474 if (!NewRegs.empty()) 475 MIRBuilder.buildMerge(Info.OrigRet.Regs[0], NewRegs); 476 } 477 478 CallSeqStart.addImm(Handler.getStackSize()) 479 .addImm(0 /* see getFrameTotalSize */) 480 .addImm(0 /* see getFrameAdjustment */); 481 482 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); 483 MIRBuilder.buildInstr(AdjStackUp) 484 .addImm(Handler.getStackSize()) 485 .addImm(0 /* NumBytesForCalleeToPop */); 486 487 return true; 488 } 489