1 //===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the InstructionSelector class for 10 /// AMDGPU. 11 /// \todo This should be generated by TableGen. 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPUInstructionSelector.h" 15 #include "AMDGPUInstrInfo.h" 16 #include "AMDGPUGlobalISelUtils.h" 17 #include "AMDGPURegisterBankInfo.h" 18 #include "AMDGPUSubtarget.h" 19 #include "AMDGPUTargetMachine.h" 20 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 21 #include "SIMachineFunctionInfo.h" 22 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" 23 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 24 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h" 25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 26 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" 27 #include "llvm/CodeGen/GlobalISel/Utils.h" 28 #include "llvm/CodeGen/MachineBasicBlock.h" 29 #include "llvm/CodeGen/MachineFunction.h" 30 #include "llvm/CodeGen/MachineInstr.h" 31 #include "llvm/CodeGen/MachineInstrBuilder.h" 32 #include "llvm/CodeGen/MachineRegisterInfo.h" 33 #include "llvm/IR/Type.h" 34 #include "llvm/Support/Debug.h" 35 #include "llvm/Support/raw_ostream.h" 36 37 #define DEBUG_TYPE "amdgpu-isel" 38 39 using namespace llvm; 40 using namespace MIPatternMatch; 41 42 #define GET_GLOBALISEL_IMPL 43 #define AMDGPUSubtarget GCNSubtarget 44 #include "AMDGPUGenGlobalISel.inc" 45 #undef GET_GLOBALISEL_IMPL 46 #undef AMDGPUSubtarget 47 48 AMDGPUInstructionSelector::AMDGPUInstructionSelector( 49 const GCNSubtarget &STI, const AMDGPURegisterBankInfo &RBI, 50 const AMDGPUTargetMachine &TM) 51 : InstructionSelector(), TII(*STI.getInstrInfo()), 52 TRI(*STI.getRegisterInfo()), RBI(RBI), TM(TM), 53 STI(STI), 54 EnableLateStructurizeCFG(AMDGPUTargetMachine::EnableLateStructurizeCFG), 55 #define GET_GLOBALISEL_PREDICATES_INIT 56 #include "AMDGPUGenGlobalISel.inc" 57 #undef GET_GLOBALISEL_PREDICATES_INIT 58 #define GET_GLOBALISEL_TEMPORARIES_INIT 59 #include "AMDGPUGenGlobalISel.inc" 60 #undef GET_GLOBALISEL_TEMPORARIES_INIT 61 { 62 } 63 64 const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; } 65 66 void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits &KB, 67 CodeGenCoverage &CoverageInfo) { 68 MRI = &MF.getRegInfo(); 69 InstructionSelector::setupMF(MF, KB, CoverageInfo); 70 } 71 72 bool AMDGPUInstructionSelector::isVCC(Register Reg, 73 const MachineRegisterInfo &MRI) const { 74 if (Register::isPhysicalRegister(Reg)) 75 return Reg == TRI.getVCC(); 76 77 auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); 78 const TargetRegisterClass *RC = 79 RegClassOrBank.dyn_cast<const TargetRegisterClass*>(); 80 if (RC) { 81 const LLT Ty = MRI.getType(Reg); 82 return RC->hasSuperClassEq(TRI.getBoolRC()) && 83 Ty.isValid() && Ty.getSizeInBits() == 1; 84 } 85 86 const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>(); 87 return RB->getID() == AMDGPU::VCCRegBankID; 88 } 89 90 bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI, 91 unsigned NewOpc) const { 92 MI.setDesc(TII.get(NewOpc)); 93 MI.RemoveOperand(1); // Remove intrinsic ID. 94 MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 95 96 MachineOperand &Dst = MI.getOperand(0); 97 MachineOperand &Src = MI.getOperand(1); 98 99 // TODO: This should be legalized to s32 if needed 100 if (MRI->getType(Dst.getReg()) == LLT::scalar(1)) 101 return false; 102 103 const TargetRegisterClass *DstRC 104 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); 105 const TargetRegisterClass *SrcRC 106 = TRI.getConstrainedRegClassForOperand(Src, *MRI); 107 if (!DstRC || DstRC != SrcRC) 108 return false; 109 110 return RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI) && 111 RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI); 112 } 113 114 bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const { 115 const DebugLoc &DL = I.getDebugLoc(); 116 MachineBasicBlock *BB = I.getParent(); 117 I.setDesc(TII.get(TargetOpcode::COPY)); 118 119 const MachineOperand &Src = I.getOperand(1); 120 MachineOperand &Dst = I.getOperand(0); 121 Register DstReg = Dst.getReg(); 122 Register SrcReg = Src.getReg(); 123 124 if (isVCC(DstReg, *MRI)) { 125 if (SrcReg == AMDGPU::SCC) { 126 const TargetRegisterClass *RC 127 = TRI.getConstrainedRegClassForOperand(Dst, *MRI); 128 if (!RC) 129 return true; 130 return RBI.constrainGenericRegister(DstReg, *RC, *MRI); 131 } 132 133 if (!isVCC(SrcReg, *MRI)) { 134 // TODO: Should probably leave the copy and let copyPhysReg expand it. 135 if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI)) 136 return false; 137 138 const TargetRegisterClass *SrcRC 139 = TRI.getConstrainedRegClassForOperand(Src, *MRI); 140 141 Register MaskedReg = MRI->createVirtualRegister(SrcRC); 142 143 // We can't trust the high bits at this point, so clear them. 144 145 // TODO: Skip masking high bits if def is known boolean. 146 147 unsigned AndOpc = TRI.isSGPRClass(SrcRC) ? 148 AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; 149 BuildMI(*BB, &I, DL, TII.get(AndOpc), MaskedReg) 150 .addImm(1) 151 .addReg(SrcReg); 152 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) 153 .addImm(0) 154 .addReg(MaskedReg); 155 156 if (!MRI->getRegClassOrNull(SrcReg)) 157 MRI->setRegClass(SrcReg, SrcRC); 158 I.eraseFromParent(); 159 return true; 160 } 161 162 const TargetRegisterClass *RC = 163 TRI.getConstrainedRegClassForOperand(Dst, *MRI); 164 if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI)) 165 return false; 166 167 // Don't constrain the source register to a class so the def instruction 168 // handles it (unless it's undef). 169 // 170 // FIXME: This is a hack. When selecting the def, we neeed to know 171 // specifically know that the result is VCCRegBank, and not just an SGPR 172 // with size 1. An SReg_32 with size 1 is ambiguous with wave32. 173 if (Src.isUndef()) { 174 const TargetRegisterClass *SrcRC = 175 TRI.getConstrainedRegClassForOperand(Src, *MRI); 176 if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) 177 return false; 178 } 179 180 return true; 181 } 182 183 for (const MachineOperand &MO : I.operands()) { 184 if (Register::isPhysicalRegister(MO.getReg())) 185 continue; 186 187 const TargetRegisterClass *RC = 188 TRI.getConstrainedRegClassForOperand(MO, *MRI); 189 if (!RC) 190 continue; 191 RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI); 192 } 193 return true; 194 } 195 196 bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const { 197 const Register DefReg = I.getOperand(0).getReg(); 198 const LLT DefTy = MRI->getType(DefReg); 199 200 // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy) 201 202 const RegClassOrRegBank &RegClassOrBank = 203 MRI->getRegClassOrRegBank(DefReg); 204 205 const TargetRegisterClass *DefRC 206 = RegClassOrBank.dyn_cast<const TargetRegisterClass *>(); 207 if (!DefRC) { 208 if (!DefTy.isValid()) { 209 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n"); 210 return false; 211 } 212 213 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); 214 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, *MRI); 215 if (!DefRC) { 216 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); 217 return false; 218 } 219 } 220 221 // TODO: Verify that all registers have the same bank 222 I.setDesc(TII.get(TargetOpcode::PHI)); 223 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI); 224 } 225 226 MachineOperand 227 AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO, 228 const TargetRegisterClass &SubRC, 229 unsigned SubIdx) const { 230 231 MachineInstr *MI = MO.getParent(); 232 MachineBasicBlock *BB = MO.getParent()->getParent(); 233 Register DstReg = MRI->createVirtualRegister(&SubRC); 234 235 if (MO.isReg()) { 236 unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx); 237 Register Reg = MO.getReg(); 238 BuildMI(*BB, MI, MI->getDebugLoc(), TII.get(AMDGPU::COPY), DstReg) 239 .addReg(Reg, 0, ComposedSubIdx); 240 241 return MachineOperand::CreateReg(DstReg, MO.isDef(), MO.isImplicit(), 242 MO.isKill(), MO.isDead(), MO.isUndef(), 243 MO.isEarlyClobber(), 0, MO.isDebug(), 244 MO.isInternalRead()); 245 } 246 247 assert(MO.isImm()); 248 249 APInt Imm(64, MO.getImm()); 250 251 switch (SubIdx) { 252 default: 253 llvm_unreachable("do not know to split immediate with this sub index."); 254 case AMDGPU::sub0: 255 return MachineOperand::CreateImm(Imm.getLoBits(32).getSExtValue()); 256 case AMDGPU::sub1: 257 return MachineOperand::CreateImm(Imm.getHiBits(32).getSExtValue()); 258 } 259 } 260 261 static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) { 262 switch (Opc) { 263 case AMDGPU::G_AND: 264 return Is64 ? AMDGPU::S_AND_B64 : AMDGPU::S_AND_B32; 265 case AMDGPU::G_OR: 266 return Is64 ? AMDGPU::S_OR_B64 : AMDGPU::S_OR_B32; 267 case AMDGPU::G_XOR: 268 return Is64 ? AMDGPU::S_XOR_B64 : AMDGPU::S_XOR_B32; 269 default: 270 llvm_unreachable("not a bit op"); 271 } 272 } 273 274 bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const { 275 MachineOperand &Dst = I.getOperand(0); 276 MachineOperand &Src0 = I.getOperand(1); 277 MachineOperand &Src1 = I.getOperand(2); 278 Register DstReg = Dst.getReg(); 279 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); 280 281 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 282 if (DstRB->getID() == AMDGPU::VCCRegBankID) { 283 const TargetRegisterClass *RC = TRI.getBoolRC(); 284 unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(), 285 RC == &AMDGPU::SReg_64RegClass); 286 I.setDesc(TII.get(InstOpc)); 287 // Dead implicit-def of scc 288 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef 289 true, // isImp 290 false, // isKill 291 true)); // isDead 292 293 // FIXME: Hack to avoid turning the register bank into a register class. 294 // The selector for G_ICMP relies on seeing the register bank for the result 295 // is VCC. In wave32 if we constrain the registers to SReg_32 here, it will 296 // be ambiguous whether it's a scalar or vector bool. 297 if (Src0.isUndef() && !MRI->getRegClassOrNull(Src0.getReg())) 298 MRI->setRegClass(Src0.getReg(), RC); 299 if (Src1.isUndef() && !MRI->getRegClassOrNull(Src1.getReg())) 300 MRI->setRegClass(Src1.getReg(), RC); 301 302 return RBI.constrainGenericRegister(DstReg, *RC, *MRI); 303 } 304 305 // TODO: Should this allow an SCC bank result, and produce a copy from SCC for 306 // the result? 307 if (DstRB->getID() == AMDGPU::SGPRRegBankID) { 308 unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(), Size > 32); 309 I.setDesc(TII.get(InstOpc)); 310 // Dead implicit-def of scc 311 I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef 312 true, // isImp 313 false, // isKill 314 true)); // isDead 315 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 316 } 317 318 return false; 319 } 320 321 bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const { 322 MachineBasicBlock *BB = I.getParent(); 323 MachineFunction *MF = BB->getParent(); 324 Register DstReg = I.getOperand(0).getReg(); 325 const DebugLoc &DL = I.getDebugLoc(); 326 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); 327 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 328 const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID; 329 const bool Sub = I.getOpcode() == TargetOpcode::G_SUB; 330 331 if (Size == 32) { 332 if (IsSALU) { 333 const unsigned Opc = Sub ? AMDGPU::S_SUB_U32 : AMDGPU::S_ADD_U32; 334 MachineInstr *Add = 335 BuildMI(*BB, &I, DL, TII.get(Opc), DstReg) 336 .add(I.getOperand(1)) 337 .add(I.getOperand(2)); 338 I.eraseFromParent(); 339 return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI); 340 } 341 342 if (STI.hasAddNoCarry()) { 343 const unsigned Opc = Sub ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_ADD_U32_e64; 344 I.setDesc(TII.get(Opc)); 345 I.addOperand(*MF, MachineOperand::CreateImm(0)); 346 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 347 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 348 } 349 350 const unsigned Opc = Sub ? AMDGPU::V_SUB_I32_e64 : AMDGPU::V_ADD_I32_e64; 351 352 Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass()); 353 MachineInstr *Add 354 = BuildMI(*BB, &I, DL, TII.get(Opc), DstReg) 355 .addDef(UnusedCarry, RegState::Dead) 356 .add(I.getOperand(1)) 357 .add(I.getOperand(2)) 358 .addImm(0); 359 I.eraseFromParent(); 360 return constrainSelectedInstRegOperands(*Add, TII, TRI, RBI); 361 } 362 363 assert(!Sub && "illegal sub should not reach here"); 364 365 const TargetRegisterClass &RC 366 = IsSALU ? AMDGPU::SReg_64_XEXECRegClass : AMDGPU::VReg_64RegClass; 367 const TargetRegisterClass &HalfRC 368 = IsSALU ? AMDGPU::SReg_32RegClass : AMDGPU::VGPR_32RegClass; 369 370 MachineOperand Lo1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub0)); 371 MachineOperand Lo2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub0)); 372 MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1)); 373 MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1)); 374 375 Register DstLo = MRI->createVirtualRegister(&HalfRC); 376 Register DstHi = MRI->createVirtualRegister(&HalfRC); 377 378 if (IsSALU) { 379 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo) 380 .add(Lo1) 381 .add(Lo2); 382 BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADDC_U32), DstHi) 383 .add(Hi1) 384 .add(Hi2); 385 } else { 386 const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass(); 387 Register CarryReg = MRI->createVirtualRegister(CarryRC); 388 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_I32_e64), DstLo) 389 .addDef(CarryReg) 390 .add(Lo1) 391 .add(Lo2) 392 .addImm(0); 393 MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi) 394 .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead) 395 .add(Hi1) 396 .add(Hi2) 397 .addReg(CarryReg, RegState::Kill) 398 .addImm(0); 399 400 if (!constrainSelectedInstRegOperands(*Addc, TII, TRI, RBI)) 401 return false; 402 } 403 404 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) 405 .addReg(DstLo) 406 .addImm(AMDGPU::sub0) 407 .addReg(DstHi) 408 .addImm(AMDGPU::sub1); 409 410 411 if (!RBI.constrainGenericRegister(DstReg, RC, *MRI)) 412 return false; 413 414 I.eraseFromParent(); 415 return true; 416 } 417 418 bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE( 419 MachineInstr &I) const { 420 MachineBasicBlock *BB = I.getParent(); 421 MachineFunction *MF = BB->getParent(); 422 const DebugLoc &DL = I.getDebugLoc(); 423 Register Dst0Reg = I.getOperand(0).getReg(); 424 Register Dst1Reg = I.getOperand(1).getReg(); 425 const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || 426 I.getOpcode() == AMDGPU::G_UADDE; 427 const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || 428 I.getOpcode() == AMDGPU::G_USUBE; 429 430 if (isVCC(Dst1Reg, *MRI)) { 431 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 432 // carry out despite the _i32 name. These were renamed in VI to _U32. 433 // FIXME: We should probably rename the opcodes here. 434 unsigned NoCarryOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 435 unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 436 I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc)); 437 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 438 I.addOperand(*MF, MachineOperand::CreateImm(0)); 439 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 440 } 441 442 Register Src0Reg = I.getOperand(2).getReg(); 443 Register Src1Reg = I.getOperand(3).getReg(); 444 445 if (HasCarryIn) { 446 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) 447 .addReg(I.getOperand(4).getReg()); 448 } 449 450 unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; 451 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; 452 453 BuildMI(*BB, &I, DL, TII.get(HasCarryIn ? CarryOpc : NoCarryOpc), Dst0Reg) 454 .add(I.getOperand(2)) 455 .add(I.getOperand(3)); 456 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) 457 .addReg(AMDGPU::SCC); 458 459 if (!MRI->getRegClassOrNull(Dst1Reg)) 460 MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); 461 462 if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || 463 !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || 464 !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) 465 return false; 466 467 if (HasCarryIn && 468 !RBI.constrainGenericRegister(I.getOperand(4).getReg(), 469 AMDGPU::SReg_32RegClass, *MRI)) 470 return false; 471 472 I.eraseFromParent(); 473 return true; 474 } 475 476 // TODO: We should probably legalize these to only using 32-bit results. 477 bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const { 478 MachineBasicBlock *BB = I.getParent(); 479 Register DstReg = I.getOperand(0).getReg(); 480 Register SrcReg = I.getOperand(1).getReg(); 481 LLT DstTy = MRI->getType(DstReg); 482 LLT SrcTy = MRI->getType(SrcReg); 483 const unsigned SrcSize = SrcTy.getSizeInBits(); 484 const unsigned DstSize = DstTy.getSizeInBits(); 485 486 // TODO: Should handle any multiple of 32 offset. 487 unsigned Offset = I.getOperand(2).getImm(); 488 if (Offset % 32 != 0 || DstSize > 128) 489 return false; 490 491 const TargetRegisterClass *DstRC = 492 TRI.getConstrainedRegClassForOperand(I.getOperand(0), *MRI); 493 if (!DstRC || !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) 494 return false; 495 496 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); 497 const TargetRegisterClass *SrcRC = 498 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI); 499 if (!SrcRC) 500 return false; 501 unsigned SubReg = SIRegisterInfo::getSubRegFromChannel(Offset / 32, 502 DstSize / 32); 503 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubReg); 504 if (!SrcRC) 505 return false; 506 507 SrcReg = constrainOperandRegClass(*MF, TRI, *MRI, TII, RBI, I, 508 *SrcRC, I.getOperand(1)); 509 const DebugLoc &DL = I.getDebugLoc(); 510 BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY), DstReg) 511 .addReg(SrcReg, 0, SubReg); 512 513 I.eraseFromParent(); 514 return true; 515 } 516 517 bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const { 518 MachineBasicBlock *BB = MI.getParent(); 519 Register DstReg = MI.getOperand(0).getReg(); 520 LLT DstTy = MRI->getType(DstReg); 521 LLT SrcTy = MRI->getType(MI.getOperand(1).getReg()); 522 523 const unsigned SrcSize = SrcTy.getSizeInBits(); 524 if (SrcSize < 32) 525 return selectImpl(MI, *CoverageInfo); 526 527 const DebugLoc &DL = MI.getDebugLoc(); 528 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); 529 const unsigned DstSize = DstTy.getSizeInBits(); 530 const TargetRegisterClass *DstRC = 531 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI); 532 if (!DstRC) 533 return false; 534 535 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(DstRC, SrcSize / 8); 536 MachineInstrBuilder MIB = 537 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::REG_SEQUENCE), DstReg); 538 for (int I = 0, E = MI.getNumOperands() - 1; I != E; ++I) { 539 MachineOperand &Src = MI.getOperand(I + 1); 540 MIB.addReg(Src.getReg(), getUndefRegState(Src.isUndef())); 541 MIB.addImm(SubRegs[I]); 542 543 const TargetRegisterClass *SrcRC 544 = TRI.getConstrainedRegClassForOperand(Src, *MRI); 545 if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI)) 546 return false; 547 } 548 549 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) 550 return false; 551 552 MI.eraseFromParent(); 553 return true; 554 } 555 556 bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const { 557 MachineBasicBlock *BB = MI.getParent(); 558 const int NumDst = MI.getNumOperands() - 1; 559 560 MachineOperand &Src = MI.getOperand(NumDst); 561 562 Register SrcReg = Src.getReg(); 563 Register DstReg0 = MI.getOperand(0).getReg(); 564 LLT DstTy = MRI->getType(DstReg0); 565 LLT SrcTy = MRI->getType(SrcReg); 566 567 const unsigned DstSize = DstTy.getSizeInBits(); 568 const unsigned SrcSize = SrcTy.getSizeInBits(); 569 const DebugLoc &DL = MI.getDebugLoc(); 570 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); 571 572 const TargetRegisterClass *SrcRC = 573 TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI); 574 if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) 575 return false; 576 577 const unsigned SrcFlags = getUndefRegState(Src.isUndef()); 578 579 // Note we could have mixed SGPR and VGPR destination banks for an SGPR 580 // source, and this relies on the fact that the same subregister indices are 581 // used for both. 582 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8); 583 for (int I = 0, E = NumDst; I != E; ++I) { 584 MachineOperand &Dst = MI.getOperand(I); 585 BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg()) 586 .addReg(SrcReg, SrcFlags, SubRegs[I]); 587 588 const TargetRegisterClass *DstRC = 589 TRI.getConstrainedRegClassForOperand(Dst, *MRI); 590 if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI)) 591 return false; 592 } 593 594 MI.eraseFromParent(); 595 return true; 596 } 597 598 bool AMDGPUInstructionSelector::selectG_PTR_ADD(MachineInstr &I) const { 599 return selectG_ADD_SUB(I); 600 } 601 602 bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const { 603 const MachineOperand &MO = I.getOperand(0); 604 605 // FIXME: Interface for getConstrainedRegClassForOperand needs work. The 606 // regbank check here is to know why getConstrainedRegClassForOperand failed. 607 const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, *MRI); 608 if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) || 609 (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) { 610 I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF)); 611 return true; 612 } 613 614 return false; 615 } 616 617 bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const { 618 MachineBasicBlock *BB = I.getParent(); 619 620 Register DstReg = I.getOperand(0).getReg(); 621 Register Src0Reg = I.getOperand(1).getReg(); 622 Register Src1Reg = I.getOperand(2).getReg(); 623 LLT Src1Ty = MRI->getType(Src1Reg); 624 625 unsigned DstSize = MRI->getType(DstReg).getSizeInBits(); 626 unsigned InsSize = Src1Ty.getSizeInBits(); 627 628 int64_t Offset = I.getOperand(3).getImm(); 629 if (Offset % 32 != 0) 630 return false; 631 632 unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32, InsSize / 32); 633 if (SubReg == AMDGPU::NoSubRegister) 634 return false; 635 636 const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); 637 const TargetRegisterClass *DstRC = 638 TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI); 639 if (!DstRC) 640 return false; 641 642 const RegisterBank *Src0Bank = RBI.getRegBank(Src0Reg, *MRI, TRI); 643 const RegisterBank *Src1Bank = RBI.getRegBank(Src1Reg, *MRI, TRI); 644 const TargetRegisterClass *Src0RC = 645 TRI.getRegClassForSizeOnBank(DstSize, *Src0Bank, *MRI); 646 const TargetRegisterClass *Src1RC = 647 TRI.getRegClassForSizeOnBank(InsSize, *Src1Bank, *MRI); 648 649 // Deal with weird cases where the class only partially supports the subreg 650 // index. 651 Src0RC = TRI.getSubClassWithSubReg(Src0RC, SubReg); 652 if (!Src0RC) 653 return false; 654 655 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || 656 !RBI.constrainGenericRegister(Src0Reg, *Src0RC, *MRI) || 657 !RBI.constrainGenericRegister(Src1Reg, *Src1RC, *MRI)) 658 return false; 659 660 const DebugLoc &DL = I.getDebugLoc(); 661 BuildMI(*BB, &I, DL, TII.get(TargetOpcode::INSERT_SUBREG), DstReg) 662 .addReg(Src0Reg) 663 .addReg(Src1Reg) 664 .addImm(SubReg); 665 666 I.eraseFromParent(); 667 return true; 668 } 669 670 bool AMDGPUInstructionSelector::selectInterpP1F16(MachineInstr &MI) const { 671 if (STI.getLDSBankCount() != 16) 672 return selectImpl(MI, *CoverageInfo); 673 674 Register Dst = MI.getOperand(0).getReg(); 675 Register Src0 = MI.getOperand(2).getReg(); 676 Register M0Val = MI.getOperand(6).getReg(); 677 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI) || 678 !RBI.constrainGenericRegister(Dst, AMDGPU::VGPR_32RegClass, *MRI) || 679 !RBI.constrainGenericRegister(Src0, AMDGPU::VGPR_32RegClass, *MRI)) 680 return false; 681 682 // This requires 2 instructions. It is possible to write a pattern to support 683 // this, but the generated isel emitter doesn't correctly deal with multiple 684 // output instructions using the same physical register input. The copy to m0 685 // is incorrectly placed before the second instruction. 686 // 687 // TODO: Match source modifiers. 688 689 Register InterpMov = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 690 const DebugLoc &DL = MI.getDebugLoc(); 691 MachineBasicBlock *MBB = MI.getParent(); 692 693 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 694 .addReg(M0Val); 695 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_MOV_F32), InterpMov) 696 .addImm(2) 697 .addImm(MI.getOperand(4).getImm()) // $attr 698 .addImm(MI.getOperand(3).getImm()); // $attrchan 699 700 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::V_INTERP_P1LV_F16), Dst) 701 .addImm(0) // $src0_modifiers 702 .addReg(Src0) // $src0 703 .addImm(MI.getOperand(4).getImm()) // $attr 704 .addImm(MI.getOperand(3).getImm()) // $attrchan 705 .addImm(0) // $src2_modifiers 706 .addReg(InterpMov) // $src2 - 2 f16 values selected by high 707 .addImm(MI.getOperand(5).getImm()) // $high 708 .addImm(0) // $clamp 709 .addImm(0); // $omod 710 711 MI.eraseFromParent(); 712 return true; 713 } 714 715 bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const { 716 unsigned IntrinsicID = I.getIntrinsicID(); 717 switch (IntrinsicID) { 718 case Intrinsic::amdgcn_if_break: { 719 MachineBasicBlock *BB = I.getParent(); 720 721 // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick 722 // SelectionDAG uses for wave32 vs wave64. 723 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::SI_IF_BREAK)) 724 .add(I.getOperand(0)) 725 .add(I.getOperand(2)) 726 .add(I.getOperand(3)); 727 728 Register DstReg = I.getOperand(0).getReg(); 729 Register Src0Reg = I.getOperand(2).getReg(); 730 Register Src1Reg = I.getOperand(3).getReg(); 731 732 I.eraseFromParent(); 733 734 for (Register Reg : { DstReg, Src0Reg, Src1Reg }) 735 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); 736 737 return true; 738 } 739 case Intrinsic::amdgcn_interp_p1_f16: 740 return selectInterpP1F16(I); 741 case Intrinsic::amdgcn_wqm: 742 return constrainCopyLikeIntrin(I, AMDGPU::WQM); 743 case Intrinsic::amdgcn_softwqm: 744 return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM); 745 case Intrinsic::amdgcn_wwm: 746 return constrainCopyLikeIntrin(I, AMDGPU::WWM); 747 default: 748 return selectImpl(I, *CoverageInfo); 749 } 750 } 751 752 static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size) { 753 if (Size != 32 && Size != 64) 754 return -1; 755 switch (P) { 756 default: 757 llvm_unreachable("Unknown condition code!"); 758 case CmpInst::ICMP_NE: 759 return Size == 32 ? AMDGPU::V_CMP_NE_U32_e64 : AMDGPU::V_CMP_NE_U64_e64; 760 case CmpInst::ICMP_EQ: 761 return Size == 32 ? AMDGPU::V_CMP_EQ_U32_e64 : AMDGPU::V_CMP_EQ_U64_e64; 762 case CmpInst::ICMP_SGT: 763 return Size == 32 ? AMDGPU::V_CMP_GT_I32_e64 : AMDGPU::V_CMP_GT_I64_e64; 764 case CmpInst::ICMP_SGE: 765 return Size == 32 ? AMDGPU::V_CMP_GE_I32_e64 : AMDGPU::V_CMP_GE_I64_e64; 766 case CmpInst::ICMP_SLT: 767 return Size == 32 ? AMDGPU::V_CMP_LT_I32_e64 : AMDGPU::V_CMP_LT_I64_e64; 768 case CmpInst::ICMP_SLE: 769 return Size == 32 ? AMDGPU::V_CMP_LE_I32_e64 : AMDGPU::V_CMP_LE_I64_e64; 770 case CmpInst::ICMP_UGT: 771 return Size == 32 ? AMDGPU::V_CMP_GT_U32_e64 : AMDGPU::V_CMP_GT_U64_e64; 772 case CmpInst::ICMP_UGE: 773 return Size == 32 ? AMDGPU::V_CMP_GE_U32_e64 : AMDGPU::V_CMP_GE_U64_e64; 774 case CmpInst::ICMP_ULT: 775 return Size == 32 ? AMDGPU::V_CMP_LT_U32_e64 : AMDGPU::V_CMP_LT_U64_e64; 776 case CmpInst::ICMP_ULE: 777 return Size == 32 ? AMDGPU::V_CMP_LE_U32_e64 : AMDGPU::V_CMP_LE_U64_e64; 778 } 779 } 780 781 int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P, 782 unsigned Size) const { 783 if (Size == 64) { 784 if (!STI.hasScalarCompareEq64()) 785 return -1; 786 787 switch (P) { 788 case CmpInst::ICMP_NE: 789 return AMDGPU::S_CMP_LG_U64; 790 case CmpInst::ICMP_EQ: 791 return AMDGPU::S_CMP_EQ_U64; 792 default: 793 return -1; 794 } 795 } 796 797 if (Size != 32) 798 return -1; 799 800 switch (P) { 801 case CmpInst::ICMP_NE: 802 return AMDGPU::S_CMP_LG_U32; 803 case CmpInst::ICMP_EQ: 804 return AMDGPU::S_CMP_EQ_U32; 805 case CmpInst::ICMP_SGT: 806 return AMDGPU::S_CMP_GT_I32; 807 case CmpInst::ICMP_SGE: 808 return AMDGPU::S_CMP_GE_I32; 809 case CmpInst::ICMP_SLT: 810 return AMDGPU::S_CMP_LT_I32; 811 case CmpInst::ICMP_SLE: 812 return AMDGPU::S_CMP_LE_I32; 813 case CmpInst::ICMP_UGT: 814 return AMDGPU::S_CMP_GT_U32; 815 case CmpInst::ICMP_UGE: 816 return AMDGPU::S_CMP_GE_U32; 817 case CmpInst::ICMP_ULT: 818 return AMDGPU::S_CMP_LT_U32; 819 case CmpInst::ICMP_ULE: 820 return AMDGPU::S_CMP_LE_U32; 821 default: 822 llvm_unreachable("Unknown condition code!"); 823 } 824 } 825 826 bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const { 827 MachineBasicBlock *BB = I.getParent(); 828 const DebugLoc &DL = I.getDebugLoc(); 829 830 Register SrcReg = I.getOperand(2).getReg(); 831 unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI); 832 833 auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate(); 834 835 Register CCReg = I.getOperand(0).getReg(); 836 if (!isVCC(CCReg, *MRI)) { 837 int Opcode = getS_CMPOpcode(Pred, Size); 838 if (Opcode == -1) 839 return false; 840 MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode)) 841 .add(I.getOperand(2)) 842 .add(I.getOperand(3)); 843 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg) 844 .addReg(AMDGPU::SCC); 845 bool Ret = 846 constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI) && 847 RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI); 848 I.eraseFromParent(); 849 return Ret; 850 } 851 852 int Opcode = getV_CMPOpcode(Pred, Size); 853 if (Opcode == -1) 854 return false; 855 856 MachineInstr *ICmp = BuildMI(*BB, &I, DL, TII.get(Opcode), 857 I.getOperand(0).getReg()) 858 .add(I.getOperand(2)) 859 .add(I.getOperand(3)); 860 RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(), 861 *TRI.getBoolRC(), *MRI); 862 bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI); 863 I.eraseFromParent(); 864 return Ret; 865 } 866 867 bool AMDGPUInstructionSelector::selectEndCfIntrinsic(MachineInstr &MI) const { 868 // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick 869 // SelectionDAG uses for wave32 vs wave64. 870 MachineBasicBlock *BB = MI.getParent(); 871 BuildMI(*BB, &MI, MI.getDebugLoc(), TII.get(AMDGPU::SI_END_CF)) 872 .add(MI.getOperand(1)); 873 874 Register Reg = MI.getOperand(1).getReg(); 875 MI.eraseFromParent(); 876 877 if (!MRI->getRegClassOrNull(Reg)) 878 MRI->setRegClass(Reg, TRI.getWaveMaskRegClass()); 879 return true; 880 } 881 882 static unsigned getDSShaderTypeValue(const MachineFunction &MF) { 883 switch (MF.getFunction().getCallingConv()) { 884 case CallingConv::AMDGPU_PS: 885 return 1; 886 case CallingConv::AMDGPU_VS: 887 return 2; 888 case CallingConv::AMDGPU_GS: 889 return 3; 890 case CallingConv::AMDGPU_HS: 891 case CallingConv::AMDGPU_LS: 892 case CallingConv::AMDGPU_ES: 893 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 894 case CallingConv::AMDGPU_CS: 895 case CallingConv::AMDGPU_KERNEL: 896 case CallingConv::C: 897 case CallingConv::Fast: 898 default: 899 // Assume other calling conventions are various compute callable functions 900 return 0; 901 } 902 } 903 904 bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic( 905 MachineInstr &MI, Intrinsic::ID IntrID) const { 906 MachineBasicBlock *MBB = MI.getParent(); 907 MachineFunction *MF = MBB->getParent(); 908 const DebugLoc &DL = MI.getDebugLoc(); 909 910 unsigned IndexOperand = MI.getOperand(7).getImm(); 911 bool WaveRelease = MI.getOperand(8).getImm() != 0; 912 bool WaveDone = MI.getOperand(9).getImm() != 0; 913 914 if (WaveDone && !WaveRelease) 915 report_fatal_error("ds_ordered_count: wave_done requires wave_release"); 916 917 unsigned OrderedCountIndex = IndexOperand & 0x3f; 918 IndexOperand &= ~0x3f; 919 unsigned CountDw = 0; 920 921 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) { 922 CountDw = (IndexOperand >> 24) & 0xf; 923 IndexOperand &= ~(0xf << 24); 924 925 if (CountDw < 1 || CountDw > 4) { 926 report_fatal_error( 927 "ds_ordered_count: dword count must be between 1 and 4"); 928 } 929 } 930 931 if (IndexOperand) 932 report_fatal_error("ds_ordered_count: bad index operand"); 933 934 unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1; 935 unsigned ShaderType = getDSShaderTypeValue(*MF); 936 937 unsigned Offset0 = OrderedCountIndex << 2; 938 unsigned Offset1 = WaveRelease | (WaveDone << 1) | (ShaderType << 2) | 939 (Instruction << 4); 940 941 if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) 942 Offset1 |= (CountDw - 1) << 6; 943 944 unsigned Offset = Offset0 | (Offset1 << 8); 945 946 Register M0Val = MI.getOperand(2).getReg(); 947 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 948 .addReg(M0Val); 949 950 Register DstReg = MI.getOperand(0).getReg(); 951 Register ValReg = MI.getOperand(3).getReg(); 952 MachineInstrBuilder DS = 953 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) 954 .addReg(ValReg) 955 .addImm(Offset) 956 .cloneMemRefs(MI); 957 958 if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) 959 return false; 960 961 bool Ret = constrainSelectedInstRegOperands(*DS, TII, TRI, RBI); 962 MI.eraseFromParent(); 963 return Ret; 964 } 965 966 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 967 switch (IntrID) { 968 case Intrinsic::amdgcn_ds_gws_init: 969 return AMDGPU::DS_GWS_INIT; 970 case Intrinsic::amdgcn_ds_gws_barrier: 971 return AMDGPU::DS_GWS_BARRIER; 972 case Intrinsic::amdgcn_ds_gws_sema_v: 973 return AMDGPU::DS_GWS_SEMA_V; 974 case Intrinsic::amdgcn_ds_gws_sema_br: 975 return AMDGPU::DS_GWS_SEMA_BR; 976 case Intrinsic::amdgcn_ds_gws_sema_p: 977 return AMDGPU::DS_GWS_SEMA_P; 978 case Intrinsic::amdgcn_ds_gws_sema_release_all: 979 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 980 default: 981 llvm_unreachable("not a gws intrinsic"); 982 } 983 } 984 985 bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI, 986 Intrinsic::ID IID) const { 987 if (IID == Intrinsic::amdgcn_ds_gws_sema_release_all && 988 !STI.hasGWSSemaReleaseAll()) 989 return false; 990 991 // intrinsic ID, vsrc, offset 992 const bool HasVSrc = MI.getNumOperands() == 3; 993 assert(HasVSrc || MI.getNumOperands() == 2); 994 995 Register BaseOffset = MI.getOperand(HasVSrc ? 2 : 1).getReg(); 996 const RegisterBank *OffsetRB = RBI.getRegBank(BaseOffset, *MRI, TRI); 997 if (OffsetRB->getID() != AMDGPU::SGPRRegBankID) 998 return false; 999 1000 MachineInstr *OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); 1001 assert(OffsetDef); 1002 1003 unsigned ImmOffset; 1004 1005 MachineBasicBlock *MBB = MI.getParent(); 1006 const DebugLoc &DL = MI.getDebugLoc(); 1007 1008 MachineInstr *Readfirstlane = nullptr; 1009 1010 // If we legalized the VGPR input, strip out the readfirstlane to analyze the 1011 // incoming offset, in case there's an add of a constant. We'll have to put it 1012 // back later. 1013 if (OffsetDef->getOpcode() == AMDGPU::V_READFIRSTLANE_B32) { 1014 Readfirstlane = OffsetDef; 1015 BaseOffset = OffsetDef->getOperand(1).getReg(); 1016 OffsetDef = getDefIgnoringCopies(BaseOffset, *MRI); 1017 } 1018 1019 if (OffsetDef->getOpcode() == AMDGPU::G_CONSTANT) { 1020 // If we have a constant offset, try to use the 0 in m0 as the base. 1021 // TODO: Look into changing the default m0 initialization value. If the 1022 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 1023 // the immediate offset. 1024 1025 ImmOffset = OffsetDef->getOperand(1).getCImm()->getZExtValue(); 1026 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) 1027 .addImm(0); 1028 } else { 1029 std::tie(BaseOffset, ImmOffset, OffsetDef) 1030 = AMDGPU::getBaseWithConstantOffset(*MRI, BaseOffset); 1031 1032 if (Readfirstlane) { 1033 // We have the constant offset now, so put the readfirstlane back on the 1034 // variable component. 1035 if (!RBI.constrainGenericRegister(BaseOffset, AMDGPU::VGPR_32RegClass, *MRI)) 1036 return false; 1037 1038 Readfirstlane->getOperand(1).setReg(BaseOffset); 1039 BaseOffset = Readfirstlane->getOperand(0).getReg(); 1040 } else { 1041 if (!RBI.constrainGenericRegister(BaseOffset, 1042 AMDGPU::SReg_32RegClass, *MRI)) 1043 return false; 1044 } 1045 1046 Register M0Base = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1047 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_LSHL_B32), M0Base) 1048 .addReg(BaseOffset) 1049 .addImm(16); 1050 1051 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 1052 .addReg(M0Base); 1053 } 1054 1055 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 1056 // offset field) % 64. Some versions of the programming guide omit the m0 1057 // part, or claim it's from offset 0. 1058 auto MIB = BuildMI(*MBB, &MI, DL, TII.get(gwsIntrinToOpcode(IID))); 1059 1060 if (HasVSrc) { 1061 Register VSrc = MI.getOperand(1).getReg(); 1062 MIB.addReg(VSrc); 1063 if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI)) 1064 return false; 1065 } 1066 1067 MIB.addImm(ImmOffset) 1068 .addImm(-1) // $gds 1069 .cloneMemRefs(MI); 1070 1071 MI.eraseFromParent(); 1072 return true; 1073 } 1074 1075 bool AMDGPUInstructionSelector::selectDSAppendConsume(MachineInstr &MI, 1076 bool IsAppend) const { 1077 Register PtrBase = MI.getOperand(2).getReg(); 1078 LLT PtrTy = MRI->getType(PtrBase); 1079 bool IsGDS = PtrTy.getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 1080 1081 unsigned Offset; 1082 std::tie(PtrBase, Offset) = selectDS1Addr1OffsetImpl(MI.getOperand(2)); 1083 1084 // TODO: Should this try to look through readfirstlane like GWS? 1085 if (!isDSOffsetLegal(PtrBase, Offset, 16)) { 1086 PtrBase = MI.getOperand(2).getReg(); 1087 Offset = 0; 1088 } 1089 1090 MachineBasicBlock *MBB = MI.getParent(); 1091 const DebugLoc &DL = MI.getDebugLoc(); 1092 const unsigned Opc = IsAppend ? AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 1093 1094 BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 1095 .addReg(PtrBase); 1096 BuildMI(*MBB, &MI, DL, TII.get(Opc), MI.getOperand(0).getReg()) 1097 .addImm(Offset) 1098 .addImm(IsGDS ? -1 : 0) 1099 .cloneMemRefs(MI); 1100 1101 MI.eraseFromParent(); 1102 return true; 1103 } 1104 1105 bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS( 1106 MachineInstr &I) const { 1107 unsigned IntrinsicID = I.getIntrinsicID(); 1108 switch (IntrinsicID) { 1109 case Intrinsic::amdgcn_end_cf: 1110 return selectEndCfIntrinsic(I); 1111 case Intrinsic::amdgcn_ds_ordered_add: 1112 case Intrinsic::amdgcn_ds_ordered_swap: 1113 return selectDSOrderedIntrinsic(I, IntrinsicID); 1114 case Intrinsic::amdgcn_ds_gws_init: 1115 case Intrinsic::amdgcn_ds_gws_barrier: 1116 case Intrinsic::amdgcn_ds_gws_sema_v: 1117 case Intrinsic::amdgcn_ds_gws_sema_br: 1118 case Intrinsic::amdgcn_ds_gws_sema_p: 1119 case Intrinsic::amdgcn_ds_gws_sema_release_all: 1120 return selectDSGWSIntrinsic(I, IntrinsicID); 1121 case Intrinsic::amdgcn_ds_append: 1122 return selectDSAppendConsume(I, true); 1123 case Intrinsic::amdgcn_ds_consume: 1124 return selectDSAppendConsume(I, false); 1125 default: 1126 return selectImpl(I, *CoverageInfo); 1127 } 1128 } 1129 1130 bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const { 1131 if (selectImpl(I, *CoverageInfo)) 1132 return true; 1133 1134 MachineBasicBlock *BB = I.getParent(); 1135 const DebugLoc &DL = I.getDebugLoc(); 1136 1137 Register DstReg = I.getOperand(0).getReg(); 1138 unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI); 1139 assert(Size <= 32 || Size == 64); 1140 const MachineOperand &CCOp = I.getOperand(1); 1141 Register CCReg = CCOp.getReg(); 1142 if (!isVCC(CCReg, *MRI)) { 1143 unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : 1144 AMDGPU::S_CSELECT_B32; 1145 MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) 1146 .addReg(CCReg); 1147 1148 // The generic constrainSelectedInstRegOperands doesn't work for the scc register 1149 // bank, because it does not cover the register class that we used to represent 1150 // for it. So we need to manually set the register class here. 1151 if (!MRI->getRegClassOrNull(CCReg)) 1152 MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI)); 1153 MachineInstr *Select = BuildMI(*BB, &I, DL, TII.get(SelectOpcode), DstReg) 1154 .add(I.getOperand(2)) 1155 .add(I.getOperand(3)); 1156 1157 bool Ret = constrainSelectedInstRegOperands(*Select, TII, TRI, RBI) | 1158 constrainSelectedInstRegOperands(*CopySCC, TII, TRI, RBI); 1159 I.eraseFromParent(); 1160 return Ret; 1161 } 1162 1163 // Wide VGPR select should have been split in RegBankSelect. 1164 if (Size > 32) 1165 return false; 1166 1167 MachineInstr *Select = 1168 BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1169 .addImm(0) 1170 .add(I.getOperand(3)) 1171 .addImm(0) 1172 .add(I.getOperand(2)) 1173 .add(I.getOperand(1)); 1174 1175 bool Ret = constrainSelectedInstRegOperands(*Select, TII, TRI, RBI); 1176 I.eraseFromParent(); 1177 return Ret; 1178 } 1179 1180 bool AMDGPUInstructionSelector::selectG_STORE(MachineInstr &I) const { 1181 initM0(I); 1182 return selectImpl(I, *CoverageInfo); 1183 } 1184 1185 static int sizeToSubRegIndex(unsigned Size) { 1186 switch (Size) { 1187 case 32: 1188 return AMDGPU::sub0; 1189 case 64: 1190 return AMDGPU::sub0_sub1; 1191 case 96: 1192 return AMDGPU::sub0_sub1_sub2; 1193 case 128: 1194 return AMDGPU::sub0_sub1_sub2_sub3; 1195 case 256: 1196 return AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; 1197 default: 1198 if (Size < 32) 1199 return AMDGPU::sub0; 1200 if (Size > 256) 1201 return -1; 1202 return sizeToSubRegIndex(PowerOf2Ceil(Size)); 1203 } 1204 } 1205 1206 bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const { 1207 Register DstReg = I.getOperand(0).getReg(); 1208 Register SrcReg = I.getOperand(1).getReg(); 1209 const LLT DstTy = MRI->getType(DstReg); 1210 const LLT SrcTy = MRI->getType(SrcReg); 1211 if (!DstTy.isScalar()) 1212 return false; 1213 1214 const LLT S1 = LLT::scalar(1); 1215 1216 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); 1217 const RegisterBank *DstRB; 1218 if (DstTy == S1) { 1219 // This is a special case. We don't treat s1 for legalization artifacts as 1220 // vcc booleans. 1221 DstRB = SrcRB; 1222 } else { 1223 DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 1224 if (SrcRB != DstRB) 1225 return false; 1226 } 1227 1228 unsigned DstSize = DstTy.getSizeInBits(); 1229 unsigned SrcSize = SrcTy.getSizeInBits(); 1230 1231 const TargetRegisterClass *SrcRC 1232 = TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB, *MRI); 1233 const TargetRegisterClass *DstRC 1234 = TRI.getRegClassForSizeOnBank(DstSize, *DstRB, *MRI); 1235 1236 if (SrcSize > 32) { 1237 int SubRegIdx = sizeToSubRegIndex(DstSize); 1238 if (SubRegIdx == -1) 1239 return false; 1240 1241 // Deal with weird cases where the class only partially supports the subreg 1242 // index. 1243 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegIdx); 1244 if (!SrcRC) 1245 return false; 1246 1247 I.getOperand(1).setSubReg(SubRegIdx); 1248 } 1249 1250 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || 1251 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) { 1252 LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); 1253 return false; 1254 } 1255 1256 I.setDesc(TII.get(TargetOpcode::COPY)); 1257 return true; 1258 } 1259 1260 /// \returns true if a bitmask for \p Size bits will be an inline immediate. 1261 static bool shouldUseAndMask(unsigned Size, unsigned &Mask) { 1262 Mask = maskTrailingOnes<unsigned>(Size); 1263 int SignedMask = static_cast<int>(Mask); 1264 return SignedMask >= -16 && SignedMask <= 64; 1265 } 1266 1267 // Like RegisterBankInfo::getRegBank, but don't assume vcc for s1. 1268 const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank( 1269 Register Reg, const MachineRegisterInfo &MRI, 1270 const TargetRegisterInfo &TRI) const { 1271 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); 1272 if (auto *RB = RegClassOrBank.dyn_cast<const RegisterBank *>()) 1273 return RB; 1274 1275 // Ignore the type, since we don't use vcc in artifacts. 1276 if (auto *RC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>()) 1277 return &RBI.getRegBankFromRegClass(*RC, LLT()); 1278 return nullptr; 1279 } 1280 1281 bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const { 1282 bool InReg = I.getOpcode() == AMDGPU::G_SEXT_INREG; 1283 bool Signed = I.getOpcode() == AMDGPU::G_SEXT || InReg; 1284 const DebugLoc &DL = I.getDebugLoc(); 1285 MachineBasicBlock &MBB = *I.getParent(); 1286 const Register DstReg = I.getOperand(0).getReg(); 1287 const Register SrcReg = I.getOperand(1).getReg(); 1288 1289 const LLT DstTy = MRI->getType(DstReg); 1290 const LLT SrcTy = MRI->getType(SrcReg); 1291 const unsigned SrcSize = I.getOpcode() == AMDGPU::G_SEXT_INREG ? 1292 I.getOperand(2).getImm() : SrcTy.getSizeInBits(); 1293 const unsigned DstSize = DstTy.getSizeInBits(); 1294 if (!DstTy.isScalar()) 1295 return false; 1296 1297 if (I.getOpcode() == AMDGPU::G_ANYEXT) 1298 return selectCOPY(I); 1299 1300 // Artifact casts should never use vcc. 1301 const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI); 1302 1303 if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { 1304 // 64-bit should have been split up in RegBankSelect 1305 1306 // Try to use an and with a mask if it will save code size. 1307 unsigned Mask; 1308 if (!Signed && shouldUseAndMask(SrcSize, Mask)) { 1309 MachineInstr *ExtI = 1310 BuildMI(MBB, I, DL, TII.get(AMDGPU::V_AND_B32_e32), DstReg) 1311 .addImm(Mask) 1312 .addReg(SrcReg); 1313 I.eraseFromParent(); 1314 return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); 1315 } 1316 1317 const unsigned BFE = Signed ? AMDGPU::V_BFE_I32 : AMDGPU::V_BFE_U32; 1318 MachineInstr *ExtI = 1319 BuildMI(MBB, I, DL, TII.get(BFE), DstReg) 1320 .addReg(SrcReg) 1321 .addImm(0) // Offset 1322 .addImm(SrcSize); // Width 1323 I.eraseFromParent(); 1324 return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); 1325 } 1326 1327 if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) { 1328 const TargetRegisterClass &SrcRC = InReg && DstSize > 32 ? 1329 AMDGPU::SReg_64RegClass : AMDGPU::SReg_32RegClass; 1330 if (!RBI.constrainGenericRegister(SrcReg, SrcRC, *MRI)) 1331 return false; 1332 1333 if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) { 1334 const unsigned SextOpc = SrcSize == 8 ? 1335 AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16; 1336 BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg) 1337 .addReg(SrcReg); 1338 I.eraseFromParent(); 1339 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); 1340 } 1341 1342 const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64; 1343 const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1344 1345 // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width. 1346 if (DstSize > 32 && (SrcSize <= 32 || InReg)) { 1347 // We need a 64-bit register source, but the high bits don't matter. 1348 Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass); 1349 Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1350 unsigned SubReg = InReg ? AMDGPU::sub0 : 0; 1351 1352 BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg); 1353 BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg) 1354 .addReg(SrcReg, 0, SubReg) 1355 .addImm(AMDGPU::sub0) 1356 .addReg(UndefReg) 1357 .addImm(AMDGPU::sub1); 1358 1359 BuildMI(MBB, I, DL, TII.get(BFE64), DstReg) 1360 .addReg(ExtReg) 1361 .addImm(SrcSize << 16); 1362 1363 I.eraseFromParent(); 1364 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI); 1365 } 1366 1367 unsigned Mask; 1368 if (!Signed && shouldUseAndMask(SrcSize, Mask)) { 1369 BuildMI(MBB, I, DL, TII.get(AMDGPU::S_AND_B32), DstReg) 1370 .addReg(SrcReg) 1371 .addImm(Mask); 1372 } else { 1373 BuildMI(MBB, I, DL, TII.get(BFE32), DstReg) 1374 .addReg(SrcReg) 1375 .addImm(SrcSize << 16); 1376 } 1377 1378 I.eraseFromParent(); 1379 return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI); 1380 } 1381 1382 return false; 1383 } 1384 1385 bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const { 1386 MachineBasicBlock *BB = I.getParent(); 1387 MachineOperand &ImmOp = I.getOperand(1); 1388 1389 // The AMDGPU backend only supports Imm operands and not CImm or FPImm. 1390 if (ImmOp.isFPImm()) { 1391 const APInt &Imm = ImmOp.getFPImm()->getValueAPF().bitcastToAPInt(); 1392 ImmOp.ChangeToImmediate(Imm.getZExtValue()); 1393 } else if (ImmOp.isCImm()) { 1394 ImmOp.ChangeToImmediate(ImmOp.getCImm()->getZExtValue()); 1395 } 1396 1397 Register DstReg = I.getOperand(0).getReg(); 1398 unsigned Size; 1399 bool IsSgpr; 1400 const RegisterBank *RB = MRI->getRegBankOrNull(I.getOperand(0).getReg()); 1401 if (RB) { 1402 IsSgpr = RB->getID() == AMDGPU::SGPRRegBankID; 1403 Size = MRI->getType(DstReg).getSizeInBits(); 1404 } else { 1405 const TargetRegisterClass *RC = TRI.getRegClassForReg(*MRI, DstReg); 1406 IsSgpr = TRI.isSGPRClass(RC); 1407 Size = TRI.getRegSizeInBits(*RC); 1408 } 1409 1410 if (Size != 32 && Size != 64) 1411 return false; 1412 1413 unsigned Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1414 if (Size == 32) { 1415 I.setDesc(TII.get(Opcode)); 1416 I.addImplicitDefUseOperands(*MF); 1417 return constrainSelectedInstRegOperands(I, TII, TRI, RBI); 1418 } 1419 1420 const DebugLoc &DL = I.getDebugLoc(); 1421 1422 APInt Imm(Size, I.getOperand(1).getImm()); 1423 1424 MachineInstr *ResInst; 1425 if (IsSgpr && TII.isInlineConstant(Imm)) { 1426 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg) 1427 .addImm(I.getOperand(1).getImm()); 1428 } else { 1429 const TargetRegisterClass *RC = IsSgpr ? 1430 &AMDGPU::SReg_32RegClass : &AMDGPU::VGPR_32RegClass; 1431 Register LoReg = MRI->createVirtualRegister(RC); 1432 Register HiReg = MRI->createVirtualRegister(RC); 1433 1434 BuildMI(*BB, &I, DL, TII.get(Opcode), LoReg) 1435 .addImm(Imm.trunc(32).getZExtValue()); 1436 1437 BuildMI(*BB, &I, DL, TII.get(Opcode), HiReg) 1438 .addImm(Imm.ashr(32).getZExtValue()); 1439 1440 ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) 1441 .addReg(LoReg) 1442 .addImm(AMDGPU::sub0) 1443 .addReg(HiReg) 1444 .addImm(AMDGPU::sub1); 1445 } 1446 1447 // We can't call constrainSelectedInstRegOperands here, because it doesn't 1448 // work for target independent opcodes 1449 I.eraseFromParent(); 1450 const TargetRegisterClass *DstRC = 1451 TRI.getConstrainedRegClassForOperand(ResInst->getOperand(0), *MRI); 1452 if (!DstRC) 1453 return true; 1454 return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI); 1455 } 1456 1457 bool AMDGPUInstructionSelector::selectG_FNEG(MachineInstr &MI) const { 1458 // Only manually handle the f64 SGPR case. 1459 // 1460 // FIXME: This is a workaround for 2.5 different tablegen problems. Because 1461 // the bit ops theoretically have a second result due to the implicit def of 1462 // SCC, the GlobalISelEmitter is overly conservative and rejects it. Fixing 1463 // that is easy by disabling the check. The result works, but uses a 1464 // nonsensical sreg32orlds_and_sreg_1 regclass. 1465 // 1466 // The DAG emitter is more problematic, and incorrectly adds both S_XOR_B32 to 1467 // the variadic REG_SEQUENCE operands. 1468 1469 Register Dst = MI.getOperand(0).getReg(); 1470 const RegisterBank *DstRB = RBI.getRegBank(Dst, *MRI, TRI); 1471 if (DstRB->getID() != AMDGPU::SGPRRegBankID || 1472 MRI->getType(Dst) != LLT::scalar(64)) 1473 return false; 1474 1475 Register Src = MI.getOperand(1).getReg(); 1476 MachineInstr *Fabs = getOpcodeDef(TargetOpcode::G_FABS, Src, *MRI); 1477 if (Fabs) 1478 Src = Fabs->getOperand(1).getReg(); 1479 1480 if (!RBI.constrainGenericRegister(Src, AMDGPU::SReg_64RegClass, *MRI) || 1481 !RBI.constrainGenericRegister(Dst, AMDGPU::SReg_64RegClass, *MRI)) 1482 return false; 1483 1484 MachineBasicBlock *BB = MI.getParent(); 1485 const DebugLoc &DL = MI.getDebugLoc(); 1486 Register LoReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1487 Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1488 Register ConstReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1489 Register OpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 1490 1491 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), LoReg) 1492 .addReg(Src, 0, AMDGPU::sub0); 1493 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), HiReg) 1494 .addReg(Src, 0, AMDGPU::sub1); 1495 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_MOV_B32), ConstReg) 1496 .addImm(0x80000000); 1497 1498 // Set or toggle sign bit. 1499 unsigned Opc = Fabs ? AMDGPU::S_OR_B32 : AMDGPU::S_XOR_B32; 1500 BuildMI(*BB, &MI, DL, TII.get(Opc), OpReg) 1501 .addReg(HiReg) 1502 .addReg(ConstReg); 1503 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::REG_SEQUENCE), Dst) 1504 .addReg(LoReg) 1505 .addImm(AMDGPU::sub0) 1506 .addReg(OpReg) 1507 .addImm(AMDGPU::sub1); 1508 MI.eraseFromParent(); 1509 return true; 1510 } 1511 1512 static bool isConstant(const MachineInstr &MI) { 1513 return MI.getOpcode() == TargetOpcode::G_CONSTANT; 1514 } 1515 1516 void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load, 1517 const MachineRegisterInfo &MRI, SmallVectorImpl<GEPInfo> &AddrInfo) const { 1518 1519 const MachineInstr *PtrMI = MRI.getUniqueVRegDef(Load.getOperand(1).getReg()); 1520 1521 assert(PtrMI); 1522 1523 if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD) 1524 return; 1525 1526 GEPInfo GEPInfo(*PtrMI); 1527 1528 for (unsigned i = 1; i != 3; ++i) { 1529 const MachineOperand &GEPOp = PtrMI->getOperand(i); 1530 const MachineInstr *OpDef = MRI.getUniqueVRegDef(GEPOp.getReg()); 1531 assert(OpDef); 1532 if (i == 2 && isConstant(*OpDef)) { 1533 // TODO: Could handle constant base + variable offset, but a combine 1534 // probably should have commuted it. 1535 assert(GEPInfo.Imm == 0); 1536 GEPInfo.Imm = OpDef->getOperand(1).getCImm()->getSExtValue(); 1537 continue; 1538 } 1539 const RegisterBank *OpBank = RBI.getRegBank(GEPOp.getReg(), MRI, TRI); 1540 if (OpBank->getID() == AMDGPU::SGPRRegBankID) 1541 GEPInfo.SgprParts.push_back(GEPOp.getReg()); 1542 else 1543 GEPInfo.VgprParts.push_back(GEPOp.getReg()); 1544 } 1545 1546 AddrInfo.push_back(GEPInfo); 1547 getAddrModeInfo(*PtrMI, MRI, AddrInfo); 1548 } 1549 1550 bool AMDGPUInstructionSelector::isInstrUniform(const MachineInstr &MI) const { 1551 if (!MI.hasOneMemOperand()) 1552 return false; 1553 1554 const MachineMemOperand *MMO = *MI.memoperands_begin(); 1555 const Value *Ptr = MMO->getValue(); 1556 1557 // UndefValue means this is a load of a kernel input. These are uniform. 1558 // Sometimes LDS instructions have constant pointers. 1559 // If Ptr is null, then that means this mem operand contains a 1560 // PseudoSourceValue like GOT. 1561 if (!Ptr || isa<UndefValue>(Ptr) || isa<Argument>(Ptr) || 1562 isa<Constant>(Ptr) || isa<GlobalValue>(Ptr)) 1563 return true; 1564 1565 if (MMO->getAddrSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) 1566 return true; 1567 1568 const Instruction *I = dyn_cast<Instruction>(Ptr); 1569 return I && I->getMetadata("amdgpu.uniform"); 1570 } 1571 1572 bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const { 1573 for (const GEPInfo &GEPInfo : AddrInfo) { 1574 if (!GEPInfo.VgprParts.empty()) 1575 return true; 1576 } 1577 return false; 1578 } 1579 1580 void AMDGPUInstructionSelector::initM0(MachineInstr &I) const { 1581 MachineBasicBlock *BB = I.getParent(); 1582 1583 const LLT PtrTy = MRI->getType(I.getOperand(1).getReg()); 1584 unsigned AS = PtrTy.getAddressSpace(); 1585 if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) && 1586 STI.ldsRequiresM0Init()) { 1587 // If DS instructions require M0 initializtion, insert it before selecting. 1588 BuildMI(*BB, &I, I.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), AMDGPU::M0) 1589 .addImm(-1); 1590 } 1591 } 1592 1593 bool AMDGPUInstructionSelector::selectG_LOAD_ATOMICRMW(MachineInstr &I) const { 1594 initM0(I); 1595 return selectImpl(I, *CoverageInfo); 1596 } 1597 1598 bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const { 1599 MachineBasicBlock *BB = I.getParent(); 1600 MachineOperand &CondOp = I.getOperand(0); 1601 Register CondReg = CondOp.getReg(); 1602 const DebugLoc &DL = I.getDebugLoc(); 1603 1604 unsigned BrOpcode; 1605 Register CondPhysReg; 1606 const TargetRegisterClass *ConstrainRC; 1607 1608 // In SelectionDAG, we inspect the IR block for uniformity metadata to decide 1609 // whether the branch is uniform when selecting the instruction. In 1610 // GlobalISel, we should push that decision into RegBankSelect. Assume for now 1611 // RegBankSelect knows what it's doing if the branch condition is scc, even 1612 // though it currently does not. 1613 if (!isVCC(CondReg, *MRI)) { 1614 if (MRI->getType(CondReg) != LLT::scalar(32)) 1615 return false; 1616 1617 CondPhysReg = AMDGPU::SCC; 1618 BrOpcode = AMDGPU::S_CBRANCH_SCC1; 1619 // FIXME: Hack for isSCC tests 1620 ConstrainRC = &AMDGPU::SGPR_32RegClass; 1621 } else { 1622 // FIXME: Do we have to insert an and with exec here, like in SelectionDAG? 1623 // We sort of know that a VCC producer based on the register bank, that ands 1624 // inactive lanes with 0. What if there was a logical operation with vcc 1625 // producers in different blocks/with different exec masks? 1626 // FIXME: Should scc->vcc copies and with exec? 1627 CondPhysReg = TRI.getVCC(); 1628 BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; 1629 ConstrainRC = TRI.getBoolRC(); 1630 } 1631 1632 if (!MRI->getRegClassOrNull(CondReg)) 1633 MRI->setRegClass(CondReg, ConstrainRC); 1634 1635 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg) 1636 .addReg(CondReg); 1637 BuildMI(*BB, &I, DL, TII.get(BrOpcode)) 1638 .addMBB(I.getOperand(1).getMBB()); 1639 1640 I.eraseFromParent(); 1641 return true; 1642 } 1643 1644 bool AMDGPUInstructionSelector::selectG_FRAME_INDEX_GLOBAL_VALUE( 1645 MachineInstr &I) const { 1646 Register DstReg = I.getOperand(0).getReg(); 1647 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 1648 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; 1649 I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32)); 1650 if (IsVGPR) 1651 I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 1652 1653 return RBI.constrainGenericRegister( 1654 DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI); 1655 } 1656 1657 bool AMDGPUInstructionSelector::selectG_PTR_MASK(MachineInstr &I) const { 1658 uint64_t Align = I.getOperand(2).getImm(); 1659 const uint64_t Mask = ~((UINT64_C(1) << Align) - 1); 1660 1661 MachineBasicBlock *BB = I.getParent(); 1662 1663 Register DstReg = I.getOperand(0).getReg(); 1664 Register SrcReg = I.getOperand(1).getReg(); 1665 1666 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 1667 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); 1668 const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID; 1669 unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32; 1670 unsigned MovOpc = IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; 1671 const TargetRegisterClass &RegRC 1672 = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass; 1673 1674 LLT Ty = MRI->getType(DstReg); 1675 1676 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, *DstRB, 1677 *MRI); 1678 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, *SrcRB, 1679 *MRI); 1680 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || 1681 !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI)) 1682 return false; 1683 1684 const DebugLoc &DL = I.getDebugLoc(); 1685 Register ImmReg = MRI->createVirtualRegister(&RegRC); 1686 BuildMI(*BB, &I, DL, TII.get(MovOpc), ImmReg) 1687 .addImm(Mask); 1688 1689 if (Ty.getSizeInBits() == 32) { 1690 BuildMI(*BB, &I, DL, TII.get(NewOpc), DstReg) 1691 .addReg(SrcReg) 1692 .addReg(ImmReg); 1693 I.eraseFromParent(); 1694 return true; 1695 } 1696 1697 Register HiReg = MRI->createVirtualRegister(&RegRC); 1698 Register LoReg = MRI->createVirtualRegister(&RegRC); 1699 Register MaskLo = MRI->createVirtualRegister(&RegRC); 1700 1701 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg) 1702 .addReg(SrcReg, 0, AMDGPU::sub0); 1703 BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), HiReg) 1704 .addReg(SrcReg, 0, AMDGPU::sub1); 1705 1706 BuildMI(*BB, &I, DL, TII.get(NewOpc), MaskLo) 1707 .addReg(LoReg) 1708 .addReg(ImmReg); 1709 BuildMI(*BB, &I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg) 1710 .addReg(MaskLo) 1711 .addImm(AMDGPU::sub0) 1712 .addReg(HiReg) 1713 .addImm(AMDGPU::sub1); 1714 I.eraseFromParent(); 1715 return true; 1716 } 1717 1718 /// Return the register to use for the index value, and the subregister to use 1719 /// for the indirectly accessed register. 1720 static std::pair<Register, unsigned> 1721 computeIndirectRegIndex(MachineRegisterInfo &MRI, 1722 const SIRegisterInfo &TRI, 1723 const TargetRegisterClass *SuperRC, 1724 Register IdxReg, 1725 unsigned EltSize) { 1726 Register IdxBaseReg; 1727 int Offset; 1728 MachineInstr *Unused; 1729 1730 std::tie(IdxBaseReg, Offset, Unused) 1731 = AMDGPU::getBaseWithConstantOffset(MRI, IdxReg); 1732 if (IdxBaseReg == AMDGPU::NoRegister) { 1733 // This will happen if the index is a known constant. This should ordinarily 1734 // be legalized out, but handle it as a register just in case. 1735 assert(Offset == 0); 1736 IdxBaseReg = IdxReg; 1737 } 1738 1739 ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SuperRC, EltSize); 1740 1741 // Skip out of bounds offsets, or else we would end up using an undefined 1742 // register. 1743 if (static_cast<unsigned>(Offset) >= SubRegs.size()) 1744 return std::make_pair(IdxReg, SubRegs[0]); 1745 return std::make_pair(IdxBaseReg, SubRegs[Offset]); 1746 } 1747 1748 bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT( 1749 MachineInstr &MI) const { 1750 Register DstReg = MI.getOperand(0).getReg(); 1751 Register SrcReg = MI.getOperand(1).getReg(); 1752 Register IdxReg = MI.getOperand(2).getReg(); 1753 1754 LLT DstTy = MRI->getType(DstReg); 1755 LLT SrcTy = MRI->getType(SrcReg); 1756 1757 const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); 1758 const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); 1759 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); 1760 1761 // The index must be scalar. If it wasn't RegBankSelect should have moved this 1762 // into a waterfall loop. 1763 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) 1764 return false; 1765 1766 const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(SrcTy, *SrcRB, 1767 *MRI); 1768 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(DstTy, *DstRB, 1769 *MRI); 1770 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || 1771 !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || 1772 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) 1773 return false; 1774 1775 MachineBasicBlock *BB = MI.getParent(); 1776 const DebugLoc &DL = MI.getDebugLoc(); 1777 const bool Is64 = DstTy.getSizeInBits() == 64; 1778 1779 unsigned SubReg; 1780 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, SrcRC, IdxReg, 1781 DstTy.getSizeInBits() / 8); 1782 1783 if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { 1784 if (DstTy.getSizeInBits() != 32 && !Is64) 1785 return false; 1786 1787 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 1788 .addReg(IdxReg); 1789 1790 unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; 1791 BuildMI(*BB, &MI, DL, TII.get(Opc), DstReg) 1792 .addReg(SrcReg, 0, SubReg) 1793 .addReg(SrcReg, RegState::Implicit); 1794 MI.eraseFromParent(); 1795 return true; 1796 } 1797 1798 if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) 1799 return false; 1800 1801 if (!STI.useVGPRIndexMode()) { 1802 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 1803 .addReg(IdxReg); 1804 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) 1805 .addReg(SrcReg, RegState::Undef, SubReg) 1806 .addReg(SrcReg, RegState::Implicit); 1807 MI.eraseFromParent(); 1808 return true; 1809 } 1810 1811 BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON)) 1812 .addReg(IdxReg) 1813 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 1814 BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstReg) 1815 .addReg(SrcReg, RegState::Undef, SubReg) 1816 .addReg(SrcReg, RegState::Implicit) 1817 .addReg(AMDGPU::M0, RegState::Implicit); 1818 BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF)); 1819 1820 MI.eraseFromParent(); 1821 return true; 1822 } 1823 1824 // TODO: Fold insert_vector_elt (extract_vector_elt) into movrelsd 1825 bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT( 1826 MachineInstr &MI) const { 1827 Register DstReg = MI.getOperand(0).getReg(); 1828 Register VecReg = MI.getOperand(1).getReg(); 1829 Register ValReg = MI.getOperand(2).getReg(); 1830 Register IdxReg = MI.getOperand(3).getReg(); 1831 1832 LLT VecTy = MRI->getType(DstReg); 1833 LLT ValTy = MRI->getType(ValReg); 1834 unsigned VecSize = VecTy.getSizeInBits(); 1835 unsigned ValSize = ValTy.getSizeInBits(); 1836 1837 const RegisterBank *VecRB = RBI.getRegBank(VecReg, *MRI, TRI); 1838 const RegisterBank *ValRB = RBI.getRegBank(ValReg, *MRI, TRI); 1839 const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); 1840 1841 assert(VecTy.getElementType() == ValTy); 1842 1843 // The index must be scalar. If it wasn't RegBankSelect should have moved this 1844 // into a waterfall loop. 1845 if (IdxRB->getID() != AMDGPU::SGPRRegBankID) 1846 return false; 1847 1848 const TargetRegisterClass *VecRC = TRI.getRegClassForTypeOnBank(VecTy, *VecRB, 1849 *MRI); 1850 const TargetRegisterClass *ValRC = TRI.getRegClassForTypeOnBank(ValTy, *ValRB, 1851 *MRI); 1852 1853 if (!RBI.constrainGenericRegister(VecReg, *VecRC, *MRI) || 1854 !RBI.constrainGenericRegister(DstReg, *VecRC, *MRI) || 1855 !RBI.constrainGenericRegister(ValReg, *ValRC, *MRI) || 1856 !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) 1857 return false; 1858 1859 if (VecRB->getID() == AMDGPU::VGPRRegBankID && ValSize != 32) 1860 return false; 1861 1862 unsigned SubReg; 1863 std::tie(IdxReg, SubReg) = computeIndirectRegIndex(*MRI, TRI, VecRC, IdxReg, 1864 ValSize / 8); 1865 1866 const bool IndexMode = VecRB->getID() == AMDGPU::VGPRRegBankID && 1867 STI.useVGPRIndexMode(); 1868 1869 MachineBasicBlock *BB = MI.getParent(); 1870 const DebugLoc &DL = MI.getDebugLoc(); 1871 1872 if (IndexMode) { 1873 BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON)) 1874 .addReg(IdxReg) 1875 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 1876 } else { 1877 BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) 1878 .addReg(IdxReg); 1879 } 1880 1881 const MCInstrDesc &RegWriteOp 1882 = TII.getIndirectRegWritePseudo(VecSize, ValSize, 1883 VecRB->getID() == AMDGPU::SGPRRegBankID); 1884 BuildMI(*BB, MI, DL, RegWriteOp, DstReg) 1885 .addReg(VecReg) 1886 .addReg(ValReg) 1887 .addImm(SubReg); 1888 1889 if (IndexMode) 1890 BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF)); 1891 1892 MI.eraseFromParent(); 1893 return true; 1894 } 1895 1896 bool AMDGPUInstructionSelector::select(MachineInstr &I) { 1897 if (I.isPHI()) 1898 return selectPHI(I); 1899 1900 if (!I.isPreISelOpcode()) { 1901 if (I.isCopy()) 1902 return selectCOPY(I); 1903 return true; 1904 } 1905 1906 switch (I.getOpcode()) { 1907 case TargetOpcode::G_AND: 1908 case TargetOpcode::G_OR: 1909 case TargetOpcode::G_XOR: 1910 if (selectG_AND_OR_XOR(I)) 1911 return true; 1912 return selectImpl(I, *CoverageInfo); 1913 case TargetOpcode::G_ADD: 1914 case TargetOpcode::G_SUB: 1915 if (selectImpl(I, *CoverageInfo)) 1916 return true; 1917 return selectG_ADD_SUB(I); 1918 case TargetOpcode::G_UADDO: 1919 case TargetOpcode::G_USUBO: 1920 case TargetOpcode::G_UADDE: 1921 case TargetOpcode::G_USUBE: 1922 return selectG_UADDO_USUBO_UADDE_USUBE(I); 1923 case TargetOpcode::G_INTTOPTR: 1924 case TargetOpcode::G_BITCAST: 1925 case TargetOpcode::G_PTRTOINT: 1926 return selectCOPY(I); 1927 case TargetOpcode::G_CONSTANT: 1928 case TargetOpcode::G_FCONSTANT: 1929 return selectG_CONSTANT(I); 1930 case TargetOpcode::G_FNEG: 1931 if (selectImpl(I, *CoverageInfo)) 1932 return true; 1933 return selectG_FNEG(I); 1934 case TargetOpcode::G_EXTRACT: 1935 return selectG_EXTRACT(I); 1936 case TargetOpcode::G_MERGE_VALUES: 1937 case TargetOpcode::G_BUILD_VECTOR: 1938 case TargetOpcode::G_CONCAT_VECTORS: 1939 return selectG_MERGE_VALUES(I); 1940 case TargetOpcode::G_UNMERGE_VALUES: 1941 return selectG_UNMERGE_VALUES(I); 1942 case TargetOpcode::G_PTR_ADD: 1943 return selectG_PTR_ADD(I); 1944 case TargetOpcode::G_IMPLICIT_DEF: 1945 return selectG_IMPLICIT_DEF(I); 1946 case TargetOpcode::G_INSERT: 1947 return selectG_INSERT(I); 1948 case TargetOpcode::G_INTRINSIC: 1949 return selectG_INTRINSIC(I); 1950 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: 1951 return selectG_INTRINSIC_W_SIDE_EFFECTS(I); 1952 case TargetOpcode::G_ICMP: 1953 if (selectG_ICMP(I)) 1954 return true; 1955 return selectImpl(I, *CoverageInfo); 1956 case TargetOpcode::G_LOAD: 1957 case TargetOpcode::G_ATOMIC_CMPXCHG: 1958 case TargetOpcode::G_ATOMICRMW_XCHG: 1959 case TargetOpcode::G_ATOMICRMW_ADD: 1960 case TargetOpcode::G_ATOMICRMW_SUB: 1961 case TargetOpcode::G_ATOMICRMW_AND: 1962 case TargetOpcode::G_ATOMICRMW_OR: 1963 case TargetOpcode::G_ATOMICRMW_XOR: 1964 case TargetOpcode::G_ATOMICRMW_MIN: 1965 case TargetOpcode::G_ATOMICRMW_MAX: 1966 case TargetOpcode::G_ATOMICRMW_UMIN: 1967 case TargetOpcode::G_ATOMICRMW_UMAX: 1968 case TargetOpcode::G_ATOMICRMW_FADD: 1969 return selectG_LOAD_ATOMICRMW(I); 1970 case TargetOpcode::G_SELECT: 1971 return selectG_SELECT(I); 1972 case TargetOpcode::G_STORE: 1973 return selectG_STORE(I); 1974 case TargetOpcode::G_TRUNC: 1975 return selectG_TRUNC(I); 1976 case TargetOpcode::G_SEXT: 1977 case TargetOpcode::G_ZEXT: 1978 case TargetOpcode::G_ANYEXT: 1979 case TargetOpcode::G_SEXT_INREG: 1980 if (selectImpl(I, *CoverageInfo)) 1981 return true; 1982 return selectG_SZA_EXT(I); 1983 case TargetOpcode::G_BRCOND: 1984 return selectG_BRCOND(I); 1985 case TargetOpcode::G_FRAME_INDEX: 1986 case TargetOpcode::G_GLOBAL_VALUE: 1987 return selectG_FRAME_INDEX_GLOBAL_VALUE(I); 1988 case TargetOpcode::G_PTR_MASK: 1989 return selectG_PTR_MASK(I); 1990 case TargetOpcode::G_EXTRACT_VECTOR_ELT: 1991 return selectG_EXTRACT_VECTOR_ELT(I); 1992 case TargetOpcode::G_INSERT_VECTOR_ELT: 1993 return selectG_INSERT_VECTOR_ELT(I); 1994 case AMDGPU::G_AMDGPU_ATOMIC_INC: 1995 case AMDGPU::G_AMDGPU_ATOMIC_DEC: 1996 initM0(I); 1997 return selectImpl(I, *CoverageInfo); 1998 default: 1999 return selectImpl(I, *CoverageInfo); 2000 } 2001 return false; 2002 } 2003 2004 InstructionSelector::ComplexRendererFns 2005 AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const { 2006 return {{ 2007 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } 2008 }}; 2009 2010 } 2011 2012 std::pair<Register, unsigned> 2013 AMDGPUInstructionSelector::selectVOP3ModsImpl( 2014 Register Src) const { 2015 unsigned Mods = 0; 2016 MachineInstr *MI = getDefIgnoringCopies(Src, *MRI); 2017 2018 if (MI && MI->getOpcode() == AMDGPU::G_FNEG) { 2019 Src = MI->getOperand(1).getReg(); 2020 Mods |= SISrcMods::NEG; 2021 MI = getDefIgnoringCopies(Src, *MRI); 2022 } 2023 2024 if (MI && MI->getOpcode() == AMDGPU::G_FABS) { 2025 Src = MI->getOperand(1).getReg(); 2026 Mods |= SISrcMods::ABS; 2027 } 2028 2029 return std::make_pair(Src, Mods); 2030 } 2031 2032 /// 2033 /// This will select either an SGPR or VGPR operand and will save us from 2034 /// having to write an extra tablegen pattern. 2035 InstructionSelector::ComplexRendererFns 2036 AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const { 2037 return {{ 2038 [=](MachineInstrBuilder &MIB) { MIB.add(Root); } 2039 }}; 2040 } 2041 2042 InstructionSelector::ComplexRendererFns 2043 AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const { 2044 Register Src; 2045 unsigned Mods; 2046 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg()); 2047 2048 return {{ 2049 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }, 2050 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods 2051 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp 2052 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod 2053 }}; 2054 } 2055 2056 InstructionSelector::ComplexRendererFns 2057 AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const { 2058 return {{ 2059 [=](MachineInstrBuilder &MIB) { MIB.add(Root); }, 2060 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp 2061 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod 2062 }}; 2063 } 2064 2065 InstructionSelector::ComplexRendererFns 2066 AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const { 2067 Register Src; 2068 unsigned Mods; 2069 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg()); 2070 2071 return {{ 2072 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }, 2073 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods 2074 }}; 2075 } 2076 2077 InstructionSelector::ComplexRendererFns 2078 AMDGPUInstructionSelector::selectVOP3NoMods(MachineOperand &Root) const { 2079 Register Reg = Root.getReg(); 2080 const MachineInstr *Def = getDefIgnoringCopies(Reg, *MRI); 2081 if (Def && (Def->getOpcode() == AMDGPU::G_FNEG || 2082 Def->getOpcode() == AMDGPU::G_FABS)) 2083 return {}; 2084 return {{ 2085 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); }, 2086 }}; 2087 } 2088 2089 InstructionSelector::ComplexRendererFns 2090 AMDGPUInstructionSelector::selectVOP3Mods_nnan(MachineOperand &Root) const { 2091 Register Src; 2092 unsigned Mods; 2093 std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg()); 2094 if (!TM.Options.NoNaNsFPMath && !isKnownNeverNaN(Src, *MRI)) 2095 return None; 2096 2097 return {{ 2098 [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }, 2099 [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods 2100 }}; 2101 } 2102 2103 InstructionSelector::ComplexRendererFns 2104 AMDGPUInstructionSelector::selectVOP3OpSelMods0(MachineOperand &Root) const { 2105 // FIXME: Handle clamp and op_sel 2106 return {{ 2107 [=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }, 2108 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // src_mods 2109 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // clamp 2110 }}; 2111 } 2112 2113 InstructionSelector::ComplexRendererFns 2114 AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const { 2115 // FIXME: Handle op_sel 2116 return {{ 2117 [=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }, 2118 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // src_mods 2119 }}; 2120 } 2121 2122 InstructionSelector::ComplexRendererFns 2123 AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const { 2124 SmallVector<GEPInfo, 4> AddrInfo; 2125 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); 2126 2127 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1) 2128 return None; 2129 2130 const GEPInfo &GEPInfo = AddrInfo[0]; 2131 Optional<int64_t> EncodedImm = AMDGPU::getSMRDEncodedOffset(STI, GEPInfo.Imm); 2132 if (!EncodedImm) 2133 return None; 2134 2135 unsigned PtrReg = GEPInfo.SgprParts[0]; 2136 return {{ 2137 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); }, 2138 [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } 2139 }}; 2140 } 2141 2142 InstructionSelector::ComplexRendererFns 2143 AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const { 2144 SmallVector<GEPInfo, 4> AddrInfo; 2145 getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo); 2146 2147 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1) 2148 return None; 2149 2150 const GEPInfo &GEPInfo = AddrInfo[0]; 2151 unsigned PtrReg = GEPInfo.SgprParts[0]; 2152 Optional<int64_t> EncodedImm = 2153 AMDGPU::getSMRDEncodedLiteralOffset32(STI, GEPInfo.Imm); 2154 if (!EncodedImm) 2155 return None; 2156 2157 return {{ 2158 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); }, 2159 [=](MachineInstrBuilder &MIB) { MIB.addImm(*EncodedImm); } 2160 }}; 2161 } 2162 2163 InstructionSelector::ComplexRendererFns 2164 AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const { 2165 MachineInstr *MI = Root.getParent(); 2166 MachineBasicBlock *MBB = MI->getParent(); 2167 2168 SmallVector<GEPInfo, 4> AddrInfo; 2169 getAddrModeInfo(*MI, *MRI, AddrInfo); 2170 2171 // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits, 2172 // then we can select all ptr + 32-bit offsets not just immediate offsets. 2173 if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1) 2174 return None; 2175 2176 const GEPInfo &GEPInfo = AddrInfo[0]; 2177 if (!GEPInfo.Imm || !isUInt<32>(GEPInfo.Imm)) 2178 return None; 2179 2180 // If we make it this far we have a load with an 32-bit immediate offset. 2181 // It is OK to select this using a sgpr offset, because we have already 2182 // failed trying to select this load into one of the _IMM variants since 2183 // the _IMM Patterns are considered before the _SGPR patterns. 2184 unsigned PtrReg = GEPInfo.SgprParts[0]; 2185 Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 2186 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), OffsetReg) 2187 .addImm(GEPInfo.Imm); 2188 return {{ 2189 [=](MachineInstrBuilder &MIB) { MIB.addReg(PtrReg); }, 2190 [=](MachineInstrBuilder &MIB) { MIB.addReg(OffsetReg); } 2191 }}; 2192 } 2193 2194 template <bool Signed> 2195 InstructionSelector::ComplexRendererFns 2196 AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root) const { 2197 MachineInstr *MI = Root.getParent(); 2198 2199 InstructionSelector::ComplexRendererFns Default = {{ 2200 [=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }, 2201 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // offset 2202 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // slc 2203 }}; 2204 2205 if (!STI.hasFlatInstOffsets()) 2206 return Default; 2207 2208 const MachineInstr *OpDef = MRI->getVRegDef(Root.getReg()); 2209 if (!OpDef || OpDef->getOpcode() != AMDGPU::G_PTR_ADD) 2210 return Default; 2211 2212 Optional<int64_t> Offset = 2213 getConstantVRegVal(OpDef->getOperand(2).getReg(), *MRI); 2214 if (!Offset.hasValue()) 2215 return Default; 2216 2217 unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace(); 2218 if (!TII.isLegalFLATOffset(Offset.getValue(), AddrSpace, Signed)) 2219 return Default; 2220 2221 Register BasePtr = OpDef->getOperand(1).getReg(); 2222 2223 return {{ 2224 [=](MachineInstrBuilder &MIB) { MIB.addReg(BasePtr); }, 2225 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset.getValue()); }, 2226 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // slc 2227 }}; 2228 } 2229 2230 InstructionSelector::ComplexRendererFns 2231 AMDGPUInstructionSelector::selectFlatOffset(MachineOperand &Root) const { 2232 return selectFlatOffsetImpl<false>(Root); 2233 } 2234 2235 InstructionSelector::ComplexRendererFns 2236 AMDGPUInstructionSelector::selectFlatOffsetSigned(MachineOperand &Root) const { 2237 return selectFlatOffsetImpl<true>(Root); 2238 } 2239 2240 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 2241 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 2242 return PSV && PSV->isStack(); 2243 } 2244 2245 InstructionSelector::ComplexRendererFns 2246 AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const { 2247 MachineInstr *MI = Root.getParent(); 2248 MachineBasicBlock *MBB = MI->getParent(); 2249 MachineFunction *MF = MBB->getParent(); 2250 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 2251 2252 int64_t Offset = 0; 2253 if (mi_match(Root.getReg(), *MRI, m_ICst(Offset))) { 2254 Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2255 2256 // TODO: Should this be inside the render function? The iterator seems to 2257 // move. 2258 BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::V_MOV_B32_e32), 2259 HighBits) 2260 .addImm(Offset & ~4095); 2261 2262 return {{[=](MachineInstrBuilder &MIB) { // rsrc 2263 MIB.addReg(Info->getScratchRSrcReg()); 2264 }, 2265 [=](MachineInstrBuilder &MIB) { // vaddr 2266 MIB.addReg(HighBits); 2267 }, 2268 [=](MachineInstrBuilder &MIB) { // soffset 2269 const MachineMemOperand *MMO = *MI->memoperands_begin(); 2270 const MachinePointerInfo &PtrInfo = MMO->getPointerInfo(); 2271 2272 Register SOffsetReg = isStackPtrRelative(PtrInfo) 2273 ? Info->getStackPtrOffsetReg() 2274 : Info->getScratchWaveOffsetReg(); 2275 MIB.addReg(SOffsetReg); 2276 }, 2277 [=](MachineInstrBuilder &MIB) { // offset 2278 MIB.addImm(Offset & 4095); 2279 }}}; 2280 } 2281 2282 assert(Offset == 0); 2283 2284 // Try to fold a frame index directly into the MUBUF vaddr field, and any 2285 // offsets. 2286 Optional<int> FI; 2287 Register VAddr = Root.getReg(); 2288 if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) { 2289 if (isBaseWithConstantOffset(Root, *MRI)) { 2290 const MachineOperand &LHS = RootDef->getOperand(1); 2291 const MachineOperand &RHS = RootDef->getOperand(2); 2292 const MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg()); 2293 const MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg()); 2294 if (LHSDef && RHSDef) { 2295 int64_t PossibleOffset = 2296 RHSDef->getOperand(1).getCImm()->getSExtValue(); 2297 if (SIInstrInfo::isLegalMUBUFImmOffset(PossibleOffset) && 2298 (!STI.privateMemoryResourceIsRangeChecked() || 2299 KnownBits->signBitIsZero(LHS.getReg()))) { 2300 if (LHSDef->getOpcode() == AMDGPU::G_FRAME_INDEX) 2301 FI = LHSDef->getOperand(1).getIndex(); 2302 else 2303 VAddr = LHS.getReg(); 2304 Offset = PossibleOffset; 2305 } 2306 } 2307 } else if (RootDef->getOpcode() == AMDGPU::G_FRAME_INDEX) { 2308 FI = RootDef->getOperand(1).getIndex(); 2309 } 2310 } 2311 2312 // If we don't know this private access is a local stack object, it needs to 2313 // be relative to the entry point's scratch wave offset register. 2314 // TODO: Should split large offsets that don't fit like above. 2315 // TODO: Don't use scratch wave offset just because the offset didn't fit. 2316 Register SOffset = FI.hasValue() ? Info->getStackPtrOffsetReg() 2317 : Info->getScratchWaveOffsetReg(); 2318 2319 return {{[=](MachineInstrBuilder &MIB) { // rsrc 2320 MIB.addReg(Info->getScratchRSrcReg()); 2321 }, 2322 [=](MachineInstrBuilder &MIB) { // vaddr 2323 if (FI.hasValue()) 2324 MIB.addFrameIndex(FI.getValue()); 2325 else 2326 MIB.addReg(VAddr); 2327 }, 2328 [=](MachineInstrBuilder &MIB) { // soffset 2329 MIB.addReg(SOffset); 2330 }, 2331 [=](MachineInstrBuilder &MIB) { // offset 2332 MIB.addImm(Offset); 2333 }}}; 2334 } 2335 2336 bool AMDGPUInstructionSelector::isDSOffsetLegal(Register Base, 2337 int64_t Offset, 2338 unsigned OffsetBits) const { 2339 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 2340 (OffsetBits == 8 && !isUInt<8>(Offset))) 2341 return false; 2342 2343 if (STI.hasUsableDSOffset() || STI.unsafeDSOffsetFoldingEnabled()) 2344 return true; 2345 2346 // On Southern Islands instruction with a negative base value and an offset 2347 // don't seem to work. 2348 return KnownBits->signBitIsZero(Base); 2349 } 2350 2351 InstructionSelector::ComplexRendererFns 2352 AMDGPUInstructionSelector::selectMUBUFScratchOffset( 2353 MachineOperand &Root) const { 2354 MachineInstr *MI = Root.getParent(); 2355 MachineBasicBlock *MBB = MI->getParent(); 2356 2357 int64_t Offset = 0; 2358 if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) || 2359 !SIInstrInfo::isLegalMUBUFImmOffset(Offset)) 2360 return {}; 2361 2362 const MachineFunction *MF = MBB->getParent(); 2363 const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 2364 const MachineMemOperand *MMO = *MI->memoperands_begin(); 2365 const MachinePointerInfo &PtrInfo = MMO->getPointerInfo(); 2366 2367 Register SOffsetReg = isStackPtrRelative(PtrInfo) 2368 ? Info->getStackPtrOffsetReg() 2369 : Info->getScratchWaveOffsetReg(); 2370 return {{ 2371 [=](MachineInstrBuilder &MIB) { 2372 MIB.addReg(Info->getScratchRSrcReg()); 2373 }, // rsrc 2374 [=](MachineInstrBuilder &MIB) { MIB.addReg(SOffsetReg); }, // soffset 2375 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } // offset 2376 }}; 2377 } 2378 2379 std::pair<Register, unsigned> 2380 AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const { 2381 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); 2382 if (!RootDef) 2383 return std::make_pair(Root.getReg(), 0); 2384 2385 int64_t ConstAddr = 0; 2386 2387 Register PtrBase; 2388 int64_t Offset; 2389 std::tie(PtrBase, Offset) = 2390 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); 2391 2392 if (Offset) { 2393 if (isDSOffsetLegal(PtrBase, Offset, 16)) { 2394 // (add n0, c0) 2395 return std::make_pair(PtrBase, Offset); 2396 } 2397 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { 2398 // TODO 2399 2400 2401 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { 2402 // TODO 2403 2404 } 2405 2406 return std::make_pair(Root.getReg(), 0); 2407 } 2408 2409 InstructionSelector::ComplexRendererFns 2410 AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const { 2411 Register Reg; 2412 unsigned Offset; 2413 std::tie(Reg, Offset) = selectDS1Addr1OffsetImpl(Root); 2414 return {{ 2415 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); }, 2416 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); } 2417 }}; 2418 } 2419 2420 InstructionSelector::ComplexRendererFns 2421 AMDGPUInstructionSelector::selectDS64Bit4ByteAligned(MachineOperand &Root) const { 2422 Register Reg; 2423 unsigned Offset; 2424 std::tie(Reg, Offset) = selectDS64Bit4ByteAlignedImpl(Root); 2425 return {{ 2426 [=](MachineInstrBuilder &MIB) { MIB.addReg(Reg); }, 2427 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }, 2428 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset+1); } 2429 }}; 2430 } 2431 2432 std::pair<Register, unsigned> 2433 AMDGPUInstructionSelector::selectDS64Bit4ByteAlignedImpl(MachineOperand &Root) const { 2434 const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg()); 2435 if (!RootDef) 2436 return std::make_pair(Root.getReg(), 0); 2437 2438 int64_t ConstAddr = 0; 2439 2440 Register PtrBase; 2441 int64_t Offset; 2442 std::tie(PtrBase, Offset) = 2443 getPtrBaseWithConstantOffset(Root.getReg(), *MRI); 2444 2445 if (Offset) { 2446 int64_t DWordOffset0 = Offset / 4; 2447 int64_t DWordOffset1 = DWordOffset0 + 1; 2448 if (isDSOffsetLegal(PtrBase, DWordOffset1, 8)) { 2449 // (add n0, c0) 2450 return std::make_pair(PtrBase, DWordOffset0); 2451 } 2452 } else if (RootDef->getOpcode() == AMDGPU::G_SUB) { 2453 // TODO 2454 2455 } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) { 2456 // TODO 2457 2458 } 2459 2460 return std::make_pair(Root.getReg(), 0); 2461 } 2462 2463 /// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return 2464 /// the base value with the constant offset. There may be intervening copies 2465 /// between \p Root and the identified constant. Returns \p Root, 0 if this does 2466 /// not match the pattern. 2467 std::pair<Register, int64_t> 2468 AMDGPUInstructionSelector::getPtrBaseWithConstantOffset( 2469 Register Root, const MachineRegisterInfo &MRI) const { 2470 MachineInstr *RootI = MRI.getVRegDef(Root); 2471 if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD) 2472 return {Root, 0}; 2473 2474 MachineOperand &RHS = RootI->getOperand(2); 2475 Optional<ValueAndVReg> MaybeOffset 2476 = getConstantVRegValWithLookThrough(RHS.getReg(), MRI, true); 2477 if (!MaybeOffset) 2478 return {Root, 0}; 2479 return {RootI->getOperand(1).getReg(), MaybeOffset->Value}; 2480 } 2481 2482 static void addZeroImm(MachineInstrBuilder &MIB) { 2483 MIB.addImm(0); 2484 } 2485 2486 /// Return a resource descriptor for use with an arbitrary 64-bit pointer. If \p 2487 /// BasePtr is not valid, a null base pointer will be used. 2488 static Register buildRSRC(MachineIRBuilder &B, MachineRegisterInfo &MRI, 2489 uint32_t FormatLo, uint32_t FormatHi, 2490 Register BasePtr) { 2491 Register RSrc2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 2492 Register RSrc3 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 2493 Register RSrcHi = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2494 Register RSrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 2495 2496 B.buildInstr(AMDGPU::S_MOV_B32) 2497 .addDef(RSrc2) 2498 .addImm(FormatLo); 2499 B.buildInstr(AMDGPU::S_MOV_B32) 2500 .addDef(RSrc3) 2501 .addImm(FormatHi); 2502 2503 // Build the half of the subregister with the constants before building the 2504 // full 128-bit register. If we are building multiple resource descriptors, 2505 // this will allow CSEing of the 2-component register. 2506 B.buildInstr(AMDGPU::REG_SEQUENCE) 2507 .addDef(RSrcHi) 2508 .addReg(RSrc2) 2509 .addImm(AMDGPU::sub0) 2510 .addReg(RSrc3) 2511 .addImm(AMDGPU::sub1); 2512 2513 Register RSrcLo = BasePtr; 2514 if (!BasePtr) { 2515 RSrcLo = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2516 B.buildInstr(AMDGPU::S_MOV_B64) 2517 .addDef(RSrcLo) 2518 .addImm(0); 2519 } 2520 2521 B.buildInstr(AMDGPU::REG_SEQUENCE) 2522 .addDef(RSrc) 2523 .addReg(RSrcLo) 2524 .addImm(AMDGPU::sub0_sub1) 2525 .addReg(RSrcHi) 2526 .addImm(AMDGPU::sub2_sub3); 2527 2528 return RSrc; 2529 } 2530 2531 static Register buildAddr64RSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, 2532 const SIInstrInfo &TII, Register BasePtr) { 2533 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat(); 2534 2535 // FIXME: Why are half the "default" bits ignored based on the addressing 2536 // mode? 2537 return buildRSRC(B, MRI, 0, Hi_32(DefaultFormat), BasePtr); 2538 } 2539 2540 static Register buildOffsetSrc(MachineIRBuilder &B, MachineRegisterInfo &MRI, 2541 const SIInstrInfo &TII, Register BasePtr) { 2542 uint64_t DefaultFormat = TII.getDefaultRsrcDataFormat(); 2543 2544 // FIXME: Why are half the "default" bits ignored based on the addressing 2545 // mode? 2546 return buildRSRC(B, MRI, -1, Hi_32(DefaultFormat), BasePtr); 2547 } 2548 2549 AMDGPUInstructionSelector::MUBUFAddressData 2550 AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const { 2551 MUBUFAddressData Data; 2552 Data.N0 = Src; 2553 2554 Register PtrBase; 2555 int64_t Offset; 2556 2557 std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI); 2558 if (isUInt<32>(Offset)) { 2559 Data.N0 = PtrBase; 2560 Data.Offset = Offset; 2561 } 2562 2563 if (MachineInstr *InputAdd 2564 = getOpcodeDef(TargetOpcode::G_PTR_ADD, Data.N0, *MRI)) { 2565 Data.N2 = InputAdd->getOperand(1).getReg(); 2566 Data.N3 = InputAdd->getOperand(2).getReg(); 2567 2568 // FIXME: Need to fix extra SGPR->VGPRcopies inserted 2569 // FIXME: Don't know this was defined by operand 0 2570 // 2571 // TODO: Remove this when we have copy folding optimizations after 2572 // RegBankSelect. 2573 Data.N2 = getDefIgnoringCopies(Data.N2, *MRI)->getOperand(0).getReg(); 2574 Data.N3 = getDefIgnoringCopies(Data.N3, *MRI)->getOperand(0).getReg(); 2575 } 2576 2577 return Data; 2578 } 2579 2580 /// Return if the addr64 mubuf mode should be used for the given address. 2581 bool AMDGPUInstructionSelector::shouldUseAddr64(MUBUFAddressData Addr) const { 2582 // (ptr_add N2, N3) -> addr64, or 2583 // (ptr_add (ptr_add N2, N3), C1) -> addr64 2584 if (Addr.N2) 2585 return true; 2586 2587 const RegisterBank *N0Bank = RBI.getRegBank(Addr.N0, *MRI, TRI); 2588 return N0Bank->getID() == AMDGPU::VGPRRegBankID; 2589 } 2590 2591 /// Split an immediate offset \p ImmOffset depending on whether it fits in the 2592 /// immediate field. Modifies \p ImmOffset and sets \p SOffset to the variable 2593 /// component. 2594 void AMDGPUInstructionSelector::splitIllegalMUBUFOffset( 2595 MachineIRBuilder &B, Register &SOffset, int64_t &ImmOffset) const { 2596 if (SIInstrInfo::isLegalMUBUFImmOffset(ImmOffset)) 2597 return; 2598 2599 // Illegal offset, store it in soffset. 2600 SOffset = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); 2601 B.buildInstr(AMDGPU::S_MOV_B32) 2602 .addDef(SOffset) 2603 .addImm(ImmOffset); 2604 ImmOffset = 0; 2605 } 2606 2607 bool AMDGPUInstructionSelector::selectMUBUFAddr64Impl( 2608 MachineOperand &Root, Register &VAddr, Register &RSrcReg, 2609 Register &SOffset, int64_t &Offset) const { 2610 // FIXME: Predicates should stop this from reaching here. 2611 // addr64 bit was removed for volcanic islands. 2612 if (!STI.hasAddr64() || STI.useFlatForGlobal()) 2613 return false; 2614 2615 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); 2616 if (!shouldUseAddr64(AddrData)) 2617 return false; 2618 2619 Register N0 = AddrData.N0; 2620 Register N2 = AddrData.N2; 2621 Register N3 = AddrData.N3; 2622 Offset = AddrData.Offset; 2623 2624 // Base pointer for the SRD. 2625 Register SRDPtr; 2626 2627 if (N2) { 2628 if (RBI.getRegBank(N2, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { 2629 assert(N3); 2630 if (RBI.getRegBank(N3, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { 2631 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 2632 // addr64, and construct the default resource from a 0 address. 2633 VAddr = N0; 2634 } else { 2635 SRDPtr = N3; 2636 VAddr = N2; 2637 } 2638 } else { 2639 // N2 is not divergent. 2640 SRDPtr = N2; 2641 VAddr = N3; 2642 } 2643 } else if (RBI.getRegBank(N0, *MRI, TRI)->getID() == AMDGPU::VGPRRegBankID) { 2644 // Use the default null pointer in the resource 2645 VAddr = N0; 2646 } else { 2647 // N0 -> offset, or 2648 // (N0 + C1) -> offset 2649 SRDPtr = N0; 2650 } 2651 2652 MachineIRBuilder B(*Root.getParent()); 2653 RSrcReg = buildAddr64RSrc(B, *MRI, TII, SRDPtr); 2654 splitIllegalMUBUFOffset(B, SOffset, Offset); 2655 return true; 2656 } 2657 2658 bool AMDGPUInstructionSelector::selectMUBUFOffsetImpl( 2659 MachineOperand &Root, Register &RSrcReg, Register &SOffset, 2660 int64_t &Offset) const { 2661 MUBUFAddressData AddrData = parseMUBUFAddress(Root.getReg()); 2662 if (shouldUseAddr64(AddrData)) 2663 return false; 2664 2665 // N0 -> offset, or 2666 // (N0 + C1) -> offset 2667 Register SRDPtr = AddrData.N0; 2668 Offset = AddrData.Offset; 2669 2670 // TODO: Look through extensions for 32-bit soffset. 2671 MachineIRBuilder B(*Root.getParent()); 2672 2673 RSrcReg = buildOffsetSrc(B, *MRI, TII, SRDPtr); 2674 splitIllegalMUBUFOffset(B, SOffset, Offset); 2675 return true; 2676 } 2677 2678 InstructionSelector::ComplexRendererFns 2679 AMDGPUInstructionSelector::selectMUBUFAddr64(MachineOperand &Root) const { 2680 Register VAddr; 2681 Register RSrcReg; 2682 Register SOffset; 2683 int64_t Offset = 0; 2684 2685 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset)) 2686 return {}; 2687 2688 // FIXME: Use defaulted operands for trailing 0s and remove from the complex 2689 // pattern. 2690 return {{ 2691 [=](MachineInstrBuilder &MIB) { // rsrc 2692 MIB.addReg(RSrcReg); 2693 }, 2694 [=](MachineInstrBuilder &MIB) { // vaddr 2695 MIB.addReg(VAddr); 2696 }, 2697 [=](MachineInstrBuilder &MIB) { // soffset 2698 if (SOffset) 2699 MIB.addReg(SOffset); 2700 else 2701 MIB.addImm(0); 2702 }, 2703 [=](MachineInstrBuilder &MIB) { // offset 2704 MIB.addImm(Offset); 2705 }, 2706 addZeroImm, // glc 2707 addZeroImm, // slc 2708 addZeroImm, // tfe 2709 addZeroImm, // dlc 2710 addZeroImm // swz 2711 }}; 2712 } 2713 2714 InstructionSelector::ComplexRendererFns 2715 AMDGPUInstructionSelector::selectMUBUFOffset(MachineOperand &Root) const { 2716 Register RSrcReg; 2717 Register SOffset; 2718 int64_t Offset = 0; 2719 2720 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset)) 2721 return {}; 2722 2723 return {{ 2724 [=](MachineInstrBuilder &MIB) { // rsrc 2725 MIB.addReg(RSrcReg); 2726 }, 2727 [=](MachineInstrBuilder &MIB) { // soffset 2728 if (SOffset) 2729 MIB.addReg(SOffset); 2730 else 2731 MIB.addImm(0); 2732 }, 2733 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }, // offset 2734 addZeroImm, // glc 2735 addZeroImm, // slc 2736 addZeroImm, // tfe 2737 addZeroImm, // dlc 2738 addZeroImm // swz 2739 }}; 2740 } 2741 2742 InstructionSelector::ComplexRendererFns 2743 AMDGPUInstructionSelector::selectMUBUFAddr64Atomic(MachineOperand &Root) const { 2744 Register VAddr; 2745 Register RSrcReg; 2746 Register SOffset; 2747 int64_t Offset = 0; 2748 2749 if (!selectMUBUFAddr64Impl(Root, VAddr, RSrcReg, SOffset, Offset)) 2750 return {}; 2751 2752 // FIXME: Use defaulted operands for trailing 0s and remove from the complex 2753 // pattern. 2754 return {{ 2755 [=](MachineInstrBuilder &MIB) { // rsrc 2756 MIB.addReg(RSrcReg); 2757 }, 2758 [=](MachineInstrBuilder &MIB) { // vaddr 2759 MIB.addReg(VAddr); 2760 }, 2761 [=](MachineInstrBuilder &MIB) { // soffset 2762 if (SOffset) 2763 MIB.addReg(SOffset); 2764 else 2765 MIB.addImm(0); 2766 }, 2767 [=](MachineInstrBuilder &MIB) { // offset 2768 MIB.addImm(Offset); 2769 }, 2770 addZeroImm // slc 2771 }}; 2772 } 2773 2774 InstructionSelector::ComplexRendererFns 2775 AMDGPUInstructionSelector::selectMUBUFOffsetAtomic(MachineOperand &Root) const { 2776 Register RSrcReg; 2777 Register SOffset; 2778 int64_t Offset = 0; 2779 2780 if (!selectMUBUFOffsetImpl(Root, RSrcReg, SOffset, Offset)) 2781 return {}; 2782 2783 return {{ 2784 [=](MachineInstrBuilder &MIB) { // rsrc 2785 MIB.addReg(RSrcReg); 2786 }, 2787 [=](MachineInstrBuilder &MIB) { // soffset 2788 if (SOffset) 2789 MIB.addReg(SOffset); 2790 else 2791 MIB.addImm(0); 2792 }, 2793 [=](MachineInstrBuilder &MIB) { MIB.addImm(Offset); }, // offset 2794 addZeroImm // slc 2795 }}; 2796 } 2797 2798 void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB, 2799 const MachineInstr &MI, 2800 int OpIdx) const { 2801 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2802 "Expected G_CONSTANT"); 2803 MIB.addImm(MI.getOperand(1).getCImm()->getSExtValue()); 2804 } 2805 2806 void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB, 2807 const MachineInstr &MI, 2808 int OpIdx) const { 2809 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2810 "Expected G_CONSTANT"); 2811 MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue()); 2812 } 2813 2814 void AMDGPUInstructionSelector::renderBitcastImm(MachineInstrBuilder &MIB, 2815 const MachineInstr &MI, 2816 int OpIdx) const { 2817 assert(OpIdx == -1); 2818 2819 const MachineOperand &Op = MI.getOperand(1); 2820 if (MI.getOpcode() == TargetOpcode::G_FCONSTANT) 2821 MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); 2822 else { 2823 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); 2824 MIB.addImm(Op.getCImm()->getSExtValue()); 2825 } 2826 } 2827 2828 void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB, 2829 const MachineInstr &MI, 2830 int OpIdx) const { 2831 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && 2832 "Expected G_CONSTANT"); 2833 MIB.addImm(MI.getOperand(1).getCImm()->getValue().countPopulation()); 2834 } 2835 2836 /// This only really exists to satisfy DAG type checking machinery, so is a 2837 /// no-op here. 2838 void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB, 2839 const MachineInstr &MI, 2840 int OpIdx) const { 2841 MIB.addImm(MI.getOperand(OpIdx).getImm()); 2842 } 2843 2844 void AMDGPUInstructionSelector::renderExtractGLC(MachineInstrBuilder &MIB, 2845 const MachineInstr &MI, 2846 int OpIdx) const { 2847 assert(OpIdx >= 0 && "expected to match an immediate operand"); 2848 MIB.addImm(MI.getOperand(OpIdx).getImm() & 1); 2849 } 2850 2851 void AMDGPUInstructionSelector::renderExtractSLC(MachineInstrBuilder &MIB, 2852 const MachineInstr &MI, 2853 int OpIdx) const { 2854 assert(OpIdx >= 0 && "expected to match an immediate operand"); 2855 MIB.addImm((MI.getOperand(OpIdx).getImm() >> 1) & 1); 2856 } 2857 2858 void AMDGPUInstructionSelector::renderExtractDLC(MachineInstrBuilder &MIB, 2859 const MachineInstr &MI, 2860 int OpIdx) const { 2861 assert(OpIdx >= 0 && "expected to match an immediate operand"); 2862 MIB.addImm((MI.getOperand(OpIdx).getImm() >> 2) & 1); 2863 } 2864 2865 void AMDGPUInstructionSelector::renderExtractSWZ(MachineInstrBuilder &MIB, 2866 const MachineInstr &MI, 2867 int OpIdx) const { 2868 assert(OpIdx >= 0 && "expected to match an immediate operand"); 2869 MIB.addImm((MI.getOperand(OpIdx).getImm() >> 3) & 1); 2870 } 2871 2872 bool AMDGPUInstructionSelector::isInlineImmediate16(int64_t Imm) const { 2873 return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); 2874 } 2875 2876 bool AMDGPUInstructionSelector::isInlineImmediate32(int64_t Imm) const { 2877 return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm()); 2878 } 2879 2880 bool AMDGPUInstructionSelector::isInlineImmediate64(int64_t Imm) const { 2881 return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm()); 2882 } 2883 2884 bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const { 2885 return TII.isInlineConstant(Imm); 2886 } 2887