1 //===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the RegisterBankInfo class for 10 /// AMDGPU. 11 /// 12 /// \par 13 /// 14 /// AMDGPU has unique register bank constraints that require special high level 15 /// strategies to deal with. There are two main true physical register banks 16 /// VGPR (vector), and SGPR (scalar). Additionally the VCC register bank is a 17 /// sort of pseudo-register bank needed to represent SGPRs used in a vector 18 /// boolean context. There is also the AGPR bank, which is a special purpose 19 /// physical register bank present on some subtargets. 20 /// 21 /// Copying from VGPR to SGPR is generally illegal, unless the value is known to 22 /// be uniform. It is generally not valid to legalize operands by inserting 23 /// copies as on other targets. Operations which require uniform, SGPR operands 24 /// generally require scalarization by repeatedly executing the instruction, 25 /// activating each set of lanes using a unique set of input values. This is 26 /// referred to as a waterfall loop. 27 /// 28 /// \par Booleans 29 /// 30 /// Booleans (s1 values) requires special consideration. A vector compare result 31 /// is naturally a bitmask with one bit per lane, in a 32 or 64-bit 32 /// register. These are represented with the VCC bank. During selection, we need 33 /// to be able to unambiguously go back from a register class to a register 34 /// bank. To distinguish whether an SGPR should use the SGPR or VCC register 35 /// bank, we need to know the use context type. An SGPR s1 value always means a 36 /// VCC bank value, otherwise it will be the SGPR bank. A scalar compare sets 37 /// SCC, which is a 1-bit unaddressable register. This will need to be copied to 38 /// a 32-bit virtual register. Taken together, this means we need to adjust the 39 /// type of boolean operations to be regbank legal. All SALU booleans need to be 40 /// widened to 32-bits, and all VALU booleans need to be s1 values. 41 /// 42 /// A noteworthy exception to the s1-means-vcc rule is for legalization artifact 43 /// casts. G_TRUNC s1 results, and G_SEXT/G_ZEXT/G_ANYEXT sources are never vcc 44 /// bank. A non-boolean source (such as a truncate from a 1-bit load from 45 /// memory) will require a copy to the VCC bank which will require clearing the 46 /// high bits and inserting a compare. 47 /// 48 /// \par Constant bus restriction 49 /// 50 /// VALU instructions have a limitation known as the constant bus 51 /// restriction. Most VALU instructions can use SGPR operands, but may read at 52 /// most 1 SGPR or constant literal value (this to 2 in gfx10 for most 53 /// instructions). This is one unique SGPR, so the same SGPR may be used for 54 /// multiple operands. From a register bank perspective, any combination of 55 /// operands should be legal as an SGPR, but this is contextually dependent on 56 /// the SGPR operands all being the same register. There is therefore optimal to 57 /// choose the SGPR with the most uses to minimize the number of copies. 58 /// 59 /// We avoid trying to solve this problem in RegBankSelect. Any VALU G_* 60 /// operation should have its source operands all mapped to VGPRs (except for 61 /// VCC), inserting copies from any SGPR operands. This the most trival legal 62 /// mapping. Anything beyond the simplest 1:1 instruction selection would be too 63 /// complicated to solve here. Every optimization pattern or instruction 64 /// selected to multiple outputs would have to enforce this rule, and there 65 /// would be additional complexity in tracking this rule for every G_* 66 /// operation. By forcing all inputs to VGPRs, it also simplifies the task of 67 /// picking the optimal operand combination from a post-isel optimization pass. 68 /// 69 //===----------------------------------------------------------------------===// 70 71 #include "AMDGPURegisterBankInfo.h" 72 73 #include "AMDGPUGlobalISelUtils.h" 74 #include "AMDGPUInstrInfo.h" 75 #include "AMDGPUSubtarget.h" 76 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 77 #include "SIMachineFunctionInfo.h" 78 #include "SIRegisterInfo.h" 79 #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" 80 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 81 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" 82 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 83 #include "llvm/CodeGen/GlobalISel/RegisterBank.h" 84 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" 85 #include "llvm/CodeGen/TargetRegisterInfo.h" 86 #include "llvm/CodeGen/TargetSubtargetInfo.h" 87 #include "llvm/IR/Constants.h" 88 89 #define GET_TARGET_REGBANK_IMPL 90 #include "AMDGPUGenRegisterBank.inc" 91 92 // This file will be TableGen'ed at some point. 93 #include "AMDGPUGenRegisterBankInfo.def" 94 95 using namespace llvm; 96 using namespace MIPatternMatch; 97 98 namespace { 99 100 // Observer to apply a register bank to new registers created by LegalizerHelper. 101 class ApplyRegBankMapping final : public GISelChangeObserver { 102 private: 103 const AMDGPURegisterBankInfo &RBI; 104 MachineRegisterInfo &MRI; 105 const RegisterBank *NewBank; 106 SmallVector<MachineInstr *, 4> NewInsts; 107 108 public: 109 ApplyRegBankMapping(const AMDGPURegisterBankInfo &RBI_, 110 MachineRegisterInfo &MRI_, const RegisterBank *RB) 111 : RBI(RBI_), MRI(MRI_), NewBank(RB) {} 112 113 ~ApplyRegBankMapping() { 114 for (MachineInstr *MI : NewInsts) 115 applyBank(*MI); 116 } 117 118 /// Set any registers that don't have a set register class or bank to SALU. 119 void applyBank(MachineInstr &MI) { 120 const unsigned Opc = MI.getOpcode(); 121 if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT || 122 Opc == AMDGPU::G_SEXT) { 123 // LegalizerHelper wants to use the basic legalization artifacts when 124 // widening etc. We don't handle selection with vcc in artifact sources, 125 // so we need to use a sslect instead to handle these properly. 126 Register DstReg = MI.getOperand(0).getReg(); 127 Register SrcReg = MI.getOperand(1).getReg(); 128 const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, *RBI.TRI); 129 if (SrcBank == &AMDGPU::VCCRegBank) { 130 const LLT S32 = LLT::scalar(32); 131 assert(MRI.getType(SrcReg) == LLT::scalar(1)); 132 assert(MRI.getType(DstReg) == S32); 133 assert(NewBank == &AMDGPU::VGPRRegBank); 134 135 // Replace the extension with a select, which really uses the boolean 136 // source. 137 MachineIRBuilder B(MI); 138 auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1); 139 auto False = B.buildConstant(S32, 0); 140 B.buildSelect(DstReg, SrcReg, True, False); 141 MRI.setRegBank(True.getReg(0), *NewBank); 142 MRI.setRegBank(False.getReg(0), *NewBank); 143 MI.eraseFromParent(); 144 } 145 146 assert(!MRI.getRegClassOrRegBank(DstReg)); 147 MRI.setRegBank(DstReg, *NewBank); 148 return; 149 } 150 151 #ifndef NDEBUG 152 if (Opc == AMDGPU::G_TRUNC) { 153 Register DstReg = MI.getOperand(0).getReg(); 154 const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI); 155 assert(DstBank != &AMDGPU::VCCRegBank); 156 } 157 #endif 158 159 for (MachineOperand &Op : MI.operands()) { 160 if (!Op.isReg()) 161 continue; 162 163 // We may see physical registers if building a real MI 164 Register Reg = Op.getReg(); 165 if (Reg.isPhysical() || MRI.getRegClassOrRegBank(Reg)) 166 continue; 167 168 const RegisterBank *RB = NewBank; 169 if (MRI.getType(Reg) == LLT::scalar(1)) { 170 assert(NewBank == &AMDGPU::VGPRRegBank && 171 "s1 operands should only be used for vector bools"); 172 assert((MI.getOpcode() != AMDGPU::G_TRUNC && 173 MI.getOpcode() != AMDGPU::G_ANYEXT) && 174 "not expecting legalization artifacts here"); 175 RB = &AMDGPU::VCCRegBank; 176 } 177 178 MRI.setRegBank(Reg, *RB); 179 } 180 } 181 182 void erasingInstr(MachineInstr &MI) override {} 183 184 void createdInstr(MachineInstr &MI) override { 185 // At this point, the instruction was just inserted and has no operands. 186 NewInsts.push_back(&MI); 187 } 188 189 void changingInstr(MachineInstr &MI) override {} 190 void changedInstr(MachineInstr &MI) override {} 191 }; 192 193 } 194 AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST) 195 : AMDGPUGenRegisterBankInfo(), 196 Subtarget(ST), 197 TRI(Subtarget.getRegisterInfo()), 198 TII(Subtarget.getInstrInfo()) { 199 200 // HACK: Until this is fully tablegen'd. 201 static llvm::once_flag InitializeRegisterBankFlag; 202 203 static auto InitializeRegisterBankOnce = [this]() { 204 assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank && 205 &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank && 206 &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank); 207 (void)this; 208 }; 209 210 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce); 211 } 212 213 static bool isVectorRegisterBank(const RegisterBank &Bank) { 214 unsigned BankID = Bank.getID(); 215 return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID; 216 } 217 218 unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, 219 const RegisterBank &Src, 220 unsigned Size) const { 221 // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane? 222 if (Dst.getID() == AMDGPU::SGPRRegBankID && 223 (isVectorRegisterBank(Src) || Src.getID() == AMDGPU::VCCRegBankID)) { 224 return std::numeric_limits<unsigned>::max(); 225 } 226 227 // Bool values are tricky, because the meaning is based on context. The SCC 228 // and VCC banks are for the natural scalar and vector conditions produced by 229 // a compare. 230 // 231 // Legalization doesn't know about the necessary context, so an s1 use may 232 // have been a truncate from an arbitrary value, in which case a copy (lowered 233 // as a compare with 0) needs to be inserted. 234 if (Size == 1 && 235 (Dst.getID() == AMDGPU::SGPRRegBankID) && 236 (isVectorRegisterBank(Src) || 237 Src.getID() == AMDGPU::SGPRRegBankID || 238 Src.getID() == AMDGPU::VCCRegBankID)) 239 return std::numeric_limits<unsigned>::max(); 240 241 // There is no direct copy between AGPRs. 242 if (Dst.getID() == AMDGPU::AGPRRegBankID && 243 Src.getID() == AMDGPU::AGPRRegBankID) 244 return 4; 245 246 return RegisterBankInfo::copyCost(Dst, Src, Size); 247 } 248 249 unsigned AMDGPURegisterBankInfo::getBreakDownCost( 250 const ValueMapping &ValMapping, 251 const RegisterBank *CurBank) const { 252 // Check if this is a breakdown for G_LOAD to move the pointer from SGPR to 253 // VGPR. 254 // FIXME: Is there a better way to do this? 255 if (ValMapping.NumBreakDowns >= 2 || ValMapping.BreakDown[0].Length >= 64) 256 return 10; // This is expensive. 257 258 assert(ValMapping.NumBreakDowns == 2 && 259 ValMapping.BreakDown[0].Length == 32 && 260 ValMapping.BreakDown[0].StartIdx == 0 && 261 ValMapping.BreakDown[1].Length == 32 && 262 ValMapping.BreakDown[1].StartIdx == 32 && 263 ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank); 264 265 // 32-bit extract of a 64-bit value is just access of a subregister, so free. 266 // TODO: Cost of 0 hits assert, though it's not clear it's what we really 267 // want. 268 269 // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR 270 // alignment restrictions, but this probably isn't important. 271 return 1; 272 } 273 274 const RegisterBank & 275 AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC, 276 LLT Ty) const { 277 if (&RC == &AMDGPU::SReg_1RegClass) 278 return AMDGPU::VCCRegBank; 279 280 // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a 281 // VCC-like use. 282 if (TRI->isSGPRClass(&RC)) { 283 // FIXME: This probably came from a copy from a physical register, which 284 // should be inferrrable from the copied to-type. We don't have many boolean 285 // physical register constraints so just assume a normal SGPR for now. 286 if (!Ty.isValid()) 287 return AMDGPU::SGPRRegBank; 288 289 return Ty == LLT::scalar(1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank; 290 } 291 292 return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank; 293 } 294 295 template <unsigned NumOps> 296 RegisterBankInfo::InstructionMappings 297 AMDGPURegisterBankInfo::addMappingFromTable( 298 const MachineInstr &MI, const MachineRegisterInfo &MRI, 299 const std::array<unsigned, NumOps> RegSrcOpIdx, 300 ArrayRef<OpRegBankEntry<NumOps>> Table) const { 301 302 InstructionMappings AltMappings; 303 304 SmallVector<const ValueMapping *, 10> Operands(MI.getNumOperands()); 305 306 unsigned Sizes[NumOps]; 307 for (unsigned I = 0; I < NumOps; ++I) { 308 Register Reg = MI.getOperand(RegSrcOpIdx[I]).getReg(); 309 Sizes[I] = getSizeInBits(Reg, MRI, *TRI); 310 } 311 312 for (unsigned I = 0, E = MI.getNumExplicitDefs(); I != E; ++I) { 313 unsigned SizeI = getSizeInBits(MI.getOperand(I).getReg(), MRI, *TRI); 314 Operands[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SizeI); 315 } 316 317 // getInstrMapping's default mapping uses ID 1, so start at 2. 318 unsigned MappingID = 2; 319 for (const auto &Entry : Table) { 320 for (unsigned I = 0; I < NumOps; ++I) { 321 int OpIdx = RegSrcOpIdx[I]; 322 Operands[OpIdx] = AMDGPU::getValueMapping(Entry.RegBanks[I], Sizes[I]); 323 } 324 325 AltMappings.push_back(&getInstructionMapping(MappingID++, Entry.Cost, 326 getOperandsMapping(Operands), 327 Operands.size())); 328 } 329 330 return AltMappings; 331 } 332 333 RegisterBankInfo::InstructionMappings 334 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic( 335 const MachineInstr &MI, const MachineRegisterInfo &MRI) const { 336 switch (MI.getIntrinsicID()) { 337 case Intrinsic::amdgcn_readlane: { 338 static const OpRegBankEntry<3> Table[2] = { 339 // Perfectly legal. 340 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, 341 342 // Need a readfirstlane for the index. 343 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 } 344 }; 345 346 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } }; 347 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 348 } 349 case Intrinsic::amdgcn_writelane: { 350 static const OpRegBankEntry<4> Table[4] = { 351 // Perfectly legal. 352 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, 353 354 // Need readfirstlane of first op 355 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }, 356 357 // Need readfirstlane of second op 358 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 }, 359 360 // Need readfirstlane of both ops 361 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 3 } 362 }; 363 364 // rsrc, voffset, offset 365 const std::array<unsigned, 4> RegSrcOpIdx = { { 0, 2, 3, 4 } }; 366 return addMappingFromTable<4>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 367 } 368 default: 369 return RegisterBankInfo::getInstrAlternativeMappings(MI); 370 } 371 } 372 373 RegisterBankInfo::InstructionMappings 374 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects( 375 const MachineInstr &MI, const MachineRegisterInfo &MRI) const { 376 377 switch (MI.getIntrinsicID()) { 378 case Intrinsic::amdgcn_s_buffer_load: { 379 static const OpRegBankEntry<2> Table[4] = { 380 // Perfectly legal. 381 { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 1 }, 382 383 // Only need 1 register in loop 384 { { AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 300 }, 385 386 // Have to waterfall the resource. 387 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID }, 1000 }, 388 389 // Have to waterfall the resource, and the offset. 390 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1500 } 391 }; 392 393 // rsrc, offset 394 const std::array<unsigned, 2> RegSrcOpIdx = { { 2, 3 } }; 395 return addMappingFromTable<2>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 396 } 397 case Intrinsic::amdgcn_ds_ordered_add: 398 case Intrinsic::amdgcn_ds_ordered_swap: { 399 // VGPR = M0, VGPR 400 static const OpRegBankEntry<3> Table[2] = { 401 // Perfectly legal. 402 { { AMDGPU::VGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, 403 404 // Need a readfirstlane for m0 405 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 2 } 406 }; 407 408 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 2, 3 } }; 409 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 410 } 411 case Intrinsic::amdgcn_s_sendmsg: 412 case Intrinsic::amdgcn_s_sendmsghalt: { 413 // FIXME: Should have no register for immediate 414 static const OpRegBankEntry<1> Table[2] = { 415 // Perfectly legal. 416 { { AMDGPU::SGPRRegBankID }, 1 }, 417 418 // Need readlane 419 { { AMDGPU::VGPRRegBankID }, 3 } 420 }; 421 422 const std::array<unsigned, 1> RegSrcOpIdx = { { 2 } }; 423 return addMappingFromTable<1>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 424 } 425 default: 426 return RegisterBankInfo::getInstrAlternativeMappings(MI); 427 } 428 } 429 430 static bool memOpHasNoClobbered(const MachineMemOperand *MMO) { 431 const Instruction *I = dyn_cast_or_null<Instruction>(MMO->getValue()); 432 return I && I->getMetadata("amdgpu.noclobber"); 433 } 434 435 // FIXME: Returns uniform if there's no source value information. This is 436 // probably wrong. 437 static bool isScalarLoadLegal(const MachineInstr &MI) { 438 if (!MI.hasOneMemOperand()) 439 return false; 440 441 const MachineMemOperand *MMO = *MI.memoperands_begin(); 442 const unsigned AS = MMO->getAddrSpace(); 443 const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS || 444 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 445 446 // There are no extending SMRD/SMEM loads, and they require 4-byte alignment. 447 return MMO->getSize() >= 4 && MMO->getAlign() >= Align(4) && 448 // Can't do a scalar atomic load. 449 !MMO->isAtomic() && 450 // Don't use scalar loads for volatile accesses to non-constant address 451 // spaces. 452 (IsConst || !MMO->isVolatile()) && 453 // Memory must be known constant, or not written before this load. 454 (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) && 455 AMDGPUInstrInfo::isUniformMMO(MMO); 456 } 457 458 RegisterBankInfo::InstructionMappings 459 AMDGPURegisterBankInfo::getInstrAlternativeMappings( 460 const MachineInstr &MI) const { 461 462 const MachineFunction &MF = *MI.getParent()->getParent(); 463 const MachineRegisterInfo &MRI = MF.getRegInfo(); 464 465 466 InstructionMappings AltMappings; 467 switch (MI.getOpcode()) { 468 case TargetOpcode::G_CONSTANT: { 469 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 470 if (Size == 1) { 471 static const OpRegBankEntry<1> Table[3] = { 472 { { AMDGPU::VGPRRegBankID }, 1 }, 473 { { AMDGPU::SGPRRegBankID }, 1 }, 474 { { AMDGPU::VCCRegBankID }, 1 } 475 }; 476 477 return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table); 478 } 479 480 LLVM_FALLTHROUGH; 481 } 482 case TargetOpcode::G_FCONSTANT: 483 case TargetOpcode::G_FRAME_INDEX: 484 case TargetOpcode::G_GLOBAL_VALUE: { 485 static const OpRegBankEntry<1> Table[2] = { 486 { { AMDGPU::VGPRRegBankID }, 1 }, 487 { { AMDGPU::SGPRRegBankID }, 1 } 488 }; 489 490 return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table); 491 } 492 case TargetOpcode::G_AND: 493 case TargetOpcode::G_OR: 494 case TargetOpcode::G_XOR: { 495 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 496 497 if (Size == 1) { 498 // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0. 499 const InstructionMapping &SCCMapping = getInstructionMapping( 500 1, 1, getOperandsMapping( 501 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), 502 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), 503 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32)}), 504 3); // Num Operands 505 AltMappings.push_back(&SCCMapping); 506 507 const InstructionMapping &VCCMapping0 = getInstructionMapping( 508 2, 1, getOperandsMapping( 509 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), 510 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), 511 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}), 512 3); // Num Operands 513 AltMappings.push_back(&VCCMapping0); 514 return AltMappings; 515 } 516 517 if (Size != 64) 518 break; 519 520 const InstructionMapping &SSMapping = getInstructionMapping( 521 1, 1, getOperandsMapping( 522 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 523 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 524 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), 525 3); // Num Operands 526 AltMappings.push_back(&SSMapping); 527 528 const InstructionMapping &VVMapping = getInstructionMapping( 529 2, 2, getOperandsMapping( 530 {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), 531 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), 532 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), 533 3); // Num Operands 534 AltMappings.push_back(&VVMapping); 535 break; 536 } 537 case TargetOpcode::G_LOAD: 538 case TargetOpcode::G_ZEXTLOAD: 539 case TargetOpcode::G_SEXTLOAD: { 540 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 541 LLT PtrTy = MRI.getType(MI.getOperand(1).getReg()); 542 unsigned PtrSize = PtrTy.getSizeInBits(); 543 unsigned AS = PtrTy.getAddressSpace(); 544 545 if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS && 546 AS != AMDGPUAS::PRIVATE_ADDRESS) && 547 isScalarLoadLegal(MI)) { 548 const InstructionMapping &SSMapping = getInstructionMapping( 549 1, 1, getOperandsMapping( 550 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 551 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize)}), 552 2); // Num Operands 553 AltMappings.push_back(&SSMapping); 554 } 555 556 const InstructionMapping &VVMapping = getInstructionMapping( 557 2, 1, 558 getOperandsMapping( 559 {AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), 560 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize)}), 561 2); // Num Operands 562 AltMappings.push_back(&VVMapping); 563 564 // It may be possible to have a vgpr = load sgpr mapping here, because 565 // the mubuf instructions support this kind of load, but probably for only 566 // gfx7 and older. However, the addressing mode matching in the instruction 567 // selector should be able to do a better job of detecting and selecting 568 // these kinds of loads from the vgpr = load vgpr mapping. 569 570 return AltMappings; 571 572 } 573 case TargetOpcode::G_SELECT: { 574 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 575 const InstructionMapping &SSMapping = getInstructionMapping(1, 1, 576 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 577 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), 578 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 579 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), 580 4); // Num Operands 581 AltMappings.push_back(&SSMapping); 582 583 const InstructionMapping &VVMapping = getInstructionMapping(2, 1, 584 getOperandsMapping({AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), 585 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), 586 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size), 587 AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}), 588 4); // Num Operands 589 AltMappings.push_back(&VVMapping); 590 591 return AltMappings; 592 } 593 case TargetOpcode::G_SMIN: 594 case TargetOpcode::G_SMAX: 595 case TargetOpcode::G_UMIN: 596 case TargetOpcode::G_UMAX: { 597 static const OpRegBankEntry<3> Table[2] = { 598 { { AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID, AMDGPU::VGPRRegBankID }, 1 }, 599 600 // Scalar requires cmp+select, and extends if 16-bit. 601 // FIXME: Should there be separate costs for 32 and 16-bit 602 { { AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID, AMDGPU::SGPRRegBankID }, 3 } 603 }; 604 605 const std::array<unsigned, 3> RegSrcOpIdx = { { 0, 1, 2 } }; 606 return addMappingFromTable<3>(MI, MRI, RegSrcOpIdx, makeArrayRef(Table)); 607 } 608 case TargetOpcode::G_UADDE: 609 case TargetOpcode::G_USUBE: 610 case TargetOpcode::G_SADDE: 611 case TargetOpcode::G_SSUBE: { 612 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 613 const InstructionMapping &SSMapping = getInstructionMapping(1, 1, 614 getOperandsMapping( 615 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 616 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), 617 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 618 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), 619 AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1)}), 620 5); // Num Operands 621 AltMappings.push_back(&SSMapping); 622 623 const InstructionMapping &VVMapping = getInstructionMapping(2, 1, 624 getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), 625 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), 626 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), 627 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size), 628 AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1)}), 629 5); // Num Operands 630 AltMappings.push_back(&VVMapping); 631 return AltMappings; 632 } 633 case AMDGPU::G_BRCOND: { 634 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1); 635 636 // TODO: Change type to 32 for scalar 637 const InstructionMapping &SMapping = getInstructionMapping( 638 1, 1, getOperandsMapping( 639 {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr}), 640 2); // Num Operands 641 AltMappings.push_back(&SMapping); 642 643 const InstructionMapping &VMapping = getInstructionMapping( 644 1, 1, getOperandsMapping( 645 {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), nullptr }), 646 2); // Num Operands 647 AltMappings.push_back(&VMapping); 648 return AltMappings; 649 } 650 case AMDGPU::G_INTRINSIC: 651 return getInstrAlternativeMappingsIntrinsic(MI, MRI); 652 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: 653 return getInstrAlternativeMappingsIntrinsicWSideEffects(MI, MRI); 654 default: 655 break; 656 } 657 return RegisterBankInfo::getInstrAlternativeMappings(MI); 658 } 659 660 void AMDGPURegisterBankInfo::split64BitValueForMapping( 661 MachineIRBuilder &B, 662 SmallVector<Register, 2> &Regs, 663 LLT HalfTy, 664 Register Reg) const { 665 assert(HalfTy.getSizeInBits() == 32); 666 MachineRegisterInfo *MRI = B.getMRI(); 667 Register LoLHS = MRI->createGenericVirtualRegister(HalfTy); 668 Register HiLHS = MRI->createGenericVirtualRegister(HalfTy); 669 const RegisterBank *Bank = getRegBank(Reg, *MRI, *TRI); 670 MRI->setRegBank(LoLHS, *Bank); 671 MRI->setRegBank(HiLHS, *Bank); 672 673 Regs.push_back(LoLHS); 674 Regs.push_back(HiLHS); 675 676 B.buildInstr(AMDGPU::G_UNMERGE_VALUES) 677 .addDef(LoLHS) 678 .addDef(HiLHS) 679 .addUse(Reg); 680 } 681 682 /// Replace the current type each register in \p Regs has with \p NewTy 683 static void setRegsToType(MachineRegisterInfo &MRI, ArrayRef<Register> Regs, 684 LLT NewTy) { 685 for (Register Reg : Regs) { 686 assert(MRI.getType(Reg).getSizeInBits() == NewTy.getSizeInBits()); 687 MRI.setType(Reg, NewTy); 688 } 689 } 690 691 static LLT getHalfSizedType(LLT Ty) { 692 if (Ty.isVector()) { 693 assert(Ty.getNumElements() % 2 == 0); 694 return LLT::scalarOrVector(Ty.getNumElements() / 2, Ty.getElementType()); 695 } 696 697 assert(Ty.getSizeInBits() % 2 == 0); 698 return LLT::scalar(Ty.getSizeInBits() / 2); 699 } 700 701 /// Legalize instruction \p MI where operands in \p OpIndices must be SGPRs. If 702 /// any of the required SGPR operands are VGPRs, perform a waterfall loop to 703 /// execute the instruction for each unique combination of values in all lanes 704 /// in the wave. The block will be split such that rest of the instructions are 705 /// moved to a new block. 706 /// 707 /// Essentially performs this loop: 708 // 709 /// Save Execution Mask 710 /// For (Lane : Wavefront) { 711 /// Enable Lane, Disable all other lanes 712 /// SGPR = read SGPR value for current lane from VGPR 713 /// VGPRResult[Lane] = use_op SGPR 714 /// } 715 /// Restore Execution Mask 716 /// 717 /// There is additional complexity to try for compare values to identify the 718 /// unique values used. 719 bool AMDGPURegisterBankInfo::executeInWaterfallLoop( 720 MachineIRBuilder &B, 721 iterator_range<MachineBasicBlock::iterator> Range, 722 SmallSet<Register, 4> &SGPROperandRegs, 723 MachineRegisterInfo &MRI) const { 724 SmallVector<Register, 4> ResultRegs; 725 SmallVector<Register, 4> InitResultRegs; 726 SmallVector<Register, 4> PhiRegs; 727 728 // Track use registers which have already been expanded with a readfirstlane 729 // sequence. This may have multiple uses if moving a sequence. 730 DenseMap<Register, Register> WaterfalledRegMap; 731 732 MachineBasicBlock &MBB = B.getMBB(); 733 MachineFunction *MF = &B.getMF(); 734 735 const TargetRegisterClass *WaveRC = TRI->getWaveMaskRegClass(); 736 const unsigned WaveAndOpc = Subtarget.isWave32() ? 737 AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 738 const unsigned MovTermOpc = Subtarget.isWave32() ? 739 AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term; 740 const unsigned XorTermOpc = Subtarget.isWave32() ? 741 AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 742 const unsigned AndSaveExecOpc = Subtarget.isWave32() ? 743 AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 744 const unsigned ExecReg = Subtarget.isWave32() ? 745 AMDGPU::EXEC_LO : AMDGPU::EXEC; 746 747 #ifndef NDEBUG 748 const int OrigRangeSize = std::distance(Range.begin(), Range.end()); 749 #endif 750 751 for (MachineInstr &MI : Range) { 752 for (MachineOperand &Def : MI.defs()) { 753 LLT ResTy = MRI.getType(Def.getReg()); 754 const RegisterBank *DefBank = getRegBank(Def.getReg(), MRI, *TRI); 755 ResultRegs.push_back(Def.getReg()); 756 Register InitReg = B.buildUndef(ResTy).getReg(0); 757 Register PhiReg = MRI.createGenericVirtualRegister(ResTy); 758 InitResultRegs.push_back(InitReg); 759 PhiRegs.push_back(PhiReg); 760 MRI.setRegBank(PhiReg, *DefBank); 761 MRI.setRegBank(InitReg, *DefBank); 762 } 763 } 764 765 Register SaveExecReg = MRI.createVirtualRegister(WaveRC); 766 Register InitSaveExecReg = MRI.createVirtualRegister(WaveRC); 767 768 // Don't bother using generic instructions/registers for the exec mask. 769 B.buildInstr(TargetOpcode::IMPLICIT_DEF) 770 .addDef(InitSaveExecReg); 771 772 Register PhiExec = MRI.createVirtualRegister(WaveRC); 773 Register NewExec = MRI.createVirtualRegister(WaveRC); 774 775 // To insert the loop we need to split the block. Move everything before this 776 // point to a new block, and insert a new empty block before this instruction. 777 MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock(); 778 MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock(); 779 MachineBasicBlock *RestoreExecBB = MF->CreateMachineBasicBlock(); 780 MachineFunction::iterator MBBI(MBB); 781 ++MBBI; 782 MF->insert(MBBI, LoopBB); 783 MF->insert(MBBI, RestoreExecBB); 784 MF->insert(MBBI, RemainderBB); 785 786 LoopBB->addSuccessor(RestoreExecBB); 787 LoopBB->addSuccessor(LoopBB); 788 789 // Move the rest of the block into a new block. 790 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 791 RemainderBB->splice(RemainderBB->begin(), &MBB, Range.end(), MBB.end()); 792 793 MBB.addSuccessor(LoopBB); 794 RestoreExecBB->addSuccessor(RemainderBB); 795 796 B.setInsertPt(*LoopBB, LoopBB->end()); 797 798 B.buildInstr(TargetOpcode::PHI) 799 .addDef(PhiExec) 800 .addReg(InitSaveExecReg) 801 .addMBB(&MBB) 802 .addReg(NewExec) 803 .addMBB(LoopBB); 804 805 for (auto Result : zip(InitResultRegs, ResultRegs, PhiRegs)) { 806 B.buildInstr(TargetOpcode::G_PHI) 807 .addDef(std::get<2>(Result)) 808 .addReg(std::get<0>(Result)) // Initial value / implicit_def 809 .addMBB(&MBB) 810 .addReg(std::get<1>(Result)) // Mid-loop value. 811 .addMBB(LoopBB); 812 } 813 814 const DebugLoc &DL = B.getDL(); 815 816 MachineInstr &FirstInst = *Range.begin(); 817 818 // Move the instruction into the loop. Note we moved everything after 819 // Range.end() already into a new block, so Range.end() is no longer valid. 820 LoopBB->splice(LoopBB->end(), &MBB, Range.begin(), MBB.end()); 821 822 // Figure out the iterator range after splicing the instructions. 823 MachineBasicBlock::iterator NewBegin = FirstInst.getIterator(); 824 auto NewEnd = LoopBB->end(); 825 826 MachineBasicBlock::iterator I = Range.begin(); 827 B.setInsertPt(*LoopBB, I); 828 829 Register CondReg; 830 831 assert(std::distance(NewBegin, NewEnd) == OrigRangeSize); 832 833 for (MachineInstr &MI : make_range(NewBegin, NewEnd)) { 834 for (MachineOperand &Op : MI.uses()) { 835 if (!Op.isReg() || Op.isDef()) 836 continue; 837 838 Register OldReg = Op.getReg(); 839 if (!SGPROperandRegs.count(OldReg)) 840 continue; 841 842 // See if we already processed this register in another instruction in the 843 // sequence. 844 auto OldVal = WaterfalledRegMap.find(OldReg); 845 if (OldVal != WaterfalledRegMap.end()) { 846 Op.setReg(OldVal->second); 847 continue; 848 } 849 850 Register OpReg = Op.getReg(); 851 LLT OpTy = MRI.getType(OpReg); 852 853 const RegisterBank *OpBank = getRegBank(OpReg, MRI, *TRI); 854 if (OpBank != &AMDGPU::VGPRRegBank) { 855 // Insert copy from AGPR to VGPR before the loop. 856 B.setMBB(MBB); 857 OpReg = B.buildCopy(OpTy, OpReg).getReg(0); 858 MRI.setRegBank(OpReg, AMDGPU::VGPRRegBank); 859 B.setInstr(*I); 860 } 861 862 unsigned OpSize = OpTy.getSizeInBits(); 863 864 // Can only do a readlane of 32-bit pieces. 865 if (OpSize == 32) { 866 // Avoid extra copies in the simple case of one 32-bit register. 867 Register CurrentLaneOpReg 868 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 869 MRI.setType(CurrentLaneOpReg, OpTy); 870 871 constrainGenericRegister(OpReg, AMDGPU::VGPR_32RegClass, MRI); 872 // Read the next variant <- also loop target. 873 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 874 CurrentLaneOpReg) 875 .addReg(OpReg); 876 877 Register NewCondReg = MRI.createVirtualRegister(WaveRC); 878 bool First = CondReg == AMDGPU::NoRegister; 879 if (First) 880 CondReg = NewCondReg; 881 882 // Compare the just read M0 value to all possible Idx values. 883 B.buildInstr(AMDGPU::V_CMP_EQ_U32_e64) 884 .addDef(NewCondReg) 885 .addReg(CurrentLaneOpReg) 886 .addReg(OpReg); 887 Op.setReg(CurrentLaneOpReg); 888 889 if (!First) { 890 Register AndReg = MRI.createVirtualRegister(WaveRC); 891 892 // If there are multiple operands to consider, and the conditions. 893 B.buildInstr(WaveAndOpc) 894 .addDef(AndReg) 895 .addReg(NewCondReg) 896 .addReg(CondReg); 897 CondReg = AndReg; 898 } 899 } else { 900 LLT S32 = LLT::scalar(32); 901 SmallVector<Register, 8> ReadlanePieces; 902 903 // The compares can be done as 64-bit, but the extract needs to be done 904 // in 32-bit pieces. 905 906 bool Is64 = OpSize % 64 == 0; 907 908 LLT UnmergeTy = OpSize % 64 == 0 ? LLT::scalar(64) : LLT::scalar(32); 909 unsigned CmpOp = OpSize % 64 == 0 ? AMDGPU::V_CMP_EQ_U64_e64 910 : AMDGPU::V_CMP_EQ_U32_e64; 911 912 // The compares can be done as 64-bit, but the extract needs to be done 913 // in 32-bit pieces. 914 915 // Insert the unmerge before the loop. 916 917 B.setMBB(MBB); 918 auto Unmerge = B.buildUnmerge(UnmergeTy, OpReg); 919 B.setInstr(*I); 920 921 unsigned NumPieces = Unmerge->getNumOperands() - 1; 922 for (unsigned PieceIdx = 0; PieceIdx != NumPieces; ++PieceIdx) { 923 Register UnmergePiece = Unmerge.getReg(PieceIdx); 924 925 Register CurrentLaneOpReg; 926 if (Is64) { 927 Register CurrentLaneOpRegLo = MRI.createGenericVirtualRegister(S32); 928 Register CurrentLaneOpRegHi = MRI.createGenericVirtualRegister(S32); 929 930 MRI.setRegClass(UnmergePiece, &AMDGPU::VReg_64RegClass); 931 MRI.setRegClass(CurrentLaneOpRegLo, &AMDGPU::SReg_32_XM0RegClass); 932 MRI.setRegClass(CurrentLaneOpRegHi, &AMDGPU::SReg_32_XM0RegClass); 933 934 // Read the next variant <- also loop target. 935 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 936 CurrentLaneOpRegLo) 937 .addReg(UnmergePiece, 0, AMDGPU::sub0); 938 939 // Read the next variant <- also loop target. 940 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 941 CurrentLaneOpRegHi) 942 .addReg(UnmergePiece, 0, AMDGPU::sub1); 943 944 CurrentLaneOpReg = 945 B.buildMerge(LLT::scalar(64), 946 {CurrentLaneOpRegLo, CurrentLaneOpRegHi}) 947 .getReg(0); 948 949 MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_64_XEXECRegClass); 950 951 if (OpTy.getScalarSizeInBits() == 64) { 952 // If we need to produce a 64-bit element vector, so use the 953 // merged pieces 954 ReadlanePieces.push_back(CurrentLaneOpReg); 955 } else { 956 // 32-bit element type. 957 ReadlanePieces.push_back(CurrentLaneOpRegLo); 958 ReadlanePieces.push_back(CurrentLaneOpRegHi); 959 } 960 } else { 961 CurrentLaneOpReg = MRI.createGenericVirtualRegister(S32); 962 MRI.setRegClass(UnmergePiece, &AMDGPU::VGPR_32RegClass); 963 MRI.setRegClass(CurrentLaneOpReg, &AMDGPU::SReg_32_XM0RegClass); 964 965 // Read the next variant <- also loop target. 966 BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), 967 CurrentLaneOpReg) 968 .addReg(UnmergePiece); 969 ReadlanePieces.push_back(CurrentLaneOpReg); 970 } 971 972 Register NewCondReg = MRI.createVirtualRegister(WaveRC); 973 bool First = CondReg == AMDGPU::NoRegister; 974 if (First) 975 CondReg = NewCondReg; 976 977 B.buildInstr(CmpOp) 978 .addDef(NewCondReg) 979 .addReg(CurrentLaneOpReg) 980 .addReg(UnmergePiece); 981 982 if (!First) { 983 Register AndReg = MRI.createVirtualRegister(WaveRC); 984 985 // If there are multiple operands to consider, and the conditions. 986 B.buildInstr(WaveAndOpc) 987 .addDef(AndReg) 988 .addReg(NewCondReg) 989 .addReg(CondReg); 990 CondReg = AndReg; 991 } 992 } 993 994 // FIXME: Build merge seems to switch to CONCAT_VECTORS but not 995 // BUILD_VECTOR 996 if (OpTy.isVector()) { 997 auto Merge = B.buildBuildVector(OpTy, ReadlanePieces); 998 Op.setReg(Merge.getReg(0)); 999 } else { 1000 auto Merge = B.buildMerge(OpTy, ReadlanePieces); 1001 Op.setReg(Merge.getReg(0)); 1002 } 1003 1004 MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank); 1005 } 1006 1007 // Make sure we don't re-process this register again. 1008 WaterfalledRegMap.insert(std::make_pair(OldReg, Op.getReg())); 1009 } 1010 } 1011 1012 B.setInsertPt(*LoopBB, LoopBB->end()); 1013 1014 // Update EXEC, save the original EXEC value to VCC. 1015 B.buildInstr(AndSaveExecOpc) 1016 .addDef(NewExec) 1017 .addReg(CondReg, RegState::Kill); 1018 1019 MRI.setSimpleHint(NewExec, CondReg); 1020 1021 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 1022 B.buildInstr(XorTermOpc) 1023 .addDef(ExecReg) 1024 .addReg(ExecReg) 1025 .addReg(NewExec); 1026 1027 // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use 1028 // s_cbranch_scc0? 1029 1030 // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover. 1031 B.buildInstr(AMDGPU::S_CBRANCH_EXECNZ) 1032 .addMBB(LoopBB); 1033 1034 // Save the EXEC mask before the loop. 1035 BuildMI(MBB, MBB.end(), DL, TII->get(MovTermOpc), SaveExecReg) 1036 .addReg(ExecReg); 1037 1038 // Restore the EXEC mask after the loop. 1039 B.setMBB(*RestoreExecBB); 1040 B.buildInstr(MovTermOpc) 1041 .addDef(ExecReg) 1042 .addReg(SaveExecReg); 1043 1044 // Set the insert point after the original instruction, so any new 1045 // instructions will be in the remainder. 1046 B.setInsertPt(*RemainderBB, RemainderBB->begin()); 1047 1048 return true; 1049 } 1050 1051 // Return any unique registers used by \p MI at \p OpIndices that need to be 1052 // handled in a waterfall loop. Returns these registers in \p 1053 // SGPROperandRegs. Returns true if there are any operands to handle and a 1054 // waterfall loop is necessary. 1055 bool AMDGPURegisterBankInfo::collectWaterfallOperands( 1056 SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI, 1057 MachineRegisterInfo &MRI, ArrayRef<unsigned> OpIndices) const { 1058 for (unsigned Op : OpIndices) { 1059 assert(MI.getOperand(Op).isUse()); 1060 Register Reg = MI.getOperand(Op).getReg(); 1061 const RegisterBank *OpBank = getRegBank(Reg, MRI, *TRI); 1062 if (OpBank->getID() != AMDGPU::SGPRRegBankID) 1063 SGPROperandRegs.insert(Reg); 1064 } 1065 1066 // No operands need to be replaced, so no need to loop. 1067 return !SGPROperandRegs.empty(); 1068 } 1069 1070 bool AMDGPURegisterBankInfo::executeInWaterfallLoop( 1071 MachineIRBuilder &B, MachineInstr &MI, MachineRegisterInfo &MRI, 1072 ArrayRef<unsigned> OpIndices) const { 1073 // Use a set to avoid extra readfirstlanes in the case where multiple operands 1074 // are the same register. 1075 SmallSet<Register, 4> SGPROperandRegs; 1076 1077 if (!collectWaterfallOperands(SGPROperandRegs, MI, MRI, OpIndices)) 1078 return false; 1079 1080 MachineBasicBlock::iterator I = MI.getIterator(); 1081 return executeInWaterfallLoop(B, make_range(I, std::next(I)), 1082 SGPROperandRegs, MRI); 1083 } 1084 1085 bool AMDGPURegisterBankInfo::executeInWaterfallLoop( 1086 MachineInstr &MI, MachineRegisterInfo &MRI, 1087 ArrayRef<unsigned> OpIndices) const { 1088 MachineIRBuilder B(MI); 1089 return executeInWaterfallLoop(B, MI, MRI, OpIndices); 1090 } 1091 1092 // Legalize an operand that must be an SGPR by inserting a readfirstlane. 1093 void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane( 1094 MachineInstr &MI, MachineRegisterInfo &MRI, unsigned OpIdx) const { 1095 Register Reg = MI.getOperand(OpIdx).getReg(); 1096 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI); 1097 if (Bank == &AMDGPU::SGPRRegBank) 1098 return; 1099 1100 LLT Ty = MRI.getType(Reg); 1101 MachineIRBuilder B(MI); 1102 1103 if (Bank != &AMDGPU::VGPRRegBank) { 1104 // We need to copy from AGPR to VGPR 1105 Reg = B.buildCopy(Ty, Reg).getReg(0); 1106 MRI.setRegBank(Reg, AMDGPU::VGPRRegBank); 1107 } 1108 1109 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 1110 B.buildInstr(AMDGPU::V_READFIRSTLANE_B32) 1111 .addDef(SGPR) 1112 .addReg(Reg); 1113 1114 MRI.setType(SGPR, Ty); 1115 1116 const TargetRegisterClass *Constrained = 1117 constrainGenericRegister(Reg, AMDGPU::VGPR_32RegClass, MRI); 1118 (void)Constrained; 1119 assert(Constrained && "Failed to constrain readfirstlane src reg"); 1120 1121 MI.getOperand(OpIdx).setReg(SGPR); 1122 } 1123 1124 /// Split \p Ty into 2 pieces. The first will have \p FirstSize bits, and the 1125 /// rest will be in the remainder. 1126 static std::pair<LLT, LLT> splitUnequalType(LLT Ty, unsigned FirstSize) { 1127 unsigned TotalSize = Ty.getSizeInBits(); 1128 if (!Ty.isVector()) 1129 return {LLT::scalar(FirstSize), LLT::scalar(TotalSize - FirstSize)}; 1130 1131 LLT EltTy = Ty.getElementType(); 1132 unsigned EltSize = EltTy.getSizeInBits(); 1133 assert(FirstSize % EltSize == 0); 1134 1135 unsigned FirstPartNumElts = FirstSize / EltSize; 1136 unsigned RemainderElts = (TotalSize - FirstSize) / EltSize; 1137 1138 return {LLT::scalarOrVector(FirstPartNumElts, EltTy), 1139 LLT::scalarOrVector(RemainderElts, EltTy)}; 1140 } 1141 1142 static LLT widen96To128(LLT Ty) { 1143 if (!Ty.isVector()) 1144 return LLT::scalar(128); 1145 1146 LLT EltTy = Ty.getElementType(); 1147 assert(128 % EltTy.getSizeInBits() == 0); 1148 return LLT::vector(128 / EltTy.getSizeInBits(), EltTy); 1149 } 1150 1151 bool AMDGPURegisterBankInfo::applyMappingLoad(MachineInstr &MI, 1152 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, 1153 MachineRegisterInfo &MRI) const { 1154 Register DstReg = MI.getOperand(0).getReg(); 1155 const LLT LoadTy = MRI.getType(DstReg); 1156 unsigned LoadSize = LoadTy.getSizeInBits(); 1157 const unsigned MaxNonSmrdLoadSize = 128; 1158 1159 const RegisterBank *PtrBank = 1160 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 1161 if (PtrBank == &AMDGPU::SGPRRegBank) { 1162 // If the pointer is an SGPR, we ordinarily have nothing to do. 1163 if (LoadSize != 96) 1164 return false; 1165 1166 MachineMemOperand *MMO = *MI.memoperands_begin(); 1167 Register PtrReg = MI.getOperand(1).getReg(); 1168 // 96-bit loads are only available for vector loads. We need to split this 1169 // into a 64-bit part, and 32 (unless we can widen to a 128-bit load). 1170 1171 MachineIRBuilder B(MI); 1172 ApplyRegBankMapping O(*this, MRI, &AMDGPU::SGPRRegBank); 1173 GISelObserverWrapper Observer(&O); 1174 B.setChangeObserver(Observer); 1175 1176 if (MMO->getAlign() < Align(16)) { 1177 LLT Part64, Part32; 1178 std::tie(Part64, Part32) = splitUnequalType(LoadTy, 64); 1179 auto Load0 = B.buildLoadFromOffset(Part64, PtrReg, *MMO, 0); 1180 auto Load1 = B.buildLoadFromOffset(Part32, PtrReg, *MMO, 8); 1181 1182 auto Undef = B.buildUndef(LoadTy); 1183 auto Ins0 = B.buildInsert(LoadTy, Undef, Load0, 0); 1184 B.buildInsert(MI.getOperand(0), Ins0, Load1, 64); 1185 } else { 1186 LLT WiderTy = widen96To128(LoadTy); 1187 auto WideLoad = B.buildLoadFromOffset(WiderTy, PtrReg, *MMO, 0); 1188 B.buildExtract(MI.getOperand(0), WideLoad, 0); 1189 } 1190 1191 MI.eraseFromParent(); 1192 return true; 1193 } 1194 1195 // 128-bit loads are supported for all instruction types. 1196 if (LoadSize <= MaxNonSmrdLoadSize) 1197 return false; 1198 1199 SmallVector<Register, 16> DefRegs(OpdMapper.getVRegs(0)); 1200 SmallVector<Register, 1> SrcRegs(OpdMapper.getVRegs(1)); 1201 1202 if (SrcRegs.empty()) 1203 SrcRegs.push_back(MI.getOperand(1).getReg()); 1204 1205 assert(LoadSize % MaxNonSmrdLoadSize == 0); 1206 1207 // RegBankSelect only emits scalar types, so we need to reset the pointer 1208 // operand to a pointer type. 1209 Register BasePtrReg = SrcRegs[0]; 1210 LLT PtrTy = MRI.getType(MI.getOperand(1).getReg()); 1211 MRI.setType(BasePtrReg, PtrTy); 1212 1213 MachineIRBuilder B(MI); 1214 1215 unsigned NumSplitParts = LoadTy.getSizeInBits() / MaxNonSmrdLoadSize; 1216 const LLT LoadSplitTy = LoadTy.divide(NumSplitParts); 1217 ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank); 1218 GISelObserverWrapper Observer(&O); 1219 B.setChangeObserver(Observer); 1220 LegalizerHelper Helper(B.getMF(), Observer, B); 1221 1222 if (LoadTy.isVector()) { 1223 if (Helper.fewerElementsVector(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized) 1224 return false; 1225 } else { 1226 if (Helper.narrowScalar(MI, 0, LoadSplitTy) != LegalizerHelper::Legalized) 1227 return false; 1228 } 1229 1230 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); 1231 return true; 1232 } 1233 1234 bool AMDGPURegisterBankInfo::applyMappingDynStackAlloc( 1235 MachineInstr &MI, 1236 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, 1237 MachineRegisterInfo &MRI) const { 1238 const MachineFunction &MF = *MI.getMF(); 1239 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 1240 const auto &TFI = *ST.getFrameLowering(); 1241 1242 // Guard in case the stack growth direction ever changes with scratch 1243 // instructions. 1244 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown) 1245 return false; 1246 1247 Register Dst = MI.getOperand(0).getReg(); 1248 Register AllocSize = MI.getOperand(1).getReg(); 1249 Align Alignment = assumeAligned(MI.getOperand(2).getImm()); 1250 1251 const RegisterBank *SizeBank = getRegBank(AllocSize, MRI, *TRI); 1252 1253 // TODO: Need to emit a wave reduction to get the maximum size. 1254 if (SizeBank != &AMDGPU::SGPRRegBank) 1255 return false; 1256 1257 LLT PtrTy = MRI.getType(Dst); 1258 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 1259 1260 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1261 Register SPReg = Info->getStackPtrOffsetReg(); 1262 ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank); 1263 GISelObserverWrapper Observer(&ApplyBank); 1264 1265 MachineIRBuilder B(MI); 1266 B.setChangeObserver(Observer); 1267 1268 auto WaveSize = B.buildConstant(LLT::scalar(32), ST.getWavefrontSizeLog2()); 1269 auto ScaledSize = B.buildShl(IntPtrTy, AllocSize, WaveSize); 1270 1271 auto SPCopy = B.buildCopy(PtrTy, SPReg); 1272 if (Alignment > TFI.getStackAlign()) { 1273 auto PtrAdd = B.buildPtrAdd(PtrTy, SPCopy, ScaledSize); 1274 B.buildMaskLowPtrBits(Dst, PtrAdd, 1275 Log2(Alignment) + ST.getWavefrontSizeLog2()); 1276 } else { 1277 B.buildPtrAdd(Dst, SPCopy, ScaledSize); 1278 } 1279 1280 MI.eraseFromParent(); 1281 return true; 1282 } 1283 1284 bool AMDGPURegisterBankInfo::applyMappingImage( 1285 MachineInstr &MI, const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, 1286 MachineRegisterInfo &MRI, int RsrcIdx) const { 1287 const int NumDefs = MI.getNumExplicitDefs(); 1288 1289 // The reported argument index is relative to the IR intrinsic call arguments, 1290 // so we need to shift by the number of defs and the intrinsic ID. 1291 RsrcIdx += NumDefs + 1; 1292 1293 // Insert copies to VGPR arguments. 1294 applyDefaultMapping(OpdMapper); 1295 1296 // Fixup any SGPR arguments. 1297 SmallVector<unsigned, 4> SGPRIndexes; 1298 for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) { 1299 if (!MI.getOperand(I).isReg()) 1300 continue; 1301 1302 // If this intrinsic has a sampler, it immediately follows rsrc. 1303 if (I == RsrcIdx || I == RsrcIdx + 1) 1304 SGPRIndexes.push_back(I); 1305 } 1306 1307 executeInWaterfallLoop(MI, MRI, SGPRIndexes); 1308 return true; 1309 } 1310 1311 static Register getSrcRegIgnoringCopies(const MachineRegisterInfo &MRI, 1312 Register Reg) { 1313 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); 1314 if (!Def) 1315 return Reg; 1316 1317 // TODO: Guard against this being an implicit def 1318 return Def->getOperand(0).getReg(); 1319 } 1320 1321 // Analyze a combined offset from an llvm.amdgcn.s.buffer intrinsic and store 1322 // the three offsets (voffset, soffset and instoffset) 1323 static unsigned setBufferOffsets(MachineIRBuilder &B, 1324 const AMDGPURegisterBankInfo &RBI, 1325 Register CombinedOffset, Register &VOffsetReg, 1326 Register &SOffsetReg, int64_t &InstOffsetVal, 1327 Align Alignment) { 1328 const LLT S32 = LLT::scalar(32); 1329 MachineRegisterInfo *MRI = B.getMRI(); 1330 1331 if (Optional<int64_t> Imm = getConstantVRegVal(CombinedOffset, *MRI)) { 1332 uint32_t SOffset, ImmOffset; 1333 if (AMDGPU::splitMUBUFOffset(*Imm, SOffset, ImmOffset, &RBI.Subtarget, 1334 Alignment)) { 1335 VOffsetReg = B.buildConstant(S32, 0).getReg(0); 1336 SOffsetReg = B.buildConstant(S32, SOffset).getReg(0); 1337 InstOffsetVal = ImmOffset; 1338 1339 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); 1340 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); 1341 return SOffset + ImmOffset; 1342 } 1343 } 1344 1345 Register Base; 1346 unsigned Offset; 1347 MachineInstr *Unused; 1348 1349 std::tie(Base, Offset, Unused) 1350 = AMDGPU::getBaseWithConstantOffset(*MRI, CombinedOffset); 1351 1352 uint32_t SOffset, ImmOffset; 1353 if (Offset > 0 && AMDGPU::splitMUBUFOffset(Offset, SOffset, ImmOffset, 1354 &RBI.Subtarget, Alignment)) { 1355 if (RBI.getRegBank(Base, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) { 1356 VOffsetReg = Base; 1357 SOffsetReg = B.buildConstant(S32, SOffset).getReg(0); 1358 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); 1359 InstOffsetVal = ImmOffset; 1360 return 0; // XXX - Why is this 0? 1361 } 1362 1363 // If we have SGPR base, we can use it for soffset. 1364 if (SOffset == 0) { 1365 VOffsetReg = B.buildConstant(S32, 0).getReg(0); 1366 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); 1367 SOffsetReg = Base; 1368 InstOffsetVal = ImmOffset; 1369 return 0; // XXX - Why is this 0? 1370 } 1371 } 1372 1373 // Handle the variable sgpr + vgpr case. 1374 if (MachineInstr *Add = getOpcodeDef(AMDGPU::G_ADD, CombinedOffset, *MRI)) { 1375 Register Src0 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(1).getReg()); 1376 Register Src1 = getSrcRegIgnoringCopies(*MRI, Add->getOperand(2).getReg()); 1377 1378 const RegisterBank *Src0Bank = RBI.getRegBank(Src0, *MRI, *RBI.TRI); 1379 const RegisterBank *Src1Bank = RBI.getRegBank(Src1, *MRI, *RBI.TRI); 1380 1381 if (Src0Bank == &AMDGPU::VGPRRegBank && Src1Bank == &AMDGPU::SGPRRegBank) { 1382 VOffsetReg = Src0; 1383 SOffsetReg = Src1; 1384 return 0; 1385 } 1386 1387 if (Src0Bank == &AMDGPU::SGPRRegBank && Src1Bank == &AMDGPU::VGPRRegBank) { 1388 VOffsetReg = Src1; 1389 SOffsetReg = Src0; 1390 return 0; 1391 } 1392 } 1393 1394 // Ensure we have a VGPR for the combined offset. This could be an issue if we 1395 // have an SGPR offset and a VGPR resource. 1396 if (RBI.getRegBank(CombinedOffset, *MRI, *RBI.TRI) == &AMDGPU::VGPRRegBank) { 1397 VOffsetReg = CombinedOffset; 1398 } else { 1399 VOffsetReg = B.buildCopy(S32, CombinedOffset).getReg(0); 1400 B.getMRI()->setRegBank(VOffsetReg, AMDGPU::VGPRRegBank); 1401 } 1402 1403 SOffsetReg = B.buildConstant(S32, 0).getReg(0); 1404 B.getMRI()->setRegBank(SOffsetReg, AMDGPU::SGPRRegBank); 1405 return 0; 1406 } 1407 1408 bool AMDGPURegisterBankInfo::applyMappingSBufferLoad( 1409 const OperandsMapper &OpdMapper) const { 1410 MachineInstr &MI = OpdMapper.getMI(); 1411 MachineRegisterInfo &MRI = OpdMapper.getMRI(); 1412 1413 const LLT S32 = LLT::scalar(32); 1414 Register Dst = MI.getOperand(0).getReg(); 1415 LLT Ty = MRI.getType(Dst); 1416 1417 const RegisterBank *RSrcBank = 1418 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 1419 const RegisterBank *OffsetBank = 1420 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 1421 if (RSrcBank == &AMDGPU::SGPRRegBank && 1422 OffsetBank == &AMDGPU::SGPRRegBank) 1423 return true; // Legal mapping 1424 1425 // FIXME: 96-bit case was widened during legalize. We neeed to narrow it back 1426 // here but don't have an MMO. 1427 1428 unsigned LoadSize = Ty.getSizeInBits(); 1429 int NumLoads = 1; 1430 if (LoadSize == 256 || LoadSize == 512) { 1431 NumLoads = LoadSize / 128; 1432 Ty = Ty.divide(NumLoads); 1433 } 1434 1435 // Use the alignment to ensure that the required offsets will fit into the 1436 // immediate offsets. 1437 const Align Alignment = NumLoads > 1 ? Align(16 * NumLoads) : Align(1); 1438 1439 MachineIRBuilder B(MI); 1440 MachineFunction &MF = B.getMF(); 1441 1442 Register SOffset; 1443 Register VOffset; 1444 int64_t ImmOffset = 0; 1445 1446 unsigned MMOOffset = setBufferOffsets(B, *this, MI.getOperand(2).getReg(), 1447 VOffset, SOffset, ImmOffset, Alignment); 1448 1449 // TODO: 96-bit loads were widened to 128-bit results. Shrink the result if we 1450 // can, but we neeed to track an MMO for that. 1451 const unsigned MemSize = (Ty.getSizeInBits() + 7) / 8; 1452 const Align MemAlign(4); // FIXME: ABI type alignment? 1453 MachineMemOperand *BaseMMO = MF.getMachineMemOperand( 1454 MachinePointerInfo(), 1455 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | 1456 MachineMemOperand::MOInvariant, 1457 MemSize, MemAlign); 1458 if (MMOOffset != 0) 1459 BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset, MemSize); 1460 1461 // If only the offset is divergent, emit a MUBUF buffer load instead. We can 1462 // assume that the buffer is unswizzled. 1463 1464 Register RSrc = MI.getOperand(1).getReg(); 1465 Register VIndex = B.buildConstant(S32, 0).getReg(0); 1466 B.getMRI()->setRegBank(VIndex, AMDGPU::VGPRRegBank); 1467 1468 SmallVector<Register, 4> LoadParts(NumLoads); 1469 1470 MachineBasicBlock::iterator MII = MI.getIterator(); 1471 MachineInstrSpan Span(MII, &B.getMBB()); 1472 1473 for (int i = 0; i < NumLoads; ++i) { 1474 if (NumLoads == 1) { 1475 LoadParts[i] = Dst; 1476 } else { 1477 LoadParts[i] = MRI.createGenericVirtualRegister(Ty); 1478 MRI.setRegBank(LoadParts[i], AMDGPU::VGPRRegBank); 1479 } 1480 1481 MachineMemOperand *MMO = BaseMMO; 1482 if (i != 0) 1483 BaseMMO = MF.getMachineMemOperand(BaseMMO, MMOOffset + 16 * i, MemSize); 1484 1485 B.buildInstr(AMDGPU::G_AMDGPU_BUFFER_LOAD) 1486 .addDef(LoadParts[i]) // vdata 1487 .addUse(RSrc) // rsrc 1488 .addUse(VIndex) // vindex 1489 .addUse(VOffset) // voffset 1490 .addUse(SOffset) // soffset 1491 .addImm(ImmOffset + 16 * i) // offset(imm) 1492 .addImm(0) // cachepolicy, swizzled buffer(imm) 1493 .addImm(0) // idxen(imm) 1494 .addMemOperand(MMO); 1495 } 1496 1497 // TODO: If only the resource is a VGPR, it may be better to execute the 1498 // scalar load in the waterfall loop if the resource is expected to frequently 1499 // be dynamically uniform. 1500 if (RSrcBank != &AMDGPU::SGPRRegBank) { 1501 // Remove the original instruction to avoid potentially confusing the 1502 // waterfall loop logic. 1503 B.setInstr(*Span.begin()); 1504 MI.eraseFromParent(); 1505 1506 SmallSet<Register, 4> OpsToWaterfall; 1507 1508 OpsToWaterfall.insert(RSrc); 1509 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), 1510 OpsToWaterfall, MRI); 1511 } 1512 1513 if (NumLoads != 1) { 1514 if (Ty.isVector()) 1515 B.buildConcatVectors(Dst, LoadParts); 1516 else 1517 B.buildMerge(Dst, LoadParts); 1518 } 1519 1520 // We removed the instruction earlier with a waterfall loop. 1521 if (RSrcBank == &AMDGPU::SGPRRegBank) 1522 MI.eraseFromParent(); 1523 1524 return true; 1525 } 1526 1527 bool AMDGPURegisterBankInfo::applyMappingBFEIntrinsic( 1528 const OperandsMapper &OpdMapper, bool Signed) const { 1529 MachineInstr &MI = OpdMapper.getMI(); 1530 MachineRegisterInfo &MRI = OpdMapper.getMRI(); 1531 1532 // Insert basic copies 1533 applyDefaultMapping(OpdMapper); 1534 1535 Register DstReg = MI.getOperand(0).getReg(); 1536 LLT Ty = MRI.getType(DstReg); 1537 1538 const LLT S32 = LLT::scalar(32); 1539 1540 const RegisterBank *DstBank = 1541 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 1542 if (DstBank == &AMDGPU::VGPRRegBank) { 1543 if (Ty == S32) 1544 return true; 1545 1546 // TODO: 64-bit version is scalar only, so we need to expand this. 1547 return false; 1548 } 1549 1550 Register SrcReg = MI.getOperand(2).getReg(); 1551 Register OffsetReg = MI.getOperand(3).getReg(); 1552 Register WidthReg = MI.getOperand(4).getReg(); 1553 1554 // The scalar form packs the offset and width in a single operand. 1555 1556 ApplyRegBankMapping ApplyBank(*this, MRI, &AMDGPU::SGPRRegBank); 1557 GISelObserverWrapper Observer(&ApplyBank); 1558 MachineIRBuilder B(MI); 1559 B.setChangeObserver(Observer); 1560 1561 // Ensure the high bits are clear to insert the offset. 1562 auto OffsetMask = B.buildConstant(S32, maskTrailingOnes<unsigned>(6)); 1563 auto ClampOffset = B.buildAnd(S32, OffsetReg, OffsetMask); 1564 1565 // Zeros out the low bits, so don't bother clamping the input value. 1566 auto ShiftWidth = B.buildShl(S32, WidthReg, B.buildConstant(S32, 16)); 1567 1568 // Transformation function, pack the offset and width of a BFE into 1569 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1570 // source, bits [5:0] contain the offset and bits [22:16] the width. 1571 auto MergedInputs = B.buildOr(S32, ClampOffset, ShiftWidth); 1572 1573 // TODO: It might be worth using a pseudo here to avoid scc clobber and 1574 // register class constraints. 1575 unsigned Opc = Ty == S32 ? (Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32) : 1576 (Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64); 1577 1578 auto MIB = B.buildInstr(Opc, {DstReg}, {SrcReg, MergedInputs}); 1579 if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this)) 1580 llvm_unreachable("failed to constrain BFE"); 1581 1582 MI.eraseFromParent(); 1583 return true; 1584 } 1585 1586 // FIXME: Duplicated from LegalizerHelper 1587 static CmpInst::Predicate minMaxToCompare(unsigned Opc) { 1588 switch (Opc) { 1589 case TargetOpcode::G_SMIN: 1590 return CmpInst::ICMP_SLT; 1591 case TargetOpcode::G_SMAX: 1592 return CmpInst::ICMP_SGT; 1593 case TargetOpcode::G_UMIN: 1594 return CmpInst::ICMP_ULT; 1595 case TargetOpcode::G_UMAX: 1596 return CmpInst::ICMP_UGT; 1597 default: 1598 llvm_unreachable("not in integer min/max"); 1599 } 1600 } 1601 1602 static unsigned minMaxToExtend(unsigned Opc) { 1603 switch (Opc) { 1604 case TargetOpcode::G_SMIN: 1605 case TargetOpcode::G_SMAX: 1606 return TargetOpcode::G_SEXT; 1607 case TargetOpcode::G_UMIN: 1608 case TargetOpcode::G_UMAX: 1609 return TargetOpcode::G_ZEXT; 1610 default: 1611 llvm_unreachable("not in integer min/max"); 1612 } 1613 } 1614 1615 // Emit a legalized extension from <2 x s16> to 2 32-bit components, avoiding 1616 // any illegal vector extend or unmerge operations. 1617 static std::pair<Register, Register> 1618 unpackV2S16ToS32(MachineIRBuilder &B, Register Src, unsigned ExtOpcode) { 1619 const LLT S32 = LLT::scalar(32); 1620 auto Bitcast = B.buildBitcast(S32, Src); 1621 1622 if (ExtOpcode == TargetOpcode::G_SEXT) { 1623 auto ExtLo = B.buildSExtInReg(S32, Bitcast, 16); 1624 auto ShiftHi = B.buildAShr(S32, Bitcast, B.buildConstant(S32, 16)); 1625 return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0)); 1626 } 1627 1628 auto ShiftHi = B.buildLShr(S32, Bitcast, B.buildConstant(S32, 16)); 1629 if (ExtOpcode == TargetOpcode::G_ZEXT) { 1630 auto ExtLo = B.buildAnd(S32, Bitcast, B.buildConstant(S32, 0xffff)); 1631 return std::make_pair(ExtLo.getReg(0), ShiftHi.getReg(0)); 1632 } 1633 1634 assert(ExtOpcode == TargetOpcode::G_ANYEXT); 1635 return std::make_pair(Bitcast.getReg(0), ShiftHi.getReg(0)); 1636 } 1637 1638 static MachineInstr *buildExpandedScalarMinMax(MachineIRBuilder &B, 1639 CmpInst::Predicate Pred, 1640 Register Dst, Register Src0, 1641 Register Src1) { 1642 const LLT CmpType = LLT::scalar(32); 1643 auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1); 1644 return B.buildSelect(Dst, Cmp, Src0, Src1); 1645 } 1646 1647 // FIXME: Duplicated from LegalizerHelper, except changing the boolean type. 1648 void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B, 1649 MachineInstr &MI) const { 1650 Register Dst = MI.getOperand(0).getReg(); 1651 Register Src0 = MI.getOperand(1).getReg(); 1652 Register Src1 = MI.getOperand(2).getReg(); 1653 1654 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); 1655 MachineInstr *Sel = buildExpandedScalarMinMax(B, Pred, Dst, Src0, Src1); 1656 1657 Register CmpReg = Sel->getOperand(1).getReg(); 1658 B.getMRI()->setRegBank(CmpReg, AMDGPU::SGPRRegBank); 1659 MI.eraseFromParent(); 1660 } 1661 1662 // For cases where only a single copy is inserted for matching register banks. 1663 // Replace the register in the instruction operand 1664 static bool substituteSimpleCopyRegs( 1665 const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, unsigned OpIdx) { 1666 SmallVector<unsigned, 1> SrcReg(OpdMapper.getVRegs(OpIdx)); 1667 if (!SrcReg.empty()) { 1668 assert(SrcReg.size() == 1); 1669 OpdMapper.getMI().getOperand(OpIdx).setReg(SrcReg[0]); 1670 return true; 1671 } 1672 1673 return false; 1674 } 1675 1676 /// Handle register layout difference for f16 images for some subtargets. 1677 Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B, 1678 MachineRegisterInfo &MRI, 1679 Register Reg) const { 1680 if (!Subtarget.hasUnpackedD16VMem()) 1681 return Reg; 1682 1683 const LLT S16 = LLT::scalar(16); 1684 LLT StoreVT = MRI.getType(Reg); 1685 if (!StoreVT.isVector() || StoreVT.getElementType() != S16) 1686 return Reg; 1687 1688 auto Unmerge = B.buildUnmerge(S16, Reg); 1689 1690 1691 SmallVector<Register, 4> WideRegs; 1692 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I) 1693 WideRegs.push_back(Unmerge.getReg(I)); 1694 1695 const LLT S32 = LLT::scalar(32); 1696 int NumElts = StoreVT.getNumElements(); 1697 1698 return B.buildMerge(LLT::vector(NumElts, S32), WideRegs).getReg(0); 1699 } 1700 1701 static std::pair<Register, unsigned> 1702 getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) { 1703 int64_t Const; 1704 if (mi_match(Reg, MRI, m_ICst(Const))) 1705 return std::make_pair(Register(), Const); 1706 1707 Register Base; 1708 if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const)))) 1709 return std::make_pair(Base, Const); 1710 1711 // TODO: Handle G_OR used for add case 1712 return std::make_pair(Reg, 0); 1713 } 1714 1715 std::pair<Register, unsigned> 1716 AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B, 1717 Register OrigOffset) const { 1718 const unsigned MaxImm = 4095; 1719 Register BaseReg; 1720 unsigned ImmOffset; 1721 const LLT S32 = LLT::scalar(32); 1722 1723 std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(), 1724 OrigOffset); 1725 1726 unsigned C1 = 0; 1727 if (ImmOffset != 0) { 1728 // If the immediate value is too big for the immoffset field, put the value 1729 // and -4096 into the immoffset field so that the value that is copied/added 1730 // for the voffset field is a multiple of 4096, and it stands more chance 1731 // of being CSEd with the copy/add for another similar load/store. 1732 // However, do not do that rounding down to a multiple of 4096 if that is a 1733 // negative number, as it appears to be illegal to have a negative offset 1734 // in the vgpr, even if adding the immediate offset makes it positive. 1735 unsigned Overflow = ImmOffset & ~MaxImm; 1736 ImmOffset -= Overflow; 1737 if ((int32_t)Overflow < 0) { 1738 Overflow += ImmOffset; 1739 ImmOffset = 0; 1740 } 1741 1742 C1 = ImmOffset; 1743 if (Overflow != 0) { 1744 if (!BaseReg) 1745 BaseReg = B.buildConstant(S32, Overflow).getReg(0); 1746 else { 1747 auto OverflowVal = B.buildConstant(S32, Overflow); 1748 BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0); 1749 } 1750 } 1751 } 1752 1753 if (!BaseReg) 1754 BaseReg = B.buildConstant(S32, 0).getReg(0); 1755 1756 return {BaseReg, C1}; 1757 } 1758 1759 static bool isZero(Register Reg, MachineRegisterInfo &MRI) { 1760 int64_t C; 1761 return mi_match(Reg, MRI, m_ICst(C)) && C == 0; 1762 } 1763 1764 static unsigned extractGLC(unsigned CachePolicy) { 1765 return CachePolicy & 1; 1766 } 1767 1768 static unsigned extractSLC(unsigned CachePolicy) { 1769 return (CachePolicy >> 1) & 1; 1770 } 1771 1772 static unsigned extractDLC(unsigned CachePolicy) { 1773 return (CachePolicy >> 2) & 1; 1774 } 1775 1776 MachineInstr * 1777 AMDGPURegisterBankInfo::selectStoreIntrinsic(MachineIRBuilder &B, 1778 MachineInstr &MI) const { 1779 MachineRegisterInfo &MRI = *B.getMRI(); 1780 executeInWaterfallLoop(B, MI, MRI, {2, 4}); 1781 1782 // FIXME: DAG lowering brokenly changes opcode based on FP vs. integer. 1783 1784 Register VData = MI.getOperand(1).getReg(); 1785 LLT Ty = MRI.getType(VData); 1786 1787 int EltSize = Ty.getScalarSizeInBits(); 1788 int Size = Ty.getSizeInBits(); 1789 1790 // FIXME: Broken integer truncstore. 1791 if (EltSize != 32) 1792 report_fatal_error("unhandled intrinsic store"); 1793 1794 // FIXME: Verifier should enforce 1 MMO for these intrinsics. 1795 const int MemSize = (*MI.memoperands_begin())->getSize(); 1796 1797 1798 Register RSrc = MI.getOperand(2).getReg(); 1799 Register VOffset = MI.getOperand(3).getReg(); 1800 Register SOffset = MI.getOperand(4).getReg(); 1801 unsigned CachePolicy = MI.getOperand(5).getImm(); 1802 1803 unsigned ImmOffset; 1804 std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset); 1805 1806 const bool Offen = !isZero(VOffset, MRI); 1807 1808 unsigned Opc = AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact; 1809 switch (8 * MemSize) { 1810 case 8: 1811 Opc = Offen ? AMDGPU::BUFFER_STORE_BYTE_OFFEN_exact : 1812 AMDGPU::BUFFER_STORE_BYTE_OFFSET_exact; 1813 break; 1814 case 16: 1815 Opc = Offen ? AMDGPU::BUFFER_STORE_SHORT_OFFEN_exact : 1816 AMDGPU::BUFFER_STORE_SHORT_OFFSET_exact; 1817 break; 1818 default: 1819 Opc = Offen ? AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact : 1820 AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact; 1821 if (Size > 32) 1822 Opc = AMDGPU::getMUBUFOpcode(Opc, Size / 32); 1823 break; 1824 } 1825 1826 1827 // Set the insertion point back to the instruction in case it was moved into a 1828 // loop. 1829 B.setInstr(MI); 1830 1831 MachineInstrBuilder MIB = B.buildInstr(Opc) 1832 .addUse(VData); 1833 1834 if (Offen) 1835 MIB.addUse(VOffset); 1836 1837 MIB.addUse(RSrc) 1838 .addUse(SOffset) 1839 .addImm(ImmOffset) 1840 .addImm(extractGLC(CachePolicy)) 1841 .addImm(extractSLC(CachePolicy)) 1842 .addImm(0) // tfe: FIXME: Remove from inst 1843 .addImm(extractDLC(CachePolicy)) 1844 .cloneMemRefs(MI); 1845 1846 // FIXME: We need a way to report failure from applyMappingImpl. 1847 // Insert constrain copies before inserting the loop. 1848 if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this)) 1849 report_fatal_error("failed to constrain selected store intrinsic"); 1850 1851 return MIB; 1852 } 1853 1854 bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg, 1855 Register SrcReg) const { 1856 MachineRegisterInfo &MRI = *B.getMRI(); 1857 LLT SrcTy = MRI.getType(SrcReg); 1858 if (SrcTy.getSizeInBits() == 32) { 1859 // Use a v_mov_b32 here to make the exec dependency explicit. 1860 B.buildInstr(AMDGPU::V_MOV_B32_e32) 1861 .addDef(DstReg) 1862 .addUse(SrcReg); 1863 return constrainGenericRegister(DstReg, AMDGPU::VGPR_32RegClass, MRI) && 1864 constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI); 1865 } 1866 1867 Register TmpReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1868 Register TmpReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1869 1870 B.buildInstr(AMDGPU::V_MOV_B32_e32) 1871 .addDef(TmpReg0) 1872 .addUse(SrcReg, 0, AMDGPU::sub0); 1873 B.buildInstr(AMDGPU::V_MOV_B32_e32) 1874 .addDef(TmpReg1) 1875 .addUse(SrcReg, 0, AMDGPU::sub1); 1876 B.buildInstr(AMDGPU::REG_SEQUENCE) 1877 .addDef(DstReg) 1878 .addUse(TmpReg0) 1879 .addImm(AMDGPU::sub0) 1880 .addUse(TmpReg1) 1881 .addImm(AMDGPU::sub1); 1882 1883 return constrainGenericRegister(SrcReg, AMDGPU::SReg_64RegClass, MRI) && 1884 constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI); 1885 } 1886 1887 /// Utility function for pushing dynamic vector indexes with a constant offset 1888 /// into waterwall loops. 1889 static void reinsertVectorIndexAdd(MachineIRBuilder &B, 1890 MachineInstr &IdxUseInstr, 1891 unsigned OpIdx, 1892 unsigned ConstOffset) { 1893 MachineRegisterInfo &MRI = *B.getMRI(); 1894 const LLT S32 = LLT::scalar(32); 1895 Register WaterfallIdx = IdxUseInstr.getOperand(OpIdx).getReg(); 1896 B.setInsertPt(*IdxUseInstr.getParent(), IdxUseInstr.getIterator()); 1897 1898 auto MaterializedOffset = B.buildConstant(S32, ConstOffset); 1899 1900 auto Add = B.buildAdd(S32, WaterfallIdx, MaterializedOffset); 1901 MRI.setRegBank(MaterializedOffset.getReg(0), AMDGPU::SGPRRegBank); 1902 MRI.setRegBank(Add.getReg(0), AMDGPU::SGPRRegBank); 1903 IdxUseInstr.getOperand(OpIdx).setReg(Add.getReg(0)); 1904 } 1905 1906 /// Implement extending a 32-bit value to a 64-bit value. \p Lo32Reg is the 1907 /// original 32-bit source value (to be inserted in the low part of the combined 1908 /// 64-bit result), and \p Hi32Reg is the high half of the combined 64-bit 1909 /// value. 1910 static void extendLow32IntoHigh32(MachineIRBuilder &B, 1911 Register Hi32Reg, Register Lo32Reg, 1912 unsigned ExtOpc, 1913 const RegisterBank &RegBank, 1914 bool IsBooleanSrc = false) { 1915 if (ExtOpc == AMDGPU::G_ZEXT) { 1916 B.buildConstant(Hi32Reg, 0); 1917 } else if (ExtOpc == AMDGPU::G_SEXT) { 1918 if (IsBooleanSrc) { 1919 // If we know the original source was an s1, the high half is the same as 1920 // the low. 1921 B.buildCopy(Hi32Reg, Lo32Reg); 1922 } else { 1923 // Replicate sign bit from 32-bit extended part. 1924 auto ShiftAmt = B.buildConstant(LLT::scalar(32), 31); 1925 B.getMRI()->setRegBank(ShiftAmt.getReg(0), RegBank); 1926 B.buildAShr(Hi32Reg, Lo32Reg, ShiftAmt); 1927 } 1928 } else { 1929 assert(ExtOpc == AMDGPU::G_ANYEXT && "not an integer extension"); 1930 B.buildUndef(Hi32Reg); 1931 } 1932 } 1933 1934 bool AMDGPURegisterBankInfo::foldExtractEltToCmpSelect( 1935 MachineInstr &MI, MachineRegisterInfo &MRI, 1936 const OperandsMapper &OpdMapper) const { 1937 1938 Register VecReg = MI.getOperand(1).getReg(); 1939 Register Idx = MI.getOperand(2).getReg(); 1940 1941 const RegisterBank &IdxBank = 1942 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 1943 1944 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank; 1945 1946 LLT VecTy = MRI.getType(VecReg); 1947 unsigned EltSize = VecTy.getScalarSizeInBits(); 1948 unsigned NumElem = VecTy.getNumElements(); 1949 1950 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem, 1951 IsDivergentIdx)) 1952 return false; 1953 1954 MachineIRBuilder B(MI); 1955 LLT S32 = LLT::scalar(32); 1956 1957 const RegisterBank &DstBank = 1958 *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 1959 const RegisterBank &SrcBank = 1960 *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 1961 1962 const RegisterBank &CCBank = 1963 (DstBank == AMDGPU::SGPRRegBank && 1964 SrcBank == AMDGPU::SGPRRegBank && 1965 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank 1966 : AMDGPU::VCCRegBank; 1967 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); 1968 1969 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { 1970 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg(); 1971 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); 1972 } 1973 1974 LLT EltTy = VecTy.getScalarType(); 1975 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0)); 1976 unsigned NumLanes = DstRegs.size(); 1977 if (!NumLanes) 1978 NumLanes = 1; 1979 else 1980 EltTy = MRI.getType(DstRegs[0]); 1981 1982 auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg); 1983 SmallVector<Register, 2> Res(NumLanes); 1984 for (unsigned L = 0; L < NumLanes; ++L) 1985 Res[L] = UnmergeToEltTy.getReg(L); 1986 1987 for (unsigned I = 1; I < NumElem; ++I) { 1988 auto IC = B.buildConstant(S32, I); 1989 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); 1990 auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC); 1991 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank); 1992 1993 for (unsigned L = 0; L < NumLanes; ++L) { 1994 auto S = B.buildSelect(EltTy, Cmp, 1995 UnmergeToEltTy.getReg(I * NumLanes + L), Res[L]); 1996 1997 for (unsigned N : { 0, 2, 3 }) 1998 MRI.setRegBank(S->getOperand(N).getReg(), DstBank); 1999 2000 Res[L] = S->getOperand(0).getReg(); 2001 } 2002 } 2003 2004 for (unsigned L = 0; L < NumLanes; ++L) { 2005 Register DstReg = (NumLanes == 1) ? MI.getOperand(0).getReg() : DstRegs[L]; 2006 B.buildCopy(DstReg, Res[L]); 2007 MRI.setRegBank(DstReg, DstBank); 2008 } 2009 2010 MRI.setRegBank(MI.getOperand(0).getReg(), DstBank); 2011 MI.eraseFromParent(); 2012 2013 return true; 2014 } 2015 2016 bool AMDGPURegisterBankInfo::foldInsertEltToCmpSelect( 2017 MachineInstr &MI, MachineRegisterInfo &MRI, 2018 const OperandsMapper &OpdMapper) const { 2019 2020 Register VecReg = MI.getOperand(1).getReg(); 2021 Register Idx = MI.getOperand(3).getReg(); 2022 2023 const RegisterBank &IdxBank = 2024 *OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank; 2025 2026 bool IsDivergentIdx = IdxBank != AMDGPU::SGPRRegBank; 2027 2028 LLT VecTy = MRI.getType(VecReg); 2029 unsigned EltSize = VecTy.getScalarSizeInBits(); 2030 unsigned NumElem = VecTy.getNumElements(); 2031 2032 if (!SITargetLowering::shouldExpandVectorDynExt(EltSize, NumElem, 2033 IsDivergentIdx)) 2034 return false; 2035 2036 MachineIRBuilder B(MI); 2037 LLT S32 = LLT::scalar(32); 2038 2039 const RegisterBank &DstBank = 2040 *OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2041 const RegisterBank &SrcBank = 2042 *OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 2043 const RegisterBank &InsBank = 2044 *OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 2045 2046 const RegisterBank &CCBank = 2047 (DstBank == AMDGPU::SGPRRegBank && 2048 SrcBank == AMDGPU::SGPRRegBank && 2049 InsBank == AMDGPU::SGPRRegBank && 2050 IdxBank == AMDGPU::SGPRRegBank) ? AMDGPU::SGPRRegBank 2051 : AMDGPU::VCCRegBank; 2052 LLT CCTy = (CCBank == AMDGPU::SGPRRegBank) ? S32 : LLT::scalar(1); 2053 2054 if (CCBank == AMDGPU::VCCRegBank && IdxBank == AMDGPU::SGPRRegBank) { 2055 Idx = B.buildCopy(S32, Idx)->getOperand(0).getReg(); 2056 MRI.setRegBank(Idx, AMDGPU::VGPRRegBank); 2057 } 2058 2059 LLT EltTy = VecTy.getScalarType(); 2060 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2)); 2061 unsigned NumLanes = InsRegs.size(); 2062 if (!NumLanes) { 2063 NumLanes = 1; 2064 InsRegs.push_back(MI.getOperand(2).getReg()); 2065 } else { 2066 EltTy = MRI.getType(InsRegs[0]); 2067 } 2068 2069 auto UnmergeToEltTy = B.buildUnmerge(EltTy, VecReg); 2070 SmallVector<Register, 16> Ops(NumElem * NumLanes); 2071 2072 for (unsigned I = 0; I < NumElem; ++I) { 2073 auto IC = B.buildConstant(S32, I); 2074 MRI.setRegBank(IC->getOperand(0).getReg(), AMDGPU::SGPRRegBank); 2075 auto Cmp = B.buildICmp(CmpInst::ICMP_EQ, CCTy, Idx, IC); 2076 MRI.setRegBank(Cmp->getOperand(0).getReg(), CCBank); 2077 2078 for (unsigned L = 0; L < NumLanes; ++L) { 2079 auto S = B.buildSelect(EltTy, Cmp, InsRegs[L], 2080 UnmergeToEltTy.getReg(I * NumLanes + L)); 2081 2082 for (unsigned N : { 0, 2, 3 }) 2083 MRI.setRegBank(S->getOperand(N).getReg(), DstBank); 2084 2085 Ops[I * NumLanes + L] = S->getOperand(0).getReg(); 2086 } 2087 } 2088 2089 LLT MergeTy = LLT::vector(Ops.size(), EltTy); 2090 if (MergeTy == MRI.getType(MI.getOperand(0).getReg())) { 2091 B.buildBuildVector(MI.getOperand(0), Ops); 2092 } else { 2093 auto Vec = B.buildBuildVector(MergeTy, Ops); 2094 MRI.setRegBank(Vec->getOperand(0).getReg(), DstBank); 2095 B.buildBitcast(MI.getOperand(0).getReg(), Vec); 2096 } 2097 2098 MRI.setRegBank(MI.getOperand(0).getReg(), DstBank); 2099 MI.eraseFromParent(); 2100 2101 return true; 2102 } 2103 2104 void AMDGPURegisterBankInfo::applyMappingImpl( 2105 const OperandsMapper &OpdMapper) const { 2106 MachineInstr &MI = OpdMapper.getMI(); 2107 unsigned Opc = MI.getOpcode(); 2108 MachineRegisterInfo &MRI = OpdMapper.getMRI(); 2109 switch (Opc) { 2110 case AMDGPU::G_PHI: { 2111 Register DstReg = MI.getOperand(0).getReg(); 2112 LLT DstTy = MRI.getType(DstReg); 2113 if (DstTy != LLT::scalar(1)) 2114 break; 2115 2116 const LLT S32 = LLT::scalar(32); 2117 const RegisterBank *DstBank = 2118 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2119 if (DstBank == &AMDGPU::VCCRegBank) { 2120 applyDefaultMapping(OpdMapper); 2121 // The standard handling only considers the result register bank for 2122 // phis. For VCC, blindly inserting a copy when the phi is lowered will 2123 // produce an invalid copy. We can only copy with some kind of compare to 2124 // get a vector boolean result. Insert a regitser bank copy that will be 2125 // correctly lowered to a compare. 2126 MachineIRBuilder B(*MI.getParent()->getParent()); 2127 2128 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 2129 Register SrcReg = MI.getOperand(I).getReg(); 2130 const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI); 2131 2132 if (SrcBank != &AMDGPU::VCCRegBank) { 2133 MachineBasicBlock *SrcMBB = MI.getOperand(I + 1).getMBB(); 2134 B.setInsertPt(*SrcMBB, SrcMBB->getFirstTerminator()); 2135 2136 auto Copy = B.buildCopy(LLT::scalar(1), SrcReg); 2137 MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank); 2138 MI.getOperand(I).setReg(Copy.getReg(0)); 2139 } 2140 } 2141 2142 return; 2143 } 2144 2145 // Phi handling is strange and only considers the bank of the destination. 2146 substituteSimpleCopyRegs(OpdMapper, 0); 2147 2148 // Promote SGPR/VGPR booleans to s32 2149 MachineFunction *MF = MI.getParent()->getParent(); 2150 ApplyRegBankMapping ApplyBank(*this, MRI, DstBank); 2151 GISelObserverWrapper Observer(&ApplyBank); 2152 MachineIRBuilder B(MI); 2153 LegalizerHelper Helper(*MF, Observer, B); 2154 2155 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) 2156 llvm_unreachable("widen scalar should have succeeded"); 2157 2158 return; 2159 } 2160 case AMDGPU::G_ICMP: 2161 case AMDGPU::G_UADDO: 2162 case AMDGPU::G_USUBO: 2163 case AMDGPU::G_UADDE: 2164 case AMDGPU::G_SADDE: 2165 case AMDGPU::G_USUBE: 2166 case AMDGPU::G_SSUBE: { 2167 unsigned BoolDstOp = Opc == AMDGPU::G_ICMP ? 0 : 1; 2168 Register DstReg = MI.getOperand(BoolDstOp).getReg(); 2169 2170 const RegisterBank *DstBank = 2171 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2172 if (DstBank != &AMDGPU::SGPRRegBank) 2173 break; 2174 2175 const bool HasCarryIn = MI.getNumOperands() == 5; 2176 2177 // If this is a scalar compare, promote the result to s32, as the selection 2178 // will end up using a copy to a 32-bit vreg. 2179 const LLT S32 = LLT::scalar(32); 2180 Register NewDstReg = MRI.createGenericVirtualRegister(S32); 2181 MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank); 2182 MI.getOperand(BoolDstOp).setReg(NewDstReg); 2183 MachineIRBuilder B(MI); 2184 2185 if (HasCarryIn) { 2186 Register NewSrcReg = MRI.createGenericVirtualRegister(S32); 2187 MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank); 2188 B.buildZExt(NewSrcReg, MI.getOperand(4).getReg()); 2189 MI.getOperand(4).setReg(NewSrcReg); 2190 } 2191 2192 MachineBasicBlock *MBB = MI.getParent(); 2193 B.setInsertPt(*MBB, std::next(MI.getIterator())); 2194 2195 // If we had a constrained VCC result register, a copy was inserted to VCC 2196 // from SGPR. 2197 SmallVector<Register, 1> DefRegs(OpdMapper.getVRegs(0)); 2198 if (DefRegs.empty()) 2199 DefRegs.push_back(DstReg); 2200 B.buildTrunc(DefRegs[0], NewDstReg); 2201 return; 2202 } 2203 case AMDGPU::G_SELECT: { 2204 Register DstReg = MI.getOperand(0).getReg(); 2205 LLT DstTy = MRI.getType(DstReg); 2206 2207 SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(1)); 2208 if (CondRegs.empty()) 2209 CondRegs.push_back(MI.getOperand(1).getReg()); 2210 else { 2211 assert(CondRegs.size() == 1); 2212 } 2213 2214 const RegisterBank *CondBank = getRegBank(CondRegs[0], MRI, *TRI); 2215 if (CondBank == &AMDGPU::SGPRRegBank) { 2216 MachineIRBuilder B(MI); 2217 const LLT S32 = LLT::scalar(32); 2218 Register NewCondReg = MRI.createGenericVirtualRegister(S32); 2219 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); 2220 2221 MI.getOperand(1).setReg(NewCondReg); 2222 B.buildZExt(NewCondReg, CondRegs[0]); 2223 } 2224 2225 if (DstTy.getSizeInBits() != 64) 2226 break; 2227 2228 MachineIRBuilder B(MI); 2229 LLT HalfTy = getHalfSizedType(DstTy); 2230 2231 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); 2232 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2)); 2233 SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(3)); 2234 2235 // All inputs are SGPRs, nothing special to do. 2236 if (DefRegs.empty()) { 2237 assert(Src1Regs.empty() && Src2Regs.empty()); 2238 break; 2239 } 2240 2241 if (Src1Regs.empty()) 2242 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg()); 2243 else { 2244 setRegsToType(MRI, Src1Regs, HalfTy); 2245 } 2246 2247 if (Src2Regs.empty()) 2248 split64BitValueForMapping(B, Src2Regs, HalfTy, MI.getOperand(3).getReg()); 2249 else 2250 setRegsToType(MRI, Src2Regs, HalfTy); 2251 2252 setRegsToType(MRI, DefRegs, HalfTy); 2253 2254 B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0]); 2255 B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1]); 2256 2257 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); 2258 MI.eraseFromParent(); 2259 return; 2260 } 2261 case AMDGPU::G_BRCOND: { 2262 Register CondReg = MI.getOperand(0).getReg(); 2263 // FIXME: Should use legalizer helper, but should change bool ext type. 2264 const RegisterBank *CondBank = 2265 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2266 2267 if (CondBank == &AMDGPU::SGPRRegBank) { 2268 MachineIRBuilder B(MI); 2269 const LLT S32 = LLT::scalar(32); 2270 Register NewCondReg = MRI.createGenericVirtualRegister(S32); 2271 MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); 2272 2273 MI.getOperand(0).setReg(NewCondReg); 2274 B.buildZExt(NewCondReg, CondReg); 2275 return; 2276 } 2277 2278 break; 2279 } 2280 case AMDGPU::G_AND: 2281 case AMDGPU::G_OR: 2282 case AMDGPU::G_XOR: { 2283 // 64-bit and is only available on the SALU, so split into 2 32-bit ops if 2284 // there is a VGPR input. 2285 Register DstReg = MI.getOperand(0).getReg(); 2286 LLT DstTy = MRI.getType(DstReg); 2287 2288 if (DstTy.getSizeInBits() == 1) { 2289 const RegisterBank *DstBank = 2290 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2291 if (DstBank == &AMDGPU::VCCRegBank) 2292 break; 2293 2294 MachineFunction *MF = MI.getParent()->getParent(); 2295 ApplyRegBankMapping ApplyBank(*this, MRI, DstBank); 2296 GISelObserverWrapper Observer(&ApplyBank); 2297 MachineIRBuilder B(MI); 2298 LegalizerHelper Helper(*MF, Observer, B); 2299 2300 if (Helper.widenScalar(MI, 0, LLT::scalar(32)) != 2301 LegalizerHelper::Legalized) 2302 llvm_unreachable("widen scalar should have succeeded"); 2303 return; 2304 } 2305 2306 if (DstTy.getSizeInBits() != 64) 2307 break; 2308 2309 LLT HalfTy = getHalfSizedType(DstTy); 2310 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); 2311 SmallVector<Register, 2> Src0Regs(OpdMapper.getVRegs(1)); 2312 SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2)); 2313 2314 // All inputs are SGPRs, nothing special to do. 2315 if (DefRegs.empty()) { 2316 assert(Src0Regs.empty() && Src1Regs.empty()); 2317 break; 2318 } 2319 2320 assert(DefRegs.size() == 2); 2321 assert(Src0Regs.size() == Src1Regs.size() && 2322 (Src0Regs.empty() || Src0Regs.size() == 2)); 2323 2324 // Depending on where the source registers came from, the generic code may 2325 // have decided to split the inputs already or not. If not, we still need to 2326 // extract the values. 2327 MachineIRBuilder B(MI); 2328 2329 if (Src0Regs.empty()) 2330 split64BitValueForMapping(B, Src0Regs, HalfTy, MI.getOperand(1).getReg()); 2331 else 2332 setRegsToType(MRI, Src0Regs, HalfTy); 2333 2334 if (Src1Regs.empty()) 2335 split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg()); 2336 else 2337 setRegsToType(MRI, Src1Regs, HalfTy); 2338 2339 setRegsToType(MRI, DefRegs, HalfTy); 2340 2341 B.buildInstr(Opc, {DefRegs[0]}, {Src0Regs[0], Src1Regs[0]}); 2342 B.buildInstr(Opc, {DefRegs[1]}, {Src0Regs[1], Src1Regs[1]}); 2343 2344 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); 2345 MI.eraseFromParent(); 2346 return; 2347 } 2348 case AMDGPU::G_ADD: 2349 case AMDGPU::G_SUB: 2350 case AMDGPU::G_MUL: 2351 case AMDGPU::G_SHL: 2352 case AMDGPU::G_LSHR: 2353 case AMDGPU::G_ASHR: { 2354 Register DstReg = MI.getOperand(0).getReg(); 2355 LLT DstTy = MRI.getType(DstReg); 2356 2357 // 16-bit operations are VALU only, but can be promoted to 32-bit SALU. 2358 // Packed 16-bit operations need to be scalarized and promoted. 2359 if (DstTy != LLT::scalar(16) && DstTy != LLT::vector(2, 16)) 2360 break; 2361 2362 const RegisterBank *DstBank = 2363 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2364 if (DstBank == &AMDGPU::VGPRRegBank) 2365 break; 2366 2367 const LLT S32 = LLT::scalar(32); 2368 MachineBasicBlock *MBB = MI.getParent(); 2369 MachineFunction *MF = MBB->getParent(); 2370 MachineIRBuilder B(MI); 2371 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); 2372 GISelObserverWrapper Observer(&ApplySALU); 2373 2374 if (DstTy.isVector()) { 2375 B.setChangeObserver(Observer); 2376 2377 Register WideSrc0Lo, WideSrc0Hi; 2378 Register WideSrc1Lo, WideSrc1Hi; 2379 2380 std::tie(WideSrc0Lo, WideSrc0Hi) 2381 = unpackV2S16ToS32(B, MI.getOperand(1).getReg(), AMDGPU::G_ANYEXT); 2382 std::tie(WideSrc1Lo, WideSrc1Hi) 2383 = unpackV2S16ToS32(B, MI.getOperand(2).getReg(), AMDGPU::G_ANYEXT); 2384 auto Lo = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Lo, WideSrc1Lo}); 2385 auto Hi = B.buildInstr(MI.getOpcode(), {S32}, {WideSrc0Hi, WideSrc1Hi}); 2386 B.buildBuildVectorTrunc(DstReg, {Lo.getReg(0), Hi.getReg(0)}); 2387 MI.eraseFromParent(); 2388 } else { 2389 LegalizerHelper Helper(*MF, Observer, B); 2390 2391 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) 2392 llvm_unreachable("widen scalar should have succeeded"); 2393 2394 // FIXME: s16 shift amounts should be legal. 2395 if (Opc == AMDGPU::G_SHL || Opc == AMDGPU::G_LSHR || 2396 Opc == AMDGPU::G_ASHR) { 2397 B.setInsertPt(*MBB, MI.getIterator()); 2398 if (Helper.widenScalar(MI, 1, S32) != LegalizerHelper::Legalized) 2399 llvm_unreachable("widen scalar should have succeeded"); 2400 } 2401 } 2402 2403 return; 2404 } 2405 case AMDGPU::G_SMIN: 2406 case AMDGPU::G_SMAX: 2407 case AMDGPU::G_UMIN: 2408 case AMDGPU::G_UMAX: { 2409 Register DstReg = MI.getOperand(0).getReg(); 2410 const RegisterBank *DstBank = 2411 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2412 if (DstBank == &AMDGPU::VGPRRegBank) 2413 break; 2414 2415 MachineFunction *MF = MI.getParent()->getParent(); 2416 MachineIRBuilder B(MI); 2417 2418 // Turn scalar min/max into a compare and select. 2419 LLT Ty = MRI.getType(DstReg); 2420 const LLT S32 = LLT::scalar(32); 2421 const LLT S16 = LLT::scalar(16); 2422 const LLT V2S16 = LLT::vector(2, 16); 2423 2424 if (Ty == V2S16) { 2425 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); 2426 GISelObserverWrapper Observer(&ApplySALU); 2427 B.setChangeObserver(Observer); 2428 2429 // Need to widen to s32, and expand as cmp + select, and avoid producing 2430 // illegal vector extends or unmerges that would need further 2431 // legalization. 2432 // 2433 // TODO: Should we just readfirstlane? That should probably be handled 2434 // with a UniformVGPR register bank that wouldn't need special 2435 // consideration here. 2436 2437 Register Dst = MI.getOperand(0).getReg(); 2438 Register Src0 = MI.getOperand(1).getReg(); 2439 Register Src1 = MI.getOperand(2).getReg(); 2440 2441 Register WideSrc0Lo, WideSrc0Hi; 2442 Register WideSrc1Lo, WideSrc1Hi; 2443 2444 unsigned ExtendOp = minMaxToExtend(MI.getOpcode()); 2445 2446 std::tie(WideSrc0Lo, WideSrc0Hi) = unpackV2S16ToS32(B, Src0, ExtendOp); 2447 std::tie(WideSrc1Lo, WideSrc1Hi) = unpackV2S16ToS32(B, Src1, ExtendOp); 2448 2449 Register Lo = MRI.createGenericVirtualRegister(S32); 2450 Register Hi = MRI.createGenericVirtualRegister(S32); 2451 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); 2452 buildExpandedScalarMinMax(B, Pred, Lo, WideSrc0Lo, WideSrc1Lo); 2453 buildExpandedScalarMinMax(B, Pred, Hi, WideSrc0Hi, WideSrc1Hi); 2454 2455 B.buildBuildVectorTrunc(Dst, {Lo, Hi}); 2456 MI.eraseFromParent(); 2457 } else if (Ty == S16) { 2458 ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); 2459 GISelObserverWrapper Observer(&ApplySALU); 2460 LegalizerHelper Helper(*MF, Observer, B); 2461 2462 // Need to widen to s32, and expand as cmp + select. 2463 if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) 2464 llvm_unreachable("widenScalar should have succeeded"); 2465 2466 // FIXME: This is relying on widenScalar leaving MI in place. 2467 lowerScalarMinMax(B, MI); 2468 } else 2469 lowerScalarMinMax(B, MI); 2470 2471 return; 2472 } 2473 case AMDGPU::G_SEXT_INREG: { 2474 SmallVector<Register, 2> SrcRegs(OpdMapper.getVRegs(1)); 2475 if (SrcRegs.empty()) 2476 break; // Nothing to repair 2477 2478 const LLT S32 = LLT::scalar(32); 2479 MachineIRBuilder B(MI); 2480 ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank); 2481 GISelObserverWrapper Observer(&O); 2482 B.setChangeObserver(Observer); 2483 2484 // Don't use LegalizerHelper's narrowScalar. It produces unwanted G_SEXTs 2485 // we would need to further expand, and doesn't let us directly set the 2486 // result registers. 2487 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0)); 2488 2489 int Amt = MI.getOperand(2).getImm(); 2490 if (Amt <= 32) { 2491 if (Amt == 32) { 2492 // The low bits are unchanged. 2493 B.buildCopy(DstRegs[0], SrcRegs[0]); 2494 } else { 2495 // Extend in the low bits and propagate the sign bit to the high half. 2496 B.buildSExtInReg(DstRegs[0], SrcRegs[0], Amt); 2497 } 2498 2499 B.buildAShr(DstRegs[1], DstRegs[0], B.buildConstant(S32, 31)); 2500 } else { 2501 // The low bits are unchanged, and extend in the high bits. 2502 B.buildCopy(DstRegs[0], SrcRegs[0]); 2503 B.buildSExtInReg(DstRegs[1], DstRegs[0], Amt - 32); 2504 } 2505 2506 Register DstReg = MI.getOperand(0).getReg(); 2507 MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); 2508 MI.eraseFromParent(); 2509 return; 2510 } 2511 case AMDGPU::G_CTPOP: 2512 case AMDGPU::G_CTLZ_ZERO_UNDEF: 2513 case AMDGPU::G_CTTZ_ZERO_UNDEF: { 2514 MachineIRBuilder B(MI); 2515 MachineFunction &MF = B.getMF(); 2516 2517 const RegisterBank *DstBank = 2518 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2519 if (DstBank == &AMDGPU::SGPRRegBank) 2520 break; 2521 2522 Register SrcReg = MI.getOperand(1).getReg(); 2523 const LLT S32 = LLT::scalar(32); 2524 LLT Ty = MRI.getType(SrcReg); 2525 if (Ty == S32) 2526 break; 2527 2528 ApplyRegBankMapping ApplyVALU(*this, MRI, &AMDGPU::VGPRRegBank); 2529 GISelObserverWrapper Observer(&ApplyVALU); 2530 LegalizerHelper Helper(MF, Observer, B); 2531 2532 if (Helper.narrowScalar(MI, 1, S32) != LegalizerHelper::Legalized) 2533 llvm_unreachable("narrowScalar should have succeeded"); 2534 return; 2535 } 2536 case AMDGPU::G_SEXT: 2537 case AMDGPU::G_ZEXT: 2538 case AMDGPU::G_ANYEXT: { 2539 Register SrcReg = MI.getOperand(1).getReg(); 2540 LLT SrcTy = MRI.getType(SrcReg); 2541 const bool Signed = Opc == AMDGPU::G_SEXT; 2542 2543 assert(empty(OpdMapper.getVRegs(1))); 2544 2545 MachineIRBuilder B(MI); 2546 const RegisterBank *SrcBank = 2547 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 2548 2549 Register DstReg = MI.getOperand(0).getReg(); 2550 LLT DstTy = MRI.getType(DstReg); 2551 if (DstTy.isScalar() && 2552 SrcBank != &AMDGPU::SGPRRegBank && 2553 SrcBank != &AMDGPU::VCCRegBank && 2554 // FIXME: Should handle any type that round to s64 when irregular 2555 // breakdowns supported. 2556 DstTy.getSizeInBits() == 64 && 2557 SrcTy.getSizeInBits() <= 32) { 2558 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); 2559 2560 // Extend to 32-bit, and then extend the low half. 2561 if (Signed) { 2562 // TODO: Should really be buildSExtOrCopy 2563 B.buildSExtOrTrunc(DefRegs[0], SrcReg); 2564 } else if (Opc == AMDGPU::G_ZEXT) { 2565 B.buildZExtOrTrunc(DefRegs[0], SrcReg); 2566 } else { 2567 B.buildAnyExtOrTrunc(DefRegs[0], SrcReg); 2568 } 2569 2570 extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank); 2571 MRI.setRegBank(DstReg, *SrcBank); 2572 MI.eraseFromParent(); 2573 return; 2574 } 2575 2576 if (SrcTy != LLT::scalar(1)) 2577 return; 2578 2579 // It is not legal to have a legalization artifact with a VCC source. Rather 2580 // than introducing a copy, insert the select we would have to select the 2581 // copy to. 2582 if (SrcBank == &AMDGPU::VCCRegBank) { 2583 SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); 2584 2585 const RegisterBank *DstBank = &AMDGPU::VGPRRegBank; 2586 2587 unsigned DstSize = DstTy.getSizeInBits(); 2588 // 64-bit select is SGPR only 2589 const bool UseSel64 = DstSize > 32 && 2590 SrcBank->getID() == AMDGPU::SGPRRegBankID; 2591 2592 // TODO: Should s16 select be legal? 2593 LLT SelType = UseSel64 ? LLT::scalar(64) : LLT::scalar(32); 2594 auto True = B.buildConstant(SelType, Signed ? -1 : 1); 2595 auto False = B.buildConstant(SelType, 0); 2596 2597 MRI.setRegBank(True.getReg(0), *DstBank); 2598 MRI.setRegBank(False.getReg(0), *DstBank); 2599 MRI.setRegBank(DstReg, *DstBank); 2600 2601 if (DstSize > 32) { 2602 B.buildSelect(DefRegs[0], SrcReg, True, False); 2603 extendLow32IntoHigh32(B, DefRegs[1], DefRegs[0], Opc, *SrcBank, true); 2604 } else if (DstSize < 32) { 2605 auto Sel = B.buildSelect(SelType, SrcReg, True, False); 2606 MRI.setRegBank(Sel.getReg(0), *DstBank); 2607 B.buildTrunc(DstReg, Sel); 2608 } else { 2609 B.buildSelect(DstReg, SrcReg, True, False); 2610 } 2611 2612 MI.eraseFromParent(); 2613 return; 2614 } 2615 2616 break; 2617 } 2618 case AMDGPU::G_BUILD_VECTOR: 2619 case AMDGPU::G_BUILD_VECTOR_TRUNC: { 2620 Register DstReg = MI.getOperand(0).getReg(); 2621 LLT DstTy = MRI.getType(DstReg); 2622 if (DstTy != LLT::vector(2, 16)) 2623 break; 2624 2625 assert(MI.getNumOperands() == 3 && OpdMapper.getVRegs(0).empty()); 2626 substituteSimpleCopyRegs(OpdMapper, 1); 2627 substituteSimpleCopyRegs(OpdMapper, 2); 2628 2629 const RegisterBank *DstBank = 2630 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2631 if (DstBank == &AMDGPU::SGPRRegBank) 2632 break; // Can use S_PACK_* instructions. 2633 2634 MachineIRBuilder B(MI); 2635 2636 Register Lo = MI.getOperand(1).getReg(); 2637 Register Hi = MI.getOperand(2).getReg(); 2638 const LLT S32 = LLT::scalar(32); 2639 2640 const RegisterBank *BankLo = 2641 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 2642 const RegisterBank *BankHi = 2643 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 2644 2645 Register ZextLo; 2646 Register ShiftHi; 2647 2648 if (Opc == AMDGPU::G_BUILD_VECTOR) { 2649 ZextLo = B.buildZExt(S32, Lo).getReg(0); 2650 MRI.setRegBank(ZextLo, *BankLo); 2651 2652 Register ZextHi = B.buildZExt(S32, Hi).getReg(0); 2653 MRI.setRegBank(ZextHi, *BankHi); 2654 2655 auto ShiftAmt = B.buildConstant(S32, 16); 2656 MRI.setRegBank(ShiftAmt.getReg(0), *BankHi); 2657 2658 ShiftHi = B.buildShl(S32, ZextHi, ShiftAmt).getReg(0); 2659 MRI.setRegBank(ShiftHi, *BankHi); 2660 } else { 2661 Register MaskLo = B.buildConstant(S32, 0xffff).getReg(0); 2662 MRI.setRegBank(MaskLo, *BankLo); 2663 2664 auto ShiftAmt = B.buildConstant(S32, 16); 2665 MRI.setRegBank(ShiftAmt.getReg(0), *BankHi); 2666 2667 ShiftHi = B.buildShl(S32, Hi, ShiftAmt).getReg(0); 2668 MRI.setRegBank(ShiftHi, *BankHi); 2669 2670 ZextLo = B.buildAnd(S32, Lo, MaskLo).getReg(0); 2671 MRI.setRegBank(ZextLo, *BankLo); 2672 } 2673 2674 auto Or = B.buildOr(S32, ZextLo, ShiftHi); 2675 MRI.setRegBank(Or.getReg(0), *DstBank); 2676 2677 B.buildBitcast(DstReg, Or); 2678 MI.eraseFromParent(); 2679 return; 2680 } 2681 case AMDGPU::G_EXTRACT_VECTOR_ELT: { 2682 SmallVector<Register, 2> DstRegs(OpdMapper.getVRegs(0)); 2683 2684 assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty()); 2685 2686 Register DstReg = MI.getOperand(0).getReg(); 2687 Register SrcReg = MI.getOperand(1).getReg(); 2688 2689 const LLT S32 = LLT::scalar(32); 2690 LLT DstTy = MRI.getType(DstReg); 2691 LLT SrcTy = MRI.getType(SrcReg); 2692 2693 if (foldExtractEltToCmpSelect(MI, MRI, OpdMapper)) 2694 return; 2695 2696 MachineIRBuilder B(MI); 2697 2698 const ValueMapping &DstMapping 2699 = OpdMapper.getInstrMapping().getOperandMapping(0); 2700 const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank; 2701 const RegisterBank *SrcBank = 2702 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 2703 const RegisterBank *IdxBank = 2704 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 2705 2706 Register BaseIdxReg; 2707 unsigned ConstOffset; 2708 MachineInstr *OffsetDef; 2709 std::tie(BaseIdxReg, ConstOffset, OffsetDef) = 2710 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(2).getReg()); 2711 2712 // See if the index is an add of a constant which will be foldable by moving 2713 // the base register of the index later if this is going to be executed in a 2714 // waterfall loop. This is essentially to reassociate the add of a constant 2715 // with the readfirstlane. 2716 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && 2717 ConstOffset > 0 && 2718 ConstOffset < SrcTy.getNumElements(); 2719 2720 // Move the base register. We'll re-insert the add later. 2721 if (ShouldMoveIndexIntoLoop) 2722 MI.getOperand(2).setReg(BaseIdxReg); 2723 2724 // If this is a VGPR result only because the index was a VGPR result, the 2725 // actual indexing will be done on the SGPR source vector, which will 2726 // produce a scalar result. We need to copy to the VGPR result inside the 2727 // waterfall loop. 2728 const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank && 2729 SrcBank == &AMDGPU::SGPRRegBank; 2730 if (DstRegs.empty()) { 2731 applyDefaultMapping(OpdMapper); 2732 2733 executeInWaterfallLoop(MI, MRI, { 2 }); 2734 2735 if (NeedCopyToVGPR) { 2736 // We don't want a phi for this temporary reg. 2737 Register TmpReg = MRI.createGenericVirtualRegister(DstTy); 2738 MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank); 2739 MI.getOperand(0).setReg(TmpReg); 2740 B.setInsertPt(*MI.getParent(), ++MI.getIterator()); 2741 2742 // Use a v_mov_b32 here to make the exec dependency explicit. 2743 buildVCopy(B, DstReg, TmpReg); 2744 } 2745 2746 // Re-insert the constant offset add inside the waterfall loop. 2747 if (ShouldMoveIndexIntoLoop) 2748 reinsertVectorIndexAdd(B, MI, 2, ConstOffset); 2749 2750 return; 2751 } 2752 2753 assert(DstTy.getSizeInBits() == 64); 2754 2755 LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32); 2756 2757 auto CastSrc = B.buildBitcast(Vec32, SrcReg); 2758 auto One = B.buildConstant(S32, 1); 2759 2760 MachineBasicBlock::iterator MII = MI.getIterator(); 2761 2762 // Split the vector index into 32-bit pieces. Prepare to move all of the 2763 // new instructions into a waterfall loop if necessary. 2764 // 2765 // Don't put the bitcast or constant in the loop. 2766 MachineInstrSpan Span(MII, &B.getMBB()); 2767 2768 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1). 2769 auto IdxLo = B.buildShl(S32, BaseIdxReg, One); 2770 auto IdxHi = B.buildAdd(S32, IdxLo, One); 2771 2772 auto Extract0 = B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo); 2773 auto Extract1 = B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi); 2774 2775 MRI.setRegBank(DstReg, *DstBank); 2776 MRI.setRegBank(CastSrc.getReg(0), *SrcBank); 2777 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank); 2778 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank); 2779 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank); 2780 2781 SmallSet<Register, 4> OpsToWaterfall; 2782 if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 2 })) { 2783 MI.eraseFromParent(); 2784 return; 2785 } 2786 2787 // Remove the original instruction to avoid potentially confusing the 2788 // waterfall loop logic. 2789 B.setInstr(*Span.begin()); 2790 MI.eraseFromParent(); 2791 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), 2792 OpsToWaterfall, MRI); 2793 2794 if (NeedCopyToVGPR) { 2795 MachineBasicBlock *LoopBB = Extract1->getParent(); 2796 Register TmpReg0 = MRI.createGenericVirtualRegister(S32); 2797 Register TmpReg1 = MRI.createGenericVirtualRegister(S32); 2798 MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank); 2799 MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank); 2800 2801 Extract0->getOperand(0).setReg(TmpReg0); 2802 Extract1->getOperand(0).setReg(TmpReg1); 2803 2804 B.setInsertPt(*LoopBB, ++Extract1->getIterator()); 2805 2806 buildVCopy(B, DstRegs[0], TmpReg0); 2807 buildVCopy(B, DstRegs[1], TmpReg1); 2808 } 2809 2810 if (ShouldMoveIndexIntoLoop) 2811 reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset); 2812 2813 return; 2814 } 2815 case AMDGPU::G_INSERT_VECTOR_ELT: { 2816 SmallVector<Register, 2> InsRegs(OpdMapper.getVRegs(2)); 2817 2818 Register DstReg = MI.getOperand(0).getReg(); 2819 LLT VecTy = MRI.getType(DstReg); 2820 2821 assert(OpdMapper.getVRegs(0).empty()); 2822 assert(OpdMapper.getVRegs(3).empty()); 2823 2824 if (substituteSimpleCopyRegs(OpdMapper, 1)) 2825 MRI.setType(MI.getOperand(1).getReg(), VecTy); 2826 2827 if (foldInsertEltToCmpSelect(MI, MRI, OpdMapper)) 2828 return; 2829 2830 const RegisterBank *IdxBank = 2831 OpdMapper.getInstrMapping().getOperandMapping(3).BreakDown[0].RegBank; 2832 2833 Register SrcReg = MI.getOperand(1).getReg(); 2834 Register InsReg = MI.getOperand(2).getReg(); 2835 LLT InsTy = MRI.getType(InsReg); 2836 (void)InsTy; 2837 2838 Register BaseIdxReg; 2839 unsigned ConstOffset; 2840 MachineInstr *OffsetDef; 2841 std::tie(BaseIdxReg, ConstOffset, OffsetDef) = 2842 AMDGPU::getBaseWithConstantOffset(MRI, MI.getOperand(3).getReg()); 2843 2844 // See if the index is an add of a constant which will be foldable by moving 2845 // the base register of the index later if this is going to be executed in a 2846 // waterfall loop. This is essentially to reassociate the add of a constant 2847 // with the readfirstlane. 2848 bool ShouldMoveIndexIntoLoop = IdxBank != &AMDGPU::SGPRRegBank && 2849 ConstOffset > 0 && 2850 ConstOffset < VecTy.getNumElements(); 2851 2852 // Move the base register. We'll re-insert the add later. 2853 if (ShouldMoveIndexIntoLoop) 2854 MI.getOperand(3).setReg(BaseIdxReg); 2855 2856 2857 if (InsRegs.empty()) { 2858 executeInWaterfallLoop(MI, MRI, { 3 }); 2859 2860 // Re-insert the constant offset add inside the waterfall loop. 2861 if (ShouldMoveIndexIntoLoop) { 2862 MachineIRBuilder B(MI); 2863 reinsertVectorIndexAdd(B, MI, 3, ConstOffset); 2864 } 2865 2866 return; 2867 } 2868 2869 2870 assert(InsTy.getSizeInBits() == 64); 2871 2872 const LLT S32 = LLT::scalar(32); 2873 LLT Vec32 = LLT::vector(2 * VecTy.getNumElements(), 32); 2874 2875 MachineIRBuilder B(MI); 2876 auto CastSrc = B.buildBitcast(Vec32, SrcReg); 2877 auto One = B.buildConstant(S32, 1); 2878 2879 // Split the vector index into 32-bit pieces. Prepare to move all of the 2880 // new instructions into a waterfall loop if necessary. 2881 // 2882 // Don't put the bitcast or constant in the loop. 2883 MachineInstrSpan Span(MachineBasicBlock::iterator(&MI), &B.getMBB()); 2884 2885 // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1). 2886 auto IdxLo = B.buildShl(S32, BaseIdxReg, One); 2887 auto IdxHi = B.buildAdd(S32, IdxLo, One); 2888 2889 auto InsLo = B.buildInsertVectorElement(Vec32, CastSrc, InsRegs[0], IdxLo); 2890 auto InsHi = B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi); 2891 2892 const RegisterBank *DstBank = 2893 OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; 2894 const RegisterBank *SrcBank = 2895 OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; 2896 const RegisterBank *InsSrcBank = 2897 OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; 2898 2899 MRI.setRegBank(InsReg, *InsSrcBank); 2900 MRI.setRegBank(CastSrc.getReg(0), *SrcBank); 2901 MRI.setRegBank(InsLo.getReg(0), *DstBank); 2902 MRI.setRegBank(InsHi.getReg(0), *DstBank); 2903 MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank); 2904 MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank); 2905 MRI.setRegBank(IdxHi.getReg(0), AMDGPU::SGPRRegBank); 2906 2907 2908 SmallSet<Register, 4> OpsToWaterfall; 2909 if (!collectWaterfallOperands(OpsToWaterfall, MI, MRI, { 3 })) { 2910 B.setInsertPt(B.getMBB(), MI); 2911 B.buildBitcast(DstReg, InsHi); 2912 MI.eraseFromParent(); 2913 return; 2914 } 2915 2916 B.setInstr(*Span.begin()); 2917 MI.eraseFromParent(); 2918 2919 // Figure out the point after the waterfall loop before mangling the control 2920 // flow. 2921 executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), 2922 OpsToWaterfall, MRI); 2923 2924 // The insertion point is now right after the original instruction. 2925 // 2926 // Keep the bitcast to the original vector type out of the loop. Doing this 2927 // saved an extra phi we don't need inside the loop. 2928 B.buildBitcast(DstReg, InsHi); 2929 2930 // Re-insert the constant offset add inside the waterfall loop. 2931 if (ShouldMoveIndexIntoLoop) 2932 reinsertVectorIndexAdd(B, *IdxLo, 1, ConstOffset); 2933 2934 return; 2935 } 2936 case AMDGPU::G_AMDGPU_BUFFER_LOAD: 2937 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: 2938 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: 2939 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: 2940 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: 2941 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: 2942 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: 2943 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: 2944 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: 2945 case AMDGPU::G_AMDGPU_BUFFER_STORE: 2946 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: 2947 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: 2948 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: 2949 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: 2950 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: 2951 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: { 2952 applyDefaultMapping(OpdMapper); 2953 executeInWaterfallLoop(MI, MRI, {1, 4}); 2954 return; 2955 } 2956 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: 2957 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: 2958 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: 2959 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: 2960 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: 2961 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: 2962 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: 2963 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: 2964 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: 2965 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: 2966 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: 2967 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: { 2968 applyDefaultMapping(OpdMapper); 2969 executeInWaterfallLoop(MI, MRI, {2, 5}); 2970 return; 2971 } 2972 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: { 2973 applyDefaultMapping(OpdMapper); 2974 executeInWaterfallLoop(MI, MRI, {1, 4}); 2975 return; 2976 } 2977 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { 2978 applyDefaultMapping(OpdMapper); 2979 executeInWaterfallLoop(MI, MRI, {3, 6}); 2980 return; 2981 } 2982 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: { 2983 applyMappingSBufferLoad(OpdMapper); 2984 return; 2985 } 2986 case AMDGPU::G_INTRINSIC: { 2987 switch (MI.getIntrinsicID()) { 2988 case Intrinsic::amdgcn_readlane: { 2989 substituteSimpleCopyRegs(OpdMapper, 2); 2990 2991 assert(OpdMapper.getVRegs(0).empty()); 2992 assert(OpdMapper.getVRegs(3).empty()); 2993 2994 // Make sure the index is an SGPR. It doesn't make sense to run this in a 2995 // waterfall loop, so assume it's a uniform value. 2996 constrainOpWithReadfirstlane(MI, MRI, 3); // Index 2997 return; 2998 } 2999 case Intrinsic::amdgcn_writelane: { 3000 assert(OpdMapper.getVRegs(0).empty()); 3001 assert(OpdMapper.getVRegs(2).empty()); 3002 assert(OpdMapper.getVRegs(3).empty()); 3003 3004 substituteSimpleCopyRegs(OpdMapper, 4); // VGPR input val 3005 constrainOpWithReadfirstlane(MI, MRI, 2); // Source value 3006 constrainOpWithReadfirstlane(MI, MRI, 3); // Index 3007 return; 3008 } 3009 case Intrinsic::amdgcn_interp_p1: 3010 case Intrinsic::amdgcn_interp_p2: 3011 case Intrinsic::amdgcn_interp_mov: 3012 case Intrinsic::amdgcn_interp_p1_f16: 3013 case Intrinsic::amdgcn_interp_p2_f16: { 3014 applyDefaultMapping(OpdMapper); 3015 3016 // Readlane for m0 value, which is always the last operand. 3017 // FIXME: Should this be a waterfall loop instead? 3018 constrainOpWithReadfirstlane(MI, MRI, MI.getNumOperands() - 1); // Index 3019 return; 3020 } 3021 case Intrinsic::amdgcn_permlane16: 3022 case Intrinsic::amdgcn_permlanex16: { 3023 // Doing a waterfall loop over these wouldn't make any sense. 3024 substituteSimpleCopyRegs(OpdMapper, 2); 3025 substituteSimpleCopyRegs(OpdMapper, 3); 3026 constrainOpWithReadfirstlane(MI, MRI, 4); 3027 constrainOpWithReadfirstlane(MI, MRI, 5); 3028 return; 3029 } 3030 case Intrinsic::amdgcn_sbfe: 3031 applyMappingBFEIntrinsic(OpdMapper, true); 3032 return; 3033 case Intrinsic::amdgcn_ubfe: 3034 applyMappingBFEIntrinsic(OpdMapper, false); 3035 return; 3036 case Intrinsic::amdgcn_ballot: 3037 // Use default handling and insert copy to vcc source. 3038 break; 3039 } 3040 break; 3041 } 3042 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: 3043 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: { 3044 const AMDGPU::RsrcIntrinsic *RSrcIntrin 3045 = AMDGPU::lookupRsrcIntrinsic(MI.getIntrinsicID()); 3046 assert(RSrcIntrin && RSrcIntrin->IsImage); 3047 // Non-images can have complications from operands that allow both SGPR 3048 // and VGPR. For now it's too complicated to figure out the final opcode 3049 // to derive the register bank from the MCInstrDesc. 3050 applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg); 3051 return; 3052 } 3053 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: { 3054 auto IntrID = MI.getIntrinsicID(); 3055 switch (IntrID) { 3056 case Intrinsic::amdgcn_ds_ordered_add: 3057 case Intrinsic::amdgcn_ds_ordered_swap: { 3058 // This is only allowed to execute with 1 lane, so readfirstlane is safe. 3059 assert(OpdMapper.getVRegs(0).empty()); 3060 substituteSimpleCopyRegs(OpdMapper, 3); 3061 constrainOpWithReadfirstlane(MI, MRI, 2); // M0 3062 return; 3063 } 3064 case Intrinsic::amdgcn_ds_gws_init: 3065 case Intrinsic::amdgcn_ds_gws_barrier: 3066 case Intrinsic::amdgcn_ds_gws_sema_br: { 3067 // Only the first lane is executes, so readfirstlane is safe. 3068 substituteSimpleCopyRegs(OpdMapper, 1); 3069 constrainOpWithReadfirstlane(MI, MRI, 2); // M0 3070 return; 3071 } 3072 case Intrinsic::amdgcn_ds_gws_sema_v: 3073 case Intrinsic::amdgcn_ds_gws_sema_p: 3074 case Intrinsic::amdgcn_ds_gws_sema_release_all: { 3075 // Only the first lane is executes, so readfirstlane is safe. 3076 constrainOpWithReadfirstlane(MI, MRI, 1); // M0 3077 return; 3078 } 3079 case Intrinsic::amdgcn_ds_append: 3080 case Intrinsic::amdgcn_ds_consume: { 3081 constrainOpWithReadfirstlane(MI, MRI, 2); // M0 3082 return; 3083 } 3084 case Intrinsic::amdgcn_s_sendmsg: 3085 case Intrinsic::amdgcn_s_sendmsghalt: { 3086 // FIXME: Should this use a waterfall loop? 3087 constrainOpWithReadfirstlane(MI, MRI, 2); // M0 3088 return; 3089 } 3090 case Intrinsic::amdgcn_s_setreg: { 3091 constrainOpWithReadfirstlane(MI, MRI, 2); 3092 return; 3093 } 3094 default: { 3095 if (const AMDGPU::RsrcIntrinsic *RSrcIntrin = 3096 AMDGPU::lookupRsrcIntrinsic(IntrID)) { 3097 // Non-images can have complications from operands that allow both SGPR 3098 // and VGPR. For now it's too complicated to figure out the final opcode 3099 // to derive the register bank from the MCInstrDesc. 3100 if (RSrcIntrin->IsImage) { 3101 applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg); 3102 return; 3103 } 3104 } 3105 3106 break; 3107 } 3108 } 3109 break; 3110 } 3111 case AMDGPU::G_LOAD: 3112 case AMDGPU::G_ZEXTLOAD: 3113 case AMDGPU::G_SEXTLOAD: { 3114 if (applyMappingLoad(MI, OpdMapper, MRI)) 3115 return; 3116 break; 3117 } 3118 case AMDGPU::G_DYN_STACKALLOC: 3119 applyMappingDynStackAlloc(MI, OpdMapper, MRI); 3120 return; 3121 default: 3122 break; 3123 } 3124 3125 return applyDefaultMapping(OpdMapper); 3126 } 3127 3128 // vgpr, sgpr -> vgpr 3129 // vgpr, agpr -> vgpr 3130 // agpr, agpr -> agpr 3131 // agpr, sgpr -> vgpr 3132 static unsigned regBankUnion(unsigned RB0, unsigned RB1) { 3133 if (RB0 == AMDGPU::InvalidRegBankID) 3134 return RB1; 3135 if (RB1 == AMDGPU::InvalidRegBankID) 3136 return RB0; 3137 3138 if (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID) 3139 return AMDGPU::SGPRRegBankID; 3140 3141 if (RB0 == AMDGPU::AGPRRegBankID && RB1 == AMDGPU::AGPRRegBankID) 3142 return AMDGPU::AGPRRegBankID; 3143 3144 return AMDGPU::VGPRRegBankID; 3145 } 3146 3147 static unsigned regBankBoolUnion(unsigned RB0, unsigned RB1) { 3148 if (RB0 == AMDGPU::InvalidRegBankID) 3149 return RB1; 3150 if (RB1 == AMDGPU::InvalidRegBankID) 3151 return RB0; 3152 3153 // vcc, vcc -> vcc 3154 // vcc, sgpr -> vcc 3155 // vcc, vgpr -> vcc 3156 if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID) 3157 return AMDGPU::VCCRegBankID; 3158 3159 // vcc, vgpr -> vgpr 3160 return regBankUnion(RB0, RB1); 3161 } 3162 3163 unsigned AMDGPURegisterBankInfo::getMappingType(const MachineRegisterInfo &MRI, 3164 const MachineInstr &MI) const { 3165 unsigned RegBank = AMDGPU::InvalidRegBankID; 3166 3167 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 3168 if (!MI.getOperand(i).isReg()) 3169 continue; 3170 Register Reg = MI.getOperand(i).getReg(); 3171 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) { 3172 RegBank = regBankUnion(RegBank, Bank->getID()); 3173 if (RegBank == AMDGPU::VGPRRegBankID) 3174 break; 3175 } 3176 } 3177 3178 return RegBank; 3179 } 3180 3181 bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const { 3182 const MachineFunction &MF = *MI.getParent()->getParent(); 3183 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3184 for (unsigned i = 0, e = MI.getNumOperands();i != e; ++i) { 3185 if (!MI.getOperand(i).isReg()) 3186 continue; 3187 Register Reg = MI.getOperand(i).getReg(); 3188 if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) { 3189 if (Bank->getID() != AMDGPU::SGPRRegBankID) 3190 return false; 3191 } 3192 } 3193 return true; 3194 } 3195 3196 const RegisterBankInfo::InstructionMapping & 3197 AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const { 3198 const MachineFunction &MF = *MI.getParent()->getParent(); 3199 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3200 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); 3201 3202 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 3203 const MachineOperand &SrcOp = MI.getOperand(i); 3204 if (!SrcOp.isReg()) 3205 continue; 3206 3207 unsigned Size = getSizeInBits(SrcOp.getReg(), MRI, *TRI); 3208 OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 3209 } 3210 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), 3211 MI.getNumOperands()); 3212 } 3213 3214 const RegisterBankInfo::InstructionMapping & 3215 AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const { 3216 const MachineFunction &MF = *MI.getParent()->getParent(); 3217 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3218 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); 3219 3220 // Even though we technically could use SGPRs, this would require knowledge of 3221 // the constant bus restriction. Force all sources to VGPR (except for VCC). 3222 // 3223 // TODO: Unary ops are trivially OK, so accept SGPRs? 3224 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 3225 const MachineOperand &Src = MI.getOperand(i); 3226 if (!Src.isReg()) 3227 continue; 3228 3229 unsigned Size = getSizeInBits(Src.getReg(), MRI, *TRI); 3230 unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID; 3231 OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size); 3232 } 3233 3234 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), 3235 MI.getNumOperands()); 3236 } 3237 3238 const RegisterBankInfo::InstructionMapping & 3239 AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const { 3240 const MachineFunction &MF = *MI.getParent()->getParent(); 3241 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3242 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); 3243 3244 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) { 3245 const MachineOperand &Op = MI.getOperand(I); 3246 if (!Op.isReg()) 3247 continue; 3248 3249 unsigned Size = getSizeInBits(Op.getReg(), MRI, *TRI); 3250 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3251 } 3252 3253 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), 3254 MI.getNumOperands()); 3255 } 3256 3257 const RegisterBankInfo::InstructionMapping & 3258 AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI, 3259 const MachineInstr &MI, 3260 int RsrcIdx) const { 3261 // The reported argument index is relative to the IR intrinsic call arguments, 3262 // so we need to shift by the number of defs and the intrinsic ID. 3263 RsrcIdx += MI.getNumExplicitDefs() + 1; 3264 3265 const int NumOps = MI.getNumOperands(); 3266 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps); 3267 3268 // TODO: Should packed/unpacked D16 difference be reported here as part of 3269 // the value mapping? 3270 for (int I = 0; I != NumOps; ++I) { 3271 if (!MI.getOperand(I).isReg()) 3272 continue; 3273 3274 Register OpReg = MI.getOperand(I).getReg(); 3275 // We replace some dead address operands with $noreg 3276 if (!OpReg) 3277 continue; 3278 3279 unsigned Size = getSizeInBits(OpReg, MRI, *TRI); 3280 3281 // FIXME: Probably need a new intrinsic register bank searchable table to 3282 // handle arbitrary intrinsics easily. 3283 // 3284 // If this has a sampler, it immediately follows rsrc. 3285 const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1; 3286 3287 if (MustBeSGPR) { 3288 // If this must be an SGPR, so we must report whatever it is as legal. 3289 unsigned NewBank = getRegBankID(OpReg, MRI, AMDGPU::SGPRRegBankID); 3290 OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size); 3291 } else { 3292 // Some operands must be VGPR, and these are easy to copy to. 3293 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3294 } 3295 } 3296 3297 return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), NumOps); 3298 } 3299 3300 /// Return the mapping for a pointer arugment. 3301 const RegisterBankInfo::ValueMapping * 3302 AMDGPURegisterBankInfo::getValueMappingForPtr(const MachineRegisterInfo &MRI, 3303 Register PtrReg) const { 3304 LLT PtrTy = MRI.getType(PtrReg); 3305 unsigned Size = PtrTy.getSizeInBits(); 3306 if (Subtarget.useFlatForGlobal() || 3307 !AMDGPU::isFlatGlobalAddrSpace(PtrTy.getAddressSpace())) 3308 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3309 3310 // If we're using MUBUF instructions for global memory, an SGPR base register 3311 // is possible. Otherwise this needs to be a VGPR. 3312 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI); 3313 return AMDGPU::getValueMapping(PtrBank->getID(), Size); 3314 } 3315 3316 const RegisterBankInfo::InstructionMapping & 3317 AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const { 3318 3319 const MachineFunction &MF = *MI.getParent()->getParent(); 3320 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3321 SmallVector<const ValueMapping*, 2> OpdsMapping(2); 3322 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 3323 Register PtrReg = MI.getOperand(1).getReg(); 3324 LLT PtrTy = MRI.getType(PtrReg); 3325 unsigned AS = PtrTy.getAddressSpace(); 3326 unsigned PtrSize = PtrTy.getSizeInBits(); 3327 3328 const ValueMapping *ValMapping; 3329 const ValueMapping *PtrMapping; 3330 3331 const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI); 3332 3333 if (PtrBank == &AMDGPU::SGPRRegBank && AMDGPU::isFlatGlobalAddrSpace(AS)) { 3334 if (isScalarLoadLegal(MI)) { 3335 // We have a uniform instruction so we want to use an SMRD load 3336 ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 3337 PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize); 3338 } else { 3339 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3340 3341 // If we're using MUBUF instructions for global memory, an SGPR base 3342 // register is possible. Otherwise this needs to be a VGPR. 3343 unsigned PtrBankID = Subtarget.useFlatForGlobal() ? 3344 AMDGPU::VGPRRegBankID : AMDGPU::SGPRRegBankID; 3345 3346 PtrMapping = AMDGPU::getValueMapping(PtrBankID, PtrSize); 3347 } 3348 } else { 3349 ValMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3350 PtrMapping = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, PtrSize); 3351 } 3352 3353 OpdsMapping[0] = ValMapping; 3354 OpdsMapping[1] = PtrMapping; 3355 const RegisterBankInfo::InstructionMapping &Mapping = getInstructionMapping( 3356 1, 1, getOperandsMapping(OpdsMapping), MI.getNumOperands()); 3357 return Mapping; 3358 3359 // FIXME: Do we want to add a mapping for FLAT load, or should we just 3360 // handle that during instruction selection? 3361 } 3362 3363 unsigned 3364 AMDGPURegisterBankInfo::getRegBankID(Register Reg, 3365 const MachineRegisterInfo &MRI, 3366 unsigned Default) const { 3367 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI); 3368 return Bank ? Bank->getID() : Default; 3369 } 3370 3371 const RegisterBankInfo::ValueMapping * 3372 AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg, 3373 const MachineRegisterInfo &MRI, 3374 const TargetRegisterInfo &TRI) const { 3375 // Lie and claim anything is legal, even though this needs to be an SGPR 3376 // applyMapping will have to deal with it as a waterfall loop. 3377 unsigned Bank = getRegBankID(Reg, MRI, AMDGPU::SGPRRegBankID); 3378 unsigned Size = getSizeInBits(Reg, MRI, TRI); 3379 return AMDGPU::getValueMapping(Bank, Size); 3380 } 3381 3382 const RegisterBankInfo::ValueMapping * 3383 AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg, 3384 const MachineRegisterInfo &MRI, 3385 const TargetRegisterInfo &TRI) const { 3386 unsigned Size = getSizeInBits(Reg, MRI, TRI); 3387 return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3388 } 3389 3390 const RegisterBankInfo::ValueMapping * 3391 AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg, 3392 const MachineRegisterInfo &MRI, 3393 const TargetRegisterInfo &TRI) const { 3394 unsigned Size = getSizeInBits(Reg, MRI, TRI); 3395 return AMDGPU::getValueMapping(AMDGPU::AGPRRegBankID, Size); 3396 } 3397 3398 /// 3399 /// This function must return a legal mapping, because 3400 /// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called 3401 /// in RegBankSelect::Mode::Fast. Any mapping that would cause a 3402 /// VGPR to SGPR generated is illegal. 3403 /// 3404 // Operands that must be SGPRs must accept potentially divergent VGPRs as 3405 // legal. These will be dealt with in applyMappingImpl. 3406 // 3407 const RegisterBankInfo::InstructionMapping & 3408 AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { 3409 const MachineFunction &MF = *MI.getParent()->getParent(); 3410 const MachineRegisterInfo &MRI = MF.getRegInfo(); 3411 3412 if (MI.isCopy() || MI.getOpcode() == AMDGPU::G_FREEZE) { 3413 // The default logic bothers to analyze impossible alternative mappings. We 3414 // want the most straightforward mapping, so just directly handle this. 3415 const RegisterBank *DstBank = getRegBank(MI.getOperand(0).getReg(), MRI, 3416 *TRI); 3417 const RegisterBank *SrcBank = getRegBank(MI.getOperand(1).getReg(), MRI, 3418 *TRI); 3419 assert(SrcBank && "src bank should have been assigned already"); 3420 if (!DstBank) 3421 DstBank = SrcBank; 3422 3423 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 3424 if (cannotCopy(*DstBank, *SrcBank, Size)) 3425 return getInvalidInstructionMapping(); 3426 3427 const ValueMapping &ValMap = getValueMapping(0, Size, *DstBank); 3428 unsigned OpdsMappingSize = MI.isCopy() ? 1 : 2; 3429 SmallVector<const ValueMapping *, 1> OpdsMapping(OpdsMappingSize); 3430 OpdsMapping[0] = &ValMap; 3431 if (MI.getOpcode() == AMDGPU::G_FREEZE) 3432 OpdsMapping[1] = &ValMap; 3433 3434 return getInstructionMapping( 3435 1, /*Cost*/ 1, 3436 /*OperandsMapping*/ getOperandsMapping(OpdsMapping), OpdsMappingSize); 3437 } 3438 3439 if (MI.isRegSequence()) { 3440 // If any input is a VGPR, the result must be a VGPR. The default handling 3441 // assumes any copy between banks is legal. 3442 unsigned BankID = AMDGPU::SGPRRegBankID; 3443 3444 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 3445 auto OpBank = getRegBankID(MI.getOperand(I).getReg(), MRI); 3446 // It doesn't make sense to use vcc or scc banks here, so just ignore 3447 // them. 3448 if (OpBank != AMDGPU::SGPRRegBankID) { 3449 BankID = AMDGPU::VGPRRegBankID; 3450 break; 3451 } 3452 } 3453 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 3454 3455 const ValueMapping &ValMap = getValueMapping(0, Size, getRegBank(BankID)); 3456 return getInstructionMapping( 3457 1, /*Cost*/ 1, 3458 /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1); 3459 } 3460 3461 // The default handling is broken and doesn't handle illegal SGPR->VGPR copies 3462 // properly. 3463 // 3464 // TODO: There are additional exec masking dependencies to analyze. 3465 if (MI.getOpcode() == TargetOpcode::G_PHI) { 3466 unsigned ResultBank = AMDGPU::InvalidRegBankID; 3467 Register DstReg = MI.getOperand(0).getReg(); 3468 3469 // Sometimes the result may have already been assigned a bank. 3470 if (const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI)) 3471 ResultBank = DstBank->getID(); 3472 3473 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 3474 Register Reg = MI.getOperand(I).getReg(); 3475 const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI); 3476 3477 // FIXME: Assuming VGPR for any undetermined inputs. 3478 if (!Bank || Bank->getID() == AMDGPU::VGPRRegBankID) { 3479 ResultBank = AMDGPU::VGPRRegBankID; 3480 break; 3481 } 3482 3483 // FIXME: Need to promote SGPR case to s32 3484 unsigned OpBank = Bank->getID(); 3485 ResultBank = regBankBoolUnion(ResultBank, OpBank); 3486 } 3487 3488 assert(ResultBank != AMDGPU::InvalidRegBankID); 3489 3490 unsigned Size = MRI.getType(DstReg).getSizeInBits(); 3491 3492 const ValueMapping &ValMap = 3493 getValueMapping(0, Size, getRegBank(ResultBank)); 3494 return getInstructionMapping( 3495 1, /*Cost*/ 1, 3496 /*OperandsMapping*/ getOperandsMapping({&ValMap}), 1); 3497 } 3498 3499 const RegisterBankInfo::InstructionMapping &Mapping = getInstrMappingImpl(MI); 3500 if (Mapping.isValid()) 3501 return Mapping; 3502 3503 SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands()); 3504 3505 switch (MI.getOpcode()) { 3506 default: 3507 return getInvalidInstructionMapping(); 3508 3509 case AMDGPU::G_AND: 3510 case AMDGPU::G_OR: 3511 case AMDGPU::G_XOR: { 3512 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3513 if (Size == 1) { 3514 const RegisterBank *DstBank 3515 = getRegBank(MI.getOperand(0).getReg(), MRI, *TRI); 3516 3517 unsigned TargetBankID = AMDGPU::InvalidRegBankID; 3518 unsigned BankLHS = AMDGPU::InvalidRegBankID; 3519 unsigned BankRHS = AMDGPU::InvalidRegBankID; 3520 if (DstBank) { 3521 TargetBankID = DstBank->getID(); 3522 if (DstBank == &AMDGPU::VCCRegBank) { 3523 TargetBankID = AMDGPU::VCCRegBankID; 3524 BankLHS = AMDGPU::VCCRegBankID; 3525 BankRHS = AMDGPU::VCCRegBankID; 3526 } else { 3527 BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, 3528 AMDGPU::SGPRRegBankID); 3529 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, 3530 AMDGPU::SGPRRegBankID); 3531 } 3532 } else { 3533 BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, 3534 AMDGPU::VCCRegBankID); 3535 BankRHS = getRegBankID(MI.getOperand(2).getReg(), MRI, 3536 AMDGPU::VCCRegBankID); 3537 3538 // Both inputs should be true booleans to produce a boolean result. 3539 if (BankLHS == AMDGPU::VGPRRegBankID || BankRHS == AMDGPU::VGPRRegBankID) { 3540 TargetBankID = AMDGPU::VGPRRegBankID; 3541 } else if (BankLHS == AMDGPU::VCCRegBankID || BankRHS == AMDGPU::VCCRegBankID) { 3542 TargetBankID = AMDGPU::VCCRegBankID; 3543 BankLHS = AMDGPU::VCCRegBankID; 3544 BankRHS = AMDGPU::VCCRegBankID; 3545 } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) { 3546 TargetBankID = AMDGPU::SGPRRegBankID; 3547 } 3548 } 3549 3550 OpdsMapping[0] = AMDGPU::getValueMapping(TargetBankID, Size); 3551 OpdsMapping[1] = AMDGPU::getValueMapping(BankLHS, Size); 3552 OpdsMapping[2] = AMDGPU::getValueMapping(BankRHS, Size); 3553 break; 3554 } 3555 3556 if (Size == 64) { 3557 3558 if (isSALUMapping(MI)) { 3559 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size); 3560 OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0]; 3561 } else { 3562 OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size); 3563 unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI /*, DefaultBankID*/); 3564 OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size); 3565 3566 unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI /*, DefaultBankID*/); 3567 OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size); 3568 } 3569 3570 break; 3571 } 3572 3573 LLVM_FALLTHROUGH; 3574 } 3575 case AMDGPU::G_PTR_ADD: 3576 case AMDGPU::G_PTRMASK: 3577 case AMDGPU::G_ADD: 3578 case AMDGPU::G_SUB: 3579 case AMDGPU::G_MUL: 3580 case AMDGPU::G_SHL: 3581 case AMDGPU::G_LSHR: 3582 case AMDGPU::G_ASHR: 3583 case AMDGPU::G_UADDO: 3584 case AMDGPU::G_USUBO: 3585 case AMDGPU::G_UADDE: 3586 case AMDGPU::G_SADDE: 3587 case AMDGPU::G_USUBE: 3588 case AMDGPU::G_SSUBE: 3589 case AMDGPU::G_SMIN: 3590 case AMDGPU::G_SMAX: 3591 case AMDGPU::G_UMIN: 3592 case AMDGPU::G_UMAX: 3593 case AMDGPU::G_SHUFFLE_VECTOR: 3594 if (isSALUMapping(MI)) 3595 return getDefaultMappingSOP(MI); 3596 LLVM_FALLTHROUGH; 3597 3598 case AMDGPU::G_SADDSAT: // FIXME: Could lower sat ops for SALU 3599 case AMDGPU::G_SSUBSAT: 3600 case AMDGPU::G_UADDSAT: 3601 case AMDGPU::G_USUBSAT: 3602 case AMDGPU::G_FADD: 3603 case AMDGPU::G_FSUB: 3604 case AMDGPU::G_FPTOSI: 3605 case AMDGPU::G_FPTOUI: 3606 case AMDGPU::G_FMUL: 3607 case AMDGPU::G_FMA: 3608 case AMDGPU::G_FMAD: 3609 case AMDGPU::G_FSQRT: 3610 case AMDGPU::G_FFLOOR: 3611 case AMDGPU::G_FCEIL: 3612 case AMDGPU::G_FRINT: 3613 case AMDGPU::G_SITOFP: 3614 case AMDGPU::G_UITOFP: 3615 case AMDGPU::G_FPTRUNC: 3616 case AMDGPU::G_FPEXT: 3617 case AMDGPU::G_FEXP2: 3618 case AMDGPU::G_FLOG2: 3619 case AMDGPU::G_FMINNUM: 3620 case AMDGPU::G_FMAXNUM: 3621 case AMDGPU::G_FMINNUM_IEEE: 3622 case AMDGPU::G_FMAXNUM_IEEE: 3623 case AMDGPU::G_FCANONICALIZE: 3624 case AMDGPU::G_INTRINSIC_TRUNC: 3625 case AMDGPU::G_BSWAP: // TODO: Somehow expand for scalar? 3626 case AMDGPU::G_FSHR: // TODO: Expand for scalar 3627 case AMDGPU::G_AMDGPU_FFBH_U32: 3628 case AMDGPU::G_AMDGPU_FMIN_LEGACY: 3629 case AMDGPU::G_AMDGPU_FMAX_LEGACY: 3630 case AMDGPU::G_AMDGPU_RCP_IFLAG: 3631 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE0: 3632 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE1: 3633 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE2: 3634 case AMDGPU::G_AMDGPU_CVT_F32_UBYTE3: 3635 return getDefaultMappingVOP(MI); 3636 case AMDGPU::G_UMULH: 3637 case AMDGPU::G_SMULH: { 3638 if (Subtarget.hasScalarMulHiInsts() && isSALUMapping(MI)) 3639 return getDefaultMappingSOP(MI); 3640 return getDefaultMappingVOP(MI); 3641 } 3642 case AMDGPU::G_IMPLICIT_DEF: { 3643 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3644 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 3645 break; 3646 } 3647 case AMDGPU::G_FCONSTANT: 3648 case AMDGPU::G_CONSTANT: 3649 case AMDGPU::G_GLOBAL_VALUE: 3650 case AMDGPU::G_BLOCK_ADDR: 3651 case AMDGPU::G_READCYCLECOUNTER: { 3652 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3653 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 3654 break; 3655 } 3656 case AMDGPU::G_FRAME_INDEX: { 3657 // TODO: This should be the same as other constants, but eliminateFrameIndex 3658 // currently assumes VALU uses. 3659 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3660 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3661 break; 3662 } 3663 case AMDGPU::G_DYN_STACKALLOC: { 3664 // Result is always uniform, and a wave reduction is needed for the source. 3665 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32); 3666 unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3667 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, 32); 3668 break; 3669 } 3670 case AMDGPU::G_INSERT: { 3671 unsigned BankID = getMappingType(MRI, MI); 3672 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 3673 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); 3674 unsigned EltSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI); 3675 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize); 3676 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize); 3677 OpdsMapping[2] = AMDGPU::getValueMapping(BankID, EltSize); 3678 OpdsMapping[3] = nullptr; 3679 break; 3680 } 3681 case AMDGPU::G_EXTRACT: { 3682 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3683 unsigned DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 3684 unsigned SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); 3685 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, DstSize); 3686 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, SrcSize); 3687 OpdsMapping[2] = nullptr; 3688 break; 3689 } 3690 case AMDGPU::G_BUILD_VECTOR: 3691 case AMDGPU::G_BUILD_VECTOR_TRUNC: { 3692 LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); 3693 if (DstTy == LLT::vector(2, 16)) { 3694 unsigned DstSize = DstTy.getSizeInBits(); 3695 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 3696 unsigned Src0BankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3697 unsigned Src1BankID = getRegBankID(MI.getOperand(2).getReg(), MRI); 3698 unsigned DstBankID = regBankUnion(Src0BankID, Src1BankID); 3699 3700 OpdsMapping[0] = AMDGPU::getValueMapping(DstBankID, DstSize); 3701 OpdsMapping[1] = AMDGPU::getValueMapping(Src0BankID, SrcSize); 3702 OpdsMapping[2] = AMDGPU::getValueMapping(Src1BankID, SrcSize); 3703 break; 3704 } 3705 3706 LLVM_FALLTHROUGH; 3707 } 3708 case AMDGPU::G_MERGE_VALUES: 3709 case AMDGPU::G_CONCAT_VECTORS: { 3710 unsigned Bank = getMappingType(MRI, MI); 3711 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3712 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 3713 3714 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); 3715 // Op1 and Dst should use the same register bank. 3716 for (unsigned i = 1, e = MI.getNumOperands(); i != e; ++i) 3717 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, SrcSize); 3718 break; 3719 } 3720 case AMDGPU::G_BITCAST: 3721 case AMDGPU::G_INTTOPTR: 3722 case AMDGPU::G_PTRTOINT: 3723 case AMDGPU::G_BITREVERSE: 3724 case AMDGPU::G_FABS: 3725 case AMDGPU::G_FNEG: { 3726 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3727 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3728 OpdsMapping[0] = OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); 3729 break; 3730 } 3731 case AMDGPU::G_CTLZ_ZERO_UNDEF: 3732 case AMDGPU::G_CTTZ_ZERO_UNDEF: 3733 case AMDGPU::G_CTPOP: { 3734 unsigned Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 3735 unsigned BankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3736 OpdsMapping[0] = AMDGPU::getValueMapping(BankID, 32); 3737 3738 // This should really be getValueMappingSGPR64Only, but allowing the generic 3739 // code to handle the register split just makes using LegalizerHelper more 3740 // difficult. 3741 OpdsMapping[1] = AMDGPU::getValueMapping(BankID, Size); 3742 break; 3743 } 3744 case AMDGPU::G_TRUNC: { 3745 Register Dst = MI.getOperand(0).getReg(); 3746 Register Src = MI.getOperand(1).getReg(); 3747 unsigned Bank = getRegBankID(Src, MRI); 3748 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI); 3749 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI); 3750 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); 3751 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize); 3752 break; 3753 } 3754 case AMDGPU::G_ZEXT: 3755 case AMDGPU::G_SEXT: 3756 case AMDGPU::G_ANYEXT: 3757 case AMDGPU::G_SEXT_INREG: { 3758 Register Dst = MI.getOperand(0).getReg(); 3759 Register Src = MI.getOperand(1).getReg(); 3760 unsigned DstSize = getSizeInBits(Dst, MRI, *TRI); 3761 unsigned SrcSize = getSizeInBits(Src, MRI, *TRI); 3762 3763 unsigned DstBank; 3764 const RegisterBank *SrcBank = getRegBank(Src, MRI, *TRI); 3765 assert(SrcBank); 3766 switch (SrcBank->getID()) { 3767 case AMDGPU::SGPRRegBankID: 3768 DstBank = AMDGPU::SGPRRegBankID; 3769 break; 3770 default: 3771 DstBank = AMDGPU::VGPRRegBankID; 3772 break; 3773 } 3774 3775 // Scalar extend can use 64-bit BFE, but VGPRs require extending to 3776 // 32-bits, and then to 64. 3777 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(DstBank, DstSize); 3778 OpdsMapping[1] = AMDGPU::getValueMappingSGPR64Only(SrcBank->getID(), 3779 SrcSize); 3780 break; 3781 } 3782 case AMDGPU::G_FCMP: { 3783 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 3784 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI); 3785 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); 3786 OpdsMapping[1] = nullptr; // Predicate Operand. 3787 OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size); 3788 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3789 break; 3790 } 3791 case AMDGPU::G_STORE: { 3792 assert(MI.getOperand(0).isReg()); 3793 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3794 3795 // FIXME: We need to specify a different reg bank once scalar stores are 3796 // supported. 3797 const ValueMapping *ValMapping = 3798 AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 3799 OpdsMapping[0] = ValMapping; 3800 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); 3801 break; 3802 } 3803 case AMDGPU::G_ICMP: { 3804 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate()); 3805 unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 3806 3807 // See if the result register has already been constrained to vcc, which may 3808 // happen due to control flow intrinsic lowering. 3809 unsigned DstBank = getRegBankID(MI.getOperand(0).getReg(), MRI, 3810 AMDGPU::SGPRRegBankID); 3811 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI); 3812 unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI); 3813 3814 bool CanUseSCC = DstBank == AMDGPU::SGPRRegBankID && 3815 Op2Bank == AMDGPU::SGPRRegBankID && 3816 Op3Bank == AMDGPU::SGPRRegBankID && 3817 (Size == 32 || (Size == 64 && 3818 (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) && 3819 Subtarget.hasScalarCompareEq64())); 3820 3821 DstBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; 3822 unsigned SrcBank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; 3823 3824 // TODO: Use 32-bit for scalar output size. 3825 // SCC results will need to be copied to a 32-bit SGPR virtual register. 3826 const unsigned ResultSize = 1; 3827 3828 OpdsMapping[0] = AMDGPU::getValueMapping(DstBank, ResultSize); 3829 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, Size); 3830 OpdsMapping[3] = AMDGPU::getValueMapping(SrcBank, Size); 3831 break; 3832 } 3833 case AMDGPU::G_EXTRACT_VECTOR_ELT: { 3834 // VGPR index can be used for waterfall when indexing a SGPR vector. 3835 unsigned SrcBankID = getRegBankID(MI.getOperand(1).getReg(), MRI); 3836 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3837 unsigned SrcSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 3838 unsigned IdxSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 3839 unsigned IdxBank = getRegBankID(MI.getOperand(2).getReg(), MRI); 3840 unsigned OutputBankID = regBankUnion(SrcBankID, IdxBank); 3841 3842 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(OutputBankID, DstSize); 3843 OpdsMapping[1] = AMDGPU::getValueMapping(SrcBankID, SrcSize); 3844 3845 // The index can be either if the source vector is VGPR. 3846 OpdsMapping[2] = AMDGPU::getValueMapping(IdxBank, IdxSize); 3847 break; 3848 } 3849 case AMDGPU::G_INSERT_VECTOR_ELT: { 3850 unsigned OutputBankID = isSALUMapping(MI) ? 3851 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; 3852 3853 unsigned VecSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 3854 unsigned InsertSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 3855 unsigned IdxSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits(); 3856 unsigned InsertEltBankID = getRegBankID(MI.getOperand(2).getReg(), MRI); 3857 unsigned IdxBankID = getRegBankID(MI.getOperand(3).getReg(), MRI); 3858 3859 OpdsMapping[0] = AMDGPU::getValueMapping(OutputBankID, VecSize); 3860 OpdsMapping[1] = AMDGPU::getValueMapping(OutputBankID, VecSize); 3861 3862 // This is a weird case, because we need to break down the mapping based on 3863 // the register bank of a different operand. 3864 if (InsertSize == 64 && OutputBankID == AMDGPU::VGPRRegBankID) { 3865 OpdsMapping[2] = AMDGPU::getValueMappingSplit64(InsertEltBankID, 3866 InsertSize); 3867 } else { 3868 assert(InsertSize == 32 || InsertSize == 64); 3869 OpdsMapping[2] = AMDGPU::getValueMapping(InsertEltBankID, InsertSize); 3870 } 3871 3872 // The index can be either if the source vector is VGPR. 3873 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBankID, IdxSize); 3874 break; 3875 } 3876 case AMDGPU::G_UNMERGE_VALUES: { 3877 unsigned Bank = getMappingType(MRI, MI); 3878 3879 // Op1 and Dst should use the same register bank. 3880 // FIXME: Shouldn't this be the default? Why do we need to handle this? 3881 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 3882 unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI); 3883 OpdsMapping[i] = AMDGPU::getValueMapping(Bank, Size); 3884 } 3885 break; 3886 } 3887 case AMDGPU::G_AMDGPU_BUFFER_LOAD: 3888 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE: 3889 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE: 3890 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT: 3891 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT: 3892 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT: 3893 case AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16: 3894 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT: 3895 case AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16: 3896 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT: 3897 case AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16: 3898 case AMDGPU::G_AMDGPU_BUFFER_STORE: 3899 case AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE: 3900 case AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT: 3901 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT: 3902 case AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16: { 3903 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 3904 3905 // rsrc 3906 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 3907 3908 // vindex 3909 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 3910 3911 // voffset 3912 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 3913 3914 // soffset 3915 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 3916 3917 // Any remaining operands are immediates and were correctly null 3918 // initialized. 3919 break; 3920 } 3921 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP: 3922 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD: 3923 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB: 3924 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN: 3925 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN: 3926 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX: 3927 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX: 3928 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND: 3929 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR: 3930 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR: 3931 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC: 3932 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC: { 3933 // vdata_out 3934 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 3935 3936 // vdata_in 3937 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 3938 3939 // rsrc 3940 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 3941 3942 // vindex 3943 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 3944 3945 // voffset 3946 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 3947 3948 // soffset 3949 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); 3950 3951 // Any remaining operands are immediates and were correctly null 3952 // initialized. 3953 break; 3954 } 3955 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD: { 3956 // vdata_in 3957 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 3958 3959 // rsrc 3960 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 3961 3962 // vindex 3963 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 3964 3965 // voffset 3966 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 3967 3968 // soffset 3969 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 3970 break; 3971 } 3972 case AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP: { 3973 // vdata_out 3974 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 3975 3976 // vdata_in 3977 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 3978 3979 // cmp 3980 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 3981 3982 // rsrc 3983 OpdsMapping[3] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 3984 3985 // vindex 3986 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 3987 3988 // voffset 3989 OpdsMapping[5] = getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); 3990 3991 // soffset 3992 OpdsMapping[6] = getSGPROpMapping(MI.getOperand(6).getReg(), MRI, *TRI); 3993 3994 // Any remaining operands are immediates and were correctly null 3995 // initialized. 3996 break; 3997 } 3998 case AMDGPU::G_AMDGPU_S_BUFFER_LOAD: { 3999 // Lie and claim everything is legal, even though some need to be 4000 // SGPRs. applyMapping will have to deal with it as a waterfall loop. 4001 OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 4002 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4003 4004 // We need to convert this to a MUBUF if either the resource of offset is 4005 // VGPR. 4006 unsigned RSrcBank = OpdsMapping[1]->BreakDown[0].RegBank->getID(); 4007 unsigned OffsetBank = OpdsMapping[2]->BreakDown[0].RegBank->getID(); 4008 unsigned ResultBank = regBankUnion(RSrcBank, OffsetBank); 4009 4010 unsigned Size0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4011 OpdsMapping[0] = AMDGPU::getValueMapping(ResultBank, Size0); 4012 break; 4013 } 4014 case AMDGPU::G_INTRINSIC: { 4015 switch (MI.getIntrinsicID()) { 4016 default: 4017 return getInvalidInstructionMapping(); 4018 case Intrinsic::amdgcn_div_fmas: 4019 case Intrinsic::amdgcn_div_fixup: 4020 case Intrinsic::amdgcn_trig_preop: 4021 case Intrinsic::amdgcn_sin: 4022 case Intrinsic::amdgcn_cos: 4023 case Intrinsic::amdgcn_log_clamp: 4024 case Intrinsic::amdgcn_rcp: 4025 case Intrinsic::amdgcn_rcp_legacy: 4026 case Intrinsic::amdgcn_sqrt: 4027 case Intrinsic::amdgcn_rsq: 4028 case Intrinsic::amdgcn_rsq_legacy: 4029 case Intrinsic::amdgcn_rsq_clamp: 4030 case Intrinsic::amdgcn_fmul_legacy: 4031 case Intrinsic::amdgcn_ldexp: 4032 case Intrinsic::amdgcn_frexp_mant: 4033 case Intrinsic::amdgcn_frexp_exp: 4034 case Intrinsic::amdgcn_fract: 4035 case Intrinsic::amdgcn_cvt_pkrtz: 4036 case Intrinsic::amdgcn_cvt_pknorm_i16: 4037 case Intrinsic::amdgcn_cvt_pknorm_u16: 4038 case Intrinsic::amdgcn_cvt_pk_i16: 4039 case Intrinsic::amdgcn_cvt_pk_u16: 4040 case Intrinsic::amdgcn_fmed3: 4041 case Intrinsic::amdgcn_cubeid: 4042 case Intrinsic::amdgcn_cubema: 4043 case Intrinsic::amdgcn_cubesc: 4044 case Intrinsic::amdgcn_cubetc: 4045 case Intrinsic::amdgcn_sffbh: 4046 case Intrinsic::amdgcn_fmad_ftz: 4047 case Intrinsic::amdgcn_mbcnt_lo: 4048 case Intrinsic::amdgcn_mbcnt_hi: 4049 case Intrinsic::amdgcn_mul_u24: 4050 case Intrinsic::amdgcn_mul_i24: 4051 case Intrinsic::amdgcn_lerp: 4052 case Intrinsic::amdgcn_sad_u8: 4053 case Intrinsic::amdgcn_msad_u8: 4054 case Intrinsic::amdgcn_sad_hi_u8: 4055 case Intrinsic::amdgcn_sad_u16: 4056 case Intrinsic::amdgcn_qsad_pk_u16_u8: 4057 case Intrinsic::amdgcn_mqsad_pk_u16_u8: 4058 case Intrinsic::amdgcn_mqsad_u32_u8: 4059 case Intrinsic::amdgcn_cvt_pk_u8_f32: 4060 case Intrinsic::amdgcn_alignbit: 4061 case Intrinsic::amdgcn_alignbyte: 4062 case Intrinsic::amdgcn_fdot2: 4063 case Intrinsic::amdgcn_sdot2: 4064 case Intrinsic::amdgcn_udot2: 4065 case Intrinsic::amdgcn_sdot4: 4066 case Intrinsic::amdgcn_udot4: 4067 case Intrinsic::amdgcn_sdot8: 4068 case Intrinsic::amdgcn_udot8: 4069 return getDefaultMappingVOP(MI); 4070 case Intrinsic::amdgcn_sbfe: 4071 case Intrinsic::amdgcn_ubfe: 4072 if (isSALUMapping(MI)) 4073 return getDefaultMappingSOP(MI); 4074 return getDefaultMappingVOP(MI); 4075 case Intrinsic::amdgcn_ds_swizzle: 4076 case Intrinsic::amdgcn_ds_permute: 4077 case Intrinsic::amdgcn_ds_bpermute: 4078 case Intrinsic::amdgcn_update_dpp: 4079 case Intrinsic::amdgcn_mov_dpp8: 4080 case Intrinsic::amdgcn_mov_dpp: 4081 case Intrinsic::amdgcn_wwm: 4082 case Intrinsic::amdgcn_wqm: 4083 case Intrinsic::amdgcn_softwqm: 4084 case Intrinsic::amdgcn_set_inactive: 4085 return getDefaultMappingAllVGPR(MI); 4086 case Intrinsic::amdgcn_kernarg_segment_ptr: 4087 case Intrinsic::amdgcn_s_getpc: 4088 case Intrinsic::amdgcn_groupstaticsize: 4089 case Intrinsic::amdgcn_reloc_constant: 4090 case Intrinsic::returnaddress: { 4091 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4092 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4093 break; 4094 } 4095 case Intrinsic::amdgcn_wqm_vote: { 4096 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4097 OpdsMapping[0] = OpdsMapping[2] 4098 = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size); 4099 break; 4100 } 4101 case Intrinsic::amdgcn_ps_live: { 4102 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); 4103 break; 4104 } 4105 case Intrinsic::amdgcn_div_scale: { 4106 unsigned Dst0Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4107 unsigned Dst1Size = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); 4108 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Dst0Size); 4109 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Dst1Size); 4110 4111 unsigned SrcSize = MRI.getType(MI.getOperand(3).getReg()).getSizeInBits(); 4112 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); 4113 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); 4114 break; 4115 } 4116 case Intrinsic::amdgcn_class: { 4117 Register Src0Reg = MI.getOperand(2).getReg(); 4118 Register Src1Reg = MI.getOperand(3).getReg(); 4119 unsigned Src0Size = MRI.getType(Src0Reg).getSizeInBits(); 4120 unsigned Src1Size = MRI.getType(Src1Reg).getSizeInBits(); 4121 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4122 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize); 4123 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src0Size); 4124 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Src1Size); 4125 break; 4126 } 4127 case Intrinsic::amdgcn_icmp: 4128 case Intrinsic::amdgcn_fcmp: { 4129 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4130 // This is not VCCRegBank because this is not used in boolean contexts. 4131 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); 4132 unsigned OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 4133 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); 4134 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, OpSize); 4135 break; 4136 } 4137 case Intrinsic::amdgcn_readlane: { 4138 // This must be an SGPR, but accept a VGPR. 4139 Register IdxReg = MI.getOperand(3).getReg(); 4140 unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits(); 4141 unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID); 4142 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize); 4143 LLVM_FALLTHROUGH; 4144 } 4145 case Intrinsic::amdgcn_readfirstlane: { 4146 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4147 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 4148 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); 4149 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); 4150 break; 4151 } 4152 case Intrinsic::amdgcn_writelane: { 4153 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4154 Register SrcReg = MI.getOperand(2).getReg(); 4155 unsigned SrcSize = MRI.getType(SrcReg).getSizeInBits(); 4156 unsigned SrcBank = getRegBankID(SrcReg, MRI, AMDGPU::SGPRRegBankID); 4157 Register IdxReg = MI.getOperand(3).getReg(); 4158 unsigned IdxSize = MRI.getType(IdxReg).getSizeInBits(); 4159 unsigned IdxBank = getRegBankID(IdxReg, MRI, AMDGPU::SGPRRegBankID); 4160 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); 4161 4162 // These 2 must be SGPRs, but accept VGPRs. Readfirstlane will be inserted 4163 // to legalize. 4164 OpdsMapping[2] = AMDGPU::getValueMapping(SrcBank, SrcSize); 4165 OpdsMapping[3] = AMDGPU::getValueMapping(IdxBank, IdxSize); 4166 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, SrcSize); 4167 break; 4168 } 4169 case Intrinsic::amdgcn_if_break: { 4170 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 4171 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4172 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); 4173 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4174 break; 4175 } 4176 case Intrinsic::amdgcn_permlane16: 4177 case Intrinsic::amdgcn_permlanex16: { 4178 unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); 4179 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 4180 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 4181 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); 4182 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4183 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4184 break; 4185 } 4186 case Intrinsic::amdgcn_mfma_f32_4x4x1f32: 4187 case Intrinsic::amdgcn_mfma_f32_4x4x4f16: 4188 case Intrinsic::amdgcn_mfma_i32_4x4x4i8: 4189 case Intrinsic::amdgcn_mfma_f32_4x4x2bf16: 4190 case Intrinsic::amdgcn_mfma_f32_16x16x1f32: 4191 case Intrinsic::amdgcn_mfma_f32_16x16x4f32: 4192 case Intrinsic::amdgcn_mfma_f32_16x16x4f16: 4193 case Intrinsic::amdgcn_mfma_f32_16x16x16f16: 4194 case Intrinsic::amdgcn_mfma_i32_16x16x4i8: 4195 case Intrinsic::amdgcn_mfma_i32_16x16x16i8: 4196 case Intrinsic::amdgcn_mfma_f32_16x16x2bf16: 4197 case Intrinsic::amdgcn_mfma_f32_16x16x8bf16: 4198 case Intrinsic::amdgcn_mfma_f32_32x32x1f32: 4199 case Intrinsic::amdgcn_mfma_f32_32x32x2f32: 4200 case Intrinsic::amdgcn_mfma_f32_32x32x4f16: 4201 case Intrinsic::amdgcn_mfma_f32_32x32x8f16: 4202 case Intrinsic::amdgcn_mfma_i32_32x32x4i8: 4203 case Intrinsic::amdgcn_mfma_i32_32x32x8i8: 4204 case Intrinsic::amdgcn_mfma_f32_32x32x2bf16: 4205 case Intrinsic::amdgcn_mfma_f32_32x32x4bf16: { 4206 // Default for MAI intrinsics. 4207 // srcC can also be an immediate which can be folded later. 4208 // FIXME: Should we eventually add an alternative mapping with AGPR src 4209 // for srcA/srcB? 4210 // 4211 // vdst, srcA, srcB, srcC 4212 OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 4213 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4214 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4215 OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4216 break; 4217 } 4218 case Intrinsic::amdgcn_interp_p1: 4219 case Intrinsic::amdgcn_interp_p2: 4220 case Intrinsic::amdgcn_interp_mov: 4221 case Intrinsic::amdgcn_interp_p1_f16: 4222 case Intrinsic::amdgcn_interp_p2_f16: { 4223 const int M0Idx = MI.getNumOperands() - 1; 4224 Register M0Reg = MI.getOperand(M0Idx).getReg(); 4225 unsigned M0Bank = getRegBankID(M0Reg, MRI, AMDGPU::SGPRRegBankID); 4226 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4227 4228 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); 4229 for (int I = 2; I != M0Idx && MI.getOperand(I).isReg(); ++I) 4230 OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4231 4232 // Must be SGPR, but we must take whatever the original bank is and fix it 4233 // later. 4234 OpdsMapping[M0Idx] = AMDGPU::getValueMapping(M0Bank, 32); 4235 break; 4236 } 4237 case Intrinsic::amdgcn_ballot: { 4238 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4239 unsigned SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); 4240 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, DstSize); 4241 OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, SrcSize); 4242 break; 4243 } 4244 } 4245 break; 4246 } 4247 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD: 4248 case AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE: { 4249 auto IntrID = MI.getIntrinsicID(); 4250 const AMDGPU::RsrcIntrinsic *RSrcIntrin = AMDGPU::lookupRsrcIntrinsic(IntrID); 4251 assert(RSrcIntrin && "missing RsrcIntrinsic for image intrinsic"); 4252 // Non-images can have complications from operands that allow both SGPR 4253 // and VGPR. For now it's too complicated to figure out the final opcode 4254 // to derive the register bank from the MCInstrDesc. 4255 assert(RSrcIntrin->IsImage); 4256 return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg); 4257 } 4258 case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: { 4259 auto IntrID = MI.getIntrinsicID(); 4260 switch (IntrID) { 4261 case Intrinsic::amdgcn_s_getreg: 4262 case Intrinsic::amdgcn_s_memtime: 4263 case Intrinsic::amdgcn_s_memrealtime: 4264 case Intrinsic::amdgcn_s_get_waveid_in_workgroup: { 4265 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4266 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4267 break; 4268 } 4269 case Intrinsic::amdgcn_global_atomic_fadd: 4270 case Intrinsic::amdgcn_global_atomic_csub: 4271 return getDefaultMappingAllVGPR(MI); 4272 case Intrinsic::amdgcn_ds_ordered_add: 4273 case Intrinsic::amdgcn_ds_ordered_swap: { 4274 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4275 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); 4276 unsigned M0Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, 4277 AMDGPU::SGPRRegBankID); 4278 OpdsMapping[2] = AMDGPU::getValueMapping(M0Bank, 32); 4279 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4280 break; 4281 } 4282 case Intrinsic::amdgcn_ds_append: 4283 case Intrinsic::amdgcn_ds_consume: { 4284 unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4285 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, DstSize); 4286 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4287 break; 4288 } 4289 case Intrinsic::amdgcn_exp_compr: 4290 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4291 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4292 break; 4293 case Intrinsic::amdgcn_exp: 4294 // FIXME: Could we support packed types here? 4295 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4296 OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4297 OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4298 OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4299 break; 4300 case Intrinsic::amdgcn_s_sendmsg: 4301 case Intrinsic::amdgcn_s_sendmsghalt: { 4302 // This must be an SGPR, but accept a VGPR. 4303 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, 4304 AMDGPU::SGPRRegBankID); 4305 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); 4306 break; 4307 } 4308 case Intrinsic::amdgcn_s_setreg: { 4309 // This must be an SGPR, but accept a VGPR. 4310 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, 4311 AMDGPU::SGPRRegBankID); 4312 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); 4313 break; 4314 } 4315 case Intrinsic::amdgcn_end_cf: { 4316 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); 4317 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4318 break; 4319 } 4320 case Intrinsic::amdgcn_else: { 4321 unsigned WaveSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); 4322 OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); 4323 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize); 4324 OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize); 4325 break; 4326 } 4327 case Intrinsic::amdgcn_kill: { 4328 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1); 4329 break; 4330 } 4331 case Intrinsic::amdgcn_raw_buffer_load: 4332 case Intrinsic::amdgcn_raw_tbuffer_load: { 4333 // FIXME: Should make intrinsic ID the last operand of the instruction, 4334 // then this would be the same as store 4335 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 4336 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4337 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4338 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4339 break; 4340 } 4341 case Intrinsic::amdgcn_raw_buffer_store: 4342 case Intrinsic::amdgcn_raw_buffer_store_format: 4343 case Intrinsic::amdgcn_raw_tbuffer_store: { 4344 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 4345 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4346 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4347 OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4348 break; 4349 } 4350 case Intrinsic::amdgcn_struct_buffer_load: 4351 case Intrinsic::amdgcn_struct_tbuffer_load: { 4352 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 4353 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4354 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4355 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4356 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); 4357 break; 4358 } 4359 case Intrinsic::amdgcn_struct_buffer_store: 4360 case Intrinsic::amdgcn_struct_tbuffer_store: { 4361 OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI); 4362 OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4363 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4364 OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); 4365 OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); 4366 break; 4367 } 4368 case Intrinsic::amdgcn_init_exec_from_input: { 4369 unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI); 4370 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); 4371 break; 4372 } 4373 case Intrinsic::amdgcn_ds_gws_init: 4374 case Intrinsic::amdgcn_ds_gws_barrier: 4375 case Intrinsic::amdgcn_ds_gws_sema_br: { 4376 OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32); 4377 4378 // This must be an SGPR, but accept a VGPR. 4379 unsigned Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, 4380 AMDGPU::SGPRRegBankID); 4381 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32); 4382 break; 4383 } 4384 case Intrinsic::amdgcn_ds_gws_sema_v: 4385 case Intrinsic::amdgcn_ds_gws_sema_p: 4386 case Intrinsic::amdgcn_ds_gws_sema_release_all: { 4387 // This must be an SGPR, but accept a VGPR. 4388 unsigned Bank = getRegBankID(MI.getOperand(1).getReg(), MRI, 4389 AMDGPU::SGPRRegBankID); 4390 OpdsMapping[1] = AMDGPU::getValueMapping(Bank, 32); 4391 break; 4392 } 4393 default: 4394 return getInvalidInstructionMapping(); 4395 } 4396 break; 4397 } 4398 case AMDGPU::G_SELECT: { 4399 unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); 4400 unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, 4401 AMDGPU::SGPRRegBankID); 4402 unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, 4403 AMDGPU::SGPRRegBankID); 4404 bool SGPRSrcs = Op2Bank == AMDGPU::SGPRRegBankID && 4405 Op3Bank == AMDGPU::SGPRRegBankID; 4406 4407 unsigned CondBankDefault = SGPRSrcs ? 4408 AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; 4409 unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI, 4410 CondBankDefault); 4411 if (CondBank == AMDGPU::SGPRRegBankID) 4412 CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; 4413 else if (CondBank == AMDGPU::VGPRRegBankID) 4414 CondBank = AMDGPU::VCCRegBankID; 4415 4416 unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ? 4417 AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; 4418 4419 assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID); 4420 4421 // TODO: Should report 32-bit for scalar condition type. 4422 if (Size == 64) { 4423 OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); 4424 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1); 4425 OpdsMapping[2] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); 4426 OpdsMapping[3] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); 4427 } else { 4428 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, Size); 4429 OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1); 4430 OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size); 4431 OpdsMapping[3] = AMDGPU::getValueMapping(Bank, Size); 4432 } 4433 4434 break; 4435 } 4436 4437 case AMDGPU::G_LOAD: 4438 case AMDGPU::G_ZEXTLOAD: 4439 case AMDGPU::G_SEXTLOAD: 4440 return getInstrMappingForLoad(MI); 4441 4442 case AMDGPU::G_ATOMICRMW_XCHG: 4443 case AMDGPU::G_ATOMICRMW_ADD: 4444 case AMDGPU::G_ATOMICRMW_SUB: 4445 case AMDGPU::G_ATOMICRMW_AND: 4446 case AMDGPU::G_ATOMICRMW_OR: 4447 case AMDGPU::G_ATOMICRMW_XOR: 4448 case AMDGPU::G_ATOMICRMW_MAX: 4449 case AMDGPU::G_ATOMICRMW_MIN: 4450 case AMDGPU::G_ATOMICRMW_UMAX: 4451 case AMDGPU::G_ATOMICRMW_UMIN: 4452 case AMDGPU::G_ATOMICRMW_FADD: 4453 case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: 4454 case AMDGPU::G_AMDGPU_ATOMIC_INC: 4455 case AMDGPU::G_AMDGPU_ATOMIC_DEC: 4456 case AMDGPU::G_AMDGPU_ATOMIC_FMIN: 4457 case AMDGPU::G_AMDGPU_ATOMIC_FMAX: { 4458 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 4459 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); 4460 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4461 break; 4462 } 4463 case AMDGPU::G_ATOMIC_CMPXCHG: { 4464 OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); 4465 OpdsMapping[1] = getValueMappingForPtr(MRI, MI.getOperand(1).getReg()); 4466 OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); 4467 OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); 4468 break; 4469 } 4470 case AMDGPU::G_BRCOND: { 4471 unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, 4472 AMDGPU::SGPRRegBankID); 4473 assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1); 4474 if (Bank != AMDGPU::SGPRRegBankID) 4475 Bank = AMDGPU::VCCRegBankID; 4476 4477 OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1); 4478 break; 4479 } 4480 } 4481 4482 return getInstructionMapping(/*ID*/1, /*Cost*/1, 4483 getOperandsMapping(OpdsMapping), 4484 MI.getNumOperands()); 4485 } 4486