1 //===-- SIInstrInfo.cpp - SI Instruction Information ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// \brief SI Implementation of TargetInstrInfo. 12 // 13 //===----------------------------------------------------------------------===// 14 15 16 #include "SIInstrInfo.h" 17 #include "AMDGPUTargetMachine.h" 18 #include "SIDefines.h" 19 #include "SIMachineFunctionInfo.h" 20 #include "llvm/CodeGen/MachineFrameInfo.h" 21 #include "llvm/CodeGen/MachineInstrBuilder.h" 22 #include "llvm/CodeGen/MachineRegisterInfo.h" 23 #include "llvm/IR/Function.h" 24 #include "llvm/CodeGen/RegisterScavenging.h" 25 #include "llvm/MC/MCInstrDesc.h" 26 #include "llvm/Support/Debug.h" 27 28 using namespace llvm; 29 30 SIInstrInfo::SIInstrInfo(const AMDGPUSubtarget &st) 31 : AMDGPUInstrInfo(st), RI() {} 32 33 //===----------------------------------------------------------------------===// 34 // TargetInstrInfo callbacks 35 //===----------------------------------------------------------------------===// 36 37 static unsigned getNumOperandsNoGlue(SDNode *Node) { 38 unsigned N = Node->getNumOperands(); 39 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 40 --N; 41 return N; 42 } 43 44 static SDValue findChainOperand(SDNode *Load) { 45 SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1); 46 assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node"); 47 return LastOp; 48 } 49 50 /// \brief Returns true if both nodes have the same value for the given 51 /// operand \p Op, or if both nodes do not have this operand. 52 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 53 unsigned Opc0 = N0->getMachineOpcode(); 54 unsigned Opc1 = N1->getMachineOpcode(); 55 56 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 57 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 58 59 if (Op0Idx == -1 && Op1Idx == -1) 60 return true; 61 62 63 if ((Op0Idx == -1 && Op1Idx != -1) || 64 (Op1Idx == -1 && Op0Idx != -1)) 65 return false; 66 67 // getNamedOperandIdx returns the index for the MachineInstr's operands, 68 // which includes the result as the first operand. We are indexing into the 69 // MachineSDNode's operands, so we need to skip the result operand to get 70 // the real index. 71 --Op0Idx; 72 --Op1Idx; 73 74 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 75 } 76 77 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr *MI, 78 AliasAnalysis *AA) const { 79 // TODO: The generic check fails for VALU instructions that should be 80 // rematerializable due to implicit reads of exec. We really want all of the 81 // generic logic for this except for this. 82 switch (MI->getOpcode()) { 83 case AMDGPU::V_MOV_B32_e32: 84 case AMDGPU::V_MOV_B32_e64: 85 case AMDGPU::V_MOV_B64_PSEUDO: 86 return true; 87 default: 88 return false; 89 } 90 } 91 92 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 93 int64_t &Offset0, 94 int64_t &Offset1) const { 95 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 96 return false; 97 98 unsigned Opc0 = Load0->getMachineOpcode(); 99 unsigned Opc1 = Load1->getMachineOpcode(); 100 101 // Make sure both are actually loads. 102 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 103 return false; 104 105 if (isDS(Opc0) && isDS(Opc1)) { 106 107 // FIXME: Handle this case: 108 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 109 return false; 110 111 // Check base reg. 112 if (Load0->getOperand(1) != Load1->getOperand(1)) 113 return false; 114 115 // Check chain. 116 if (findChainOperand(Load0) != findChainOperand(Load1)) 117 return false; 118 119 // Skip read2 / write2 variants for simplicity. 120 // TODO: We should report true if the used offsets are adjacent (excluded 121 // st64 versions). 122 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 || 123 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1) 124 return false; 125 126 Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue(); 127 Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue(); 128 return true; 129 } 130 131 if (isSMRD(Opc0) && isSMRD(Opc1)) { 132 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1)); 133 134 // Check base reg. 135 if (Load0->getOperand(0) != Load1->getOperand(0)) 136 return false; 137 138 const ConstantSDNode *Load0Offset = 139 dyn_cast<ConstantSDNode>(Load0->getOperand(1)); 140 const ConstantSDNode *Load1Offset = 141 dyn_cast<ConstantSDNode>(Load1->getOperand(1)); 142 143 if (!Load0Offset || !Load1Offset) 144 return false; 145 146 // Check chain. 147 if (findChainOperand(Load0) != findChainOperand(Load1)) 148 return false; 149 150 Offset0 = Load0Offset->getZExtValue(); 151 Offset1 = Load1Offset->getZExtValue(); 152 return true; 153 } 154 155 // MUBUF and MTBUF can access the same addresses. 156 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 157 158 // MUBUF and MTBUF have vaddr at different indices. 159 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 160 findChainOperand(Load0) != findChainOperand(Load1) || 161 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 162 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 163 return false; 164 165 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 166 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 167 168 if (OffIdx0 == -1 || OffIdx1 == -1) 169 return false; 170 171 // getNamedOperandIdx returns the index for MachineInstrs. Since they 172 // inlcude the output in the operand list, but SDNodes don't, we need to 173 // subtract the index by one. 174 --OffIdx0; 175 --OffIdx1; 176 177 SDValue Off0 = Load0->getOperand(OffIdx0); 178 SDValue Off1 = Load1->getOperand(OffIdx1); 179 180 // The offset might be a FrameIndexSDNode. 181 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 182 return false; 183 184 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue(); 185 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue(); 186 return true; 187 } 188 189 return false; 190 } 191 192 static bool isStride64(unsigned Opc) { 193 switch (Opc) { 194 case AMDGPU::DS_READ2ST64_B32: 195 case AMDGPU::DS_READ2ST64_B64: 196 case AMDGPU::DS_WRITE2ST64_B32: 197 case AMDGPU::DS_WRITE2ST64_B64: 198 return true; 199 default: 200 return false; 201 } 202 } 203 204 bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr *LdSt, unsigned &BaseReg, 205 unsigned &Offset, 206 const TargetRegisterInfo *TRI) const { 207 unsigned Opc = LdSt->getOpcode(); 208 209 if (isDS(*LdSt)) { 210 const MachineOperand *OffsetImm = getNamedOperand(*LdSt, 211 AMDGPU::OpName::offset); 212 if (OffsetImm) { 213 // Normal, single offset LDS instruction. 214 const MachineOperand *AddrReg = getNamedOperand(*LdSt, 215 AMDGPU::OpName::addr); 216 217 BaseReg = AddrReg->getReg(); 218 Offset = OffsetImm->getImm(); 219 return true; 220 } 221 222 // The 2 offset instructions use offset0 and offset1 instead. We can treat 223 // these as a load with a single offset if the 2 offsets are consecutive. We 224 // will use this for some partially aligned loads. 225 const MachineOperand *Offset0Imm = getNamedOperand(*LdSt, 226 AMDGPU::OpName::offset0); 227 const MachineOperand *Offset1Imm = getNamedOperand(*LdSt, 228 AMDGPU::OpName::offset1); 229 230 uint8_t Offset0 = Offset0Imm->getImm(); 231 uint8_t Offset1 = Offset1Imm->getImm(); 232 233 if (Offset1 > Offset0 && Offset1 - Offset0 == 1) { 234 // Each of these offsets is in element sized units, so we need to convert 235 // to bytes of the individual reads. 236 237 unsigned EltSize; 238 if (LdSt->mayLoad()) 239 EltSize = getOpRegClass(*LdSt, 0)->getSize() / 2; 240 else { 241 assert(LdSt->mayStore()); 242 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 243 EltSize = getOpRegClass(*LdSt, Data0Idx)->getSize(); 244 } 245 246 if (isStride64(Opc)) 247 EltSize *= 64; 248 249 const MachineOperand *AddrReg = getNamedOperand(*LdSt, 250 AMDGPU::OpName::addr); 251 BaseReg = AddrReg->getReg(); 252 Offset = EltSize * Offset0; 253 return true; 254 } 255 256 return false; 257 } 258 259 if (isMUBUF(*LdSt) || isMTBUF(*LdSt)) { 260 if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset) != -1) 261 return false; 262 263 const MachineOperand *AddrReg = getNamedOperand(*LdSt, 264 AMDGPU::OpName::vaddr); 265 if (!AddrReg) 266 return false; 267 268 const MachineOperand *OffsetImm = getNamedOperand(*LdSt, 269 AMDGPU::OpName::offset); 270 BaseReg = AddrReg->getReg(); 271 Offset = OffsetImm->getImm(); 272 return true; 273 } 274 275 if (isSMRD(*LdSt)) { 276 const MachineOperand *OffsetImm = getNamedOperand(*LdSt, 277 AMDGPU::OpName::offset); 278 if (!OffsetImm) 279 return false; 280 281 const MachineOperand *SBaseReg = getNamedOperand(*LdSt, 282 AMDGPU::OpName::sbase); 283 BaseReg = SBaseReg->getReg(); 284 Offset = OffsetImm->getImm(); 285 return true; 286 } 287 288 return false; 289 } 290 291 bool SIInstrInfo::shouldClusterLoads(MachineInstr *FirstLdSt, 292 MachineInstr *SecondLdSt, 293 unsigned NumLoads) const { 294 // TODO: This needs finer tuning 295 if (NumLoads > 4) 296 return false; 297 298 if (isDS(*FirstLdSt) && isDS(*SecondLdSt)) 299 return true; 300 301 if (isSMRD(*FirstLdSt) && isSMRD(*SecondLdSt)) 302 return true; 303 304 if ((isMUBUF(*FirstLdSt) || isMTBUF(*FirstLdSt)) && 305 (isMUBUF(*SecondLdSt) || isMTBUF(*SecondLdSt))) 306 return true; 307 308 return false; 309 } 310 311 void 312 SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 313 MachineBasicBlock::iterator MI, DebugLoc DL, 314 unsigned DestReg, unsigned SrcReg, 315 bool KillSrc) const { 316 317 // If we are trying to copy to or from SCC, there is a bug somewhere else in 318 // the backend. While it may be theoretically possible to do this, it should 319 // never be necessary. 320 assert(DestReg != AMDGPU::SCC && SrcReg != AMDGPU::SCC); 321 322 static const int16_t Sub0_15[] = { 323 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 324 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 325 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 326 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 0 327 }; 328 329 static const int16_t Sub0_7[] = { 330 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 331 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 0 332 }; 333 334 static const int16_t Sub0_3[] = { 335 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 0 336 }; 337 338 static const int16_t Sub0_2[] = { 339 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, 0 340 }; 341 342 static const int16_t Sub0_1[] = { 343 AMDGPU::sub0, AMDGPU::sub1, 0 344 }; 345 346 unsigned Opcode; 347 const int16_t *SubIndices; 348 349 if (AMDGPU::SReg_32RegClass.contains(DestReg)) { 350 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 351 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 352 .addReg(SrcReg, getKillRegState(KillSrc)); 353 return; 354 355 } else if (AMDGPU::SReg_64RegClass.contains(DestReg)) { 356 if (DestReg == AMDGPU::VCC) { 357 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 358 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 359 .addReg(SrcReg, getKillRegState(KillSrc)); 360 } else { 361 // FIXME: Hack until VReg_1 removed. 362 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 363 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_I32_e32)) 364 .addImm(0) 365 .addReg(SrcReg, getKillRegState(KillSrc)); 366 } 367 368 return; 369 } 370 371 assert(AMDGPU::SReg_64RegClass.contains(SrcReg)); 372 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 373 .addReg(SrcReg, getKillRegState(KillSrc)); 374 return; 375 376 } else if (AMDGPU::SReg_128RegClass.contains(DestReg)) { 377 assert(AMDGPU::SReg_128RegClass.contains(SrcReg)); 378 Opcode = AMDGPU::S_MOV_B32; 379 SubIndices = Sub0_3; 380 381 } else if (AMDGPU::SReg_256RegClass.contains(DestReg)) { 382 assert(AMDGPU::SReg_256RegClass.contains(SrcReg)); 383 Opcode = AMDGPU::S_MOV_B32; 384 SubIndices = Sub0_7; 385 386 } else if (AMDGPU::SReg_512RegClass.contains(DestReg)) { 387 assert(AMDGPU::SReg_512RegClass.contains(SrcReg)); 388 Opcode = AMDGPU::S_MOV_B32; 389 SubIndices = Sub0_15; 390 391 } else if (AMDGPU::VGPR_32RegClass.contains(DestReg)) { 392 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 393 AMDGPU::SReg_32RegClass.contains(SrcReg)); 394 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 395 .addReg(SrcReg, getKillRegState(KillSrc)); 396 return; 397 398 } else if (AMDGPU::VReg_64RegClass.contains(DestReg)) { 399 assert(AMDGPU::VReg_64RegClass.contains(SrcReg) || 400 AMDGPU::SReg_64RegClass.contains(SrcReg)); 401 Opcode = AMDGPU::V_MOV_B32_e32; 402 SubIndices = Sub0_1; 403 404 } else if (AMDGPU::VReg_96RegClass.contains(DestReg)) { 405 assert(AMDGPU::VReg_96RegClass.contains(SrcReg)); 406 Opcode = AMDGPU::V_MOV_B32_e32; 407 SubIndices = Sub0_2; 408 409 } else if (AMDGPU::VReg_128RegClass.contains(DestReg)) { 410 assert(AMDGPU::VReg_128RegClass.contains(SrcReg) || 411 AMDGPU::SReg_128RegClass.contains(SrcReg)); 412 Opcode = AMDGPU::V_MOV_B32_e32; 413 SubIndices = Sub0_3; 414 415 } else if (AMDGPU::VReg_256RegClass.contains(DestReg)) { 416 assert(AMDGPU::VReg_256RegClass.contains(SrcReg) || 417 AMDGPU::SReg_256RegClass.contains(SrcReg)); 418 Opcode = AMDGPU::V_MOV_B32_e32; 419 SubIndices = Sub0_7; 420 421 } else if (AMDGPU::VReg_512RegClass.contains(DestReg)) { 422 assert(AMDGPU::VReg_512RegClass.contains(SrcReg) || 423 AMDGPU::SReg_512RegClass.contains(SrcReg)); 424 Opcode = AMDGPU::V_MOV_B32_e32; 425 SubIndices = Sub0_15; 426 427 } else { 428 llvm_unreachable("Can't copy register!"); 429 } 430 431 while (unsigned SubIdx = *SubIndices++) { 432 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 433 get(Opcode), RI.getSubReg(DestReg, SubIdx)); 434 435 Builder.addReg(RI.getSubReg(SrcReg, SubIdx), getKillRegState(KillSrc)); 436 437 if (*SubIndices) 438 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 439 } 440 } 441 442 int SIInstrInfo::commuteOpcode(const MachineInstr &MI) const { 443 const unsigned Opcode = MI.getOpcode(); 444 445 int NewOpc; 446 447 // Try to map original to commuted opcode 448 NewOpc = AMDGPU::getCommuteRev(Opcode); 449 if (NewOpc != -1) 450 // Check if the commuted (REV) opcode exists on the target. 451 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 452 453 // Try to map commuted to original opcode 454 NewOpc = AMDGPU::getCommuteOrig(Opcode); 455 if (NewOpc != -1) 456 // Check if the original (non-REV) opcode exists on the target. 457 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 458 459 return Opcode; 460 } 461 462 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 463 464 if (DstRC->getSize() == 4) { 465 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 466 } else if (DstRC->getSize() == 8 && RI.isSGPRClass(DstRC)) { 467 return AMDGPU::S_MOV_B64; 468 } else if (DstRC->getSize() == 8 && !RI.isSGPRClass(DstRC)) { 469 return AMDGPU::V_MOV_B64_PSEUDO; 470 } 471 return AMDGPU::COPY; 472 } 473 474 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 475 MachineBasicBlock::iterator MI, 476 unsigned SrcReg, bool isKill, 477 int FrameIndex, 478 const TargetRegisterClass *RC, 479 const TargetRegisterInfo *TRI) const { 480 MachineFunction *MF = MBB.getParent(); 481 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 482 MachineFrameInfo *FrameInfo = MF->getFrameInfo(); 483 DebugLoc DL = MBB.findDebugLoc(MI); 484 int Opcode = -1; 485 486 if (RI.isSGPRClass(RC)) { 487 // We are only allowed to create one new instruction when spilling 488 // registers, so we need to use pseudo instruction for spilling 489 // SGPRs. 490 switch (RC->getSize() * 8) { 491 case 32: Opcode = AMDGPU::SI_SPILL_S32_SAVE; break; 492 case 64: Opcode = AMDGPU::SI_SPILL_S64_SAVE; break; 493 case 128: Opcode = AMDGPU::SI_SPILL_S128_SAVE; break; 494 case 256: Opcode = AMDGPU::SI_SPILL_S256_SAVE; break; 495 case 512: Opcode = AMDGPU::SI_SPILL_S512_SAVE; break; 496 } 497 } else if(RI.hasVGPRs(RC) && ST.isVGPRSpillingEnabled(MFI)) { 498 MFI->setHasSpilledVGPRs(); 499 500 switch(RC->getSize() * 8) { 501 case 32: Opcode = AMDGPU::SI_SPILL_V32_SAVE; break; 502 case 64: Opcode = AMDGPU::SI_SPILL_V64_SAVE; break; 503 case 96: Opcode = AMDGPU::SI_SPILL_V96_SAVE; break; 504 case 128: Opcode = AMDGPU::SI_SPILL_V128_SAVE; break; 505 case 256: Opcode = AMDGPU::SI_SPILL_V256_SAVE; break; 506 case 512: Opcode = AMDGPU::SI_SPILL_V512_SAVE; break; 507 } 508 } 509 510 if (Opcode != -1) { 511 MachinePointerInfo PtrInfo 512 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 513 unsigned Size = FrameInfo->getObjectSize(FrameIndex); 514 unsigned Align = FrameInfo->getObjectAlignment(FrameIndex); 515 MachineMemOperand *MMO 516 = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 517 Size, Align); 518 519 FrameInfo->setObjectAlignment(FrameIndex, 4); 520 BuildMI(MBB, MI, DL, get(Opcode)) 521 .addReg(SrcReg) 522 .addFrameIndex(FrameIndex) 523 // Place-holder registers, these will be filled in by 524 // SIPrepareScratchRegs. 525 .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef) 526 .addReg(AMDGPU::SGPR0, RegState::Undef) 527 .addMemOperand(MMO); 528 } else { 529 LLVMContext &Ctx = MF->getFunction()->getContext(); 530 Ctx.emitError("SIInstrInfo::storeRegToStackSlot - Do not know how to" 531 " spill register"); 532 BuildMI(MBB, MI, DL, get(AMDGPU::KILL)) 533 .addReg(SrcReg); 534 } 535 } 536 537 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 538 MachineBasicBlock::iterator MI, 539 unsigned DestReg, int FrameIndex, 540 const TargetRegisterClass *RC, 541 const TargetRegisterInfo *TRI) const { 542 MachineFunction *MF = MBB.getParent(); 543 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 544 MachineFrameInfo *FrameInfo = MF->getFrameInfo(); 545 DebugLoc DL = MBB.findDebugLoc(MI); 546 int Opcode = -1; 547 548 if (RI.isSGPRClass(RC)){ 549 switch(RC->getSize() * 8) { 550 case 32: Opcode = AMDGPU::SI_SPILL_S32_RESTORE; break; 551 case 64: Opcode = AMDGPU::SI_SPILL_S64_RESTORE; break; 552 case 128: Opcode = AMDGPU::SI_SPILL_S128_RESTORE; break; 553 case 256: Opcode = AMDGPU::SI_SPILL_S256_RESTORE; break; 554 case 512: Opcode = AMDGPU::SI_SPILL_S512_RESTORE; break; 555 } 556 } else if(RI.hasVGPRs(RC) && ST.isVGPRSpillingEnabled(MFI)) { 557 switch(RC->getSize() * 8) { 558 case 32: Opcode = AMDGPU::SI_SPILL_V32_RESTORE; break; 559 case 64: Opcode = AMDGPU::SI_SPILL_V64_RESTORE; break; 560 case 96: Opcode = AMDGPU::SI_SPILL_V96_RESTORE; break; 561 case 128: Opcode = AMDGPU::SI_SPILL_V128_RESTORE; break; 562 case 256: Opcode = AMDGPU::SI_SPILL_V256_RESTORE; break; 563 case 512: Opcode = AMDGPU::SI_SPILL_V512_RESTORE; break; 564 } 565 } 566 567 if (Opcode != -1) { 568 unsigned Align = 4; 569 FrameInfo->setObjectAlignment(FrameIndex, Align); 570 unsigned Size = FrameInfo->getObjectSize(FrameIndex); 571 572 MachinePointerInfo PtrInfo 573 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 574 MachineMemOperand *MMO = MF->getMachineMemOperand( 575 PtrInfo, MachineMemOperand::MOLoad, Size, Align); 576 577 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 578 .addFrameIndex(FrameIndex) 579 // Place-holder registers, these will be filled in by 580 // SIPrepareScratchRegs. 581 .addReg(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, RegState::Undef) 582 .addReg(AMDGPU::SGPR0, RegState::Undef) 583 .addMemOperand(MMO); 584 } else { 585 LLVMContext &Ctx = MF->getFunction()->getContext(); 586 Ctx.emitError("SIInstrInfo::loadRegFromStackSlot - Do not know how to" 587 " restore register"); 588 BuildMI(MBB, MI, DL, get(AMDGPU::IMPLICIT_DEF), DestReg); 589 } 590 } 591 592 /// \param @Offset Offset in bytes of the FrameIndex being spilled 593 unsigned SIInstrInfo::calculateLDSSpillAddress(MachineBasicBlock &MBB, 594 MachineBasicBlock::iterator MI, 595 RegScavenger *RS, unsigned TmpReg, 596 unsigned FrameOffset, 597 unsigned Size) const { 598 MachineFunction *MF = MBB.getParent(); 599 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 600 const AMDGPUSubtarget &ST = MF->getSubtarget<AMDGPUSubtarget>(); 601 const SIRegisterInfo *TRI = 602 static_cast<const SIRegisterInfo*>(ST.getRegisterInfo()); 603 DebugLoc DL = MBB.findDebugLoc(MI); 604 unsigned WorkGroupSize = MFI->getMaximumWorkGroupSize(*MF); 605 unsigned WavefrontSize = ST.getWavefrontSize(); 606 607 unsigned TIDReg = MFI->getTIDReg(); 608 if (!MFI->hasCalculatedTID()) { 609 MachineBasicBlock &Entry = MBB.getParent()->front(); 610 MachineBasicBlock::iterator Insert = Entry.front(); 611 DebugLoc DL = Insert->getDebugLoc(); 612 613 TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass); 614 if (TIDReg == AMDGPU::NoRegister) 615 return TIDReg; 616 617 618 if (MFI->getShaderType() == ShaderType::COMPUTE && 619 WorkGroupSize > WavefrontSize) { 620 621 unsigned TIDIGXReg = TRI->getPreloadedValue(*MF, SIRegisterInfo::TIDIG_X); 622 unsigned TIDIGYReg = TRI->getPreloadedValue(*MF, SIRegisterInfo::TIDIG_Y); 623 unsigned TIDIGZReg = TRI->getPreloadedValue(*MF, SIRegisterInfo::TIDIG_Z); 624 unsigned InputPtrReg = 625 TRI->getPreloadedValue(*MF, SIRegisterInfo::INPUT_PTR); 626 for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) { 627 if (!Entry.isLiveIn(Reg)) 628 Entry.addLiveIn(Reg); 629 } 630 631 RS->enterBasicBlock(&Entry); 632 unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0); 633 unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0); 634 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0) 635 .addReg(InputPtrReg) 636 .addImm(SI::KernelInputOffsets::NGROUPS_Z); 637 BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1) 638 .addReg(InputPtrReg) 639 .addImm(SI::KernelInputOffsets::NGROUPS_Y); 640 641 // NGROUPS.X * NGROUPS.Y 642 BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1) 643 .addReg(STmp1) 644 .addReg(STmp0); 645 // (NGROUPS.X * NGROUPS.Y) * TIDIG.X 646 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg) 647 .addReg(STmp1) 648 .addReg(TIDIGXReg); 649 // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X) 650 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg) 651 .addReg(STmp0) 652 .addReg(TIDIGYReg) 653 .addReg(TIDReg); 654 // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z 655 BuildMI(Entry, Insert, DL, get(AMDGPU::V_ADD_I32_e32), TIDReg) 656 .addReg(TIDReg) 657 .addReg(TIDIGZReg); 658 } else { 659 // Get the wave id 660 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64), 661 TIDReg) 662 .addImm(-1) 663 .addImm(0); 664 665 BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64), 666 TIDReg) 667 .addImm(-1) 668 .addReg(TIDReg); 669 } 670 671 BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32), 672 TIDReg) 673 .addImm(2) 674 .addReg(TIDReg); 675 MFI->setTIDReg(TIDReg); 676 } 677 678 // Add FrameIndex to LDS offset 679 unsigned LDSOffset = MFI->LDSSize + (FrameOffset * WorkGroupSize); 680 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), TmpReg) 681 .addImm(LDSOffset) 682 .addReg(TIDReg); 683 684 return TmpReg; 685 } 686 687 void SIInstrInfo::insertNOPs(MachineBasicBlock::iterator MI, 688 int Count) const { 689 while (Count > 0) { 690 int Arg; 691 if (Count >= 8) 692 Arg = 7; 693 else 694 Arg = Count - 1; 695 Count -= 8; 696 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), get(AMDGPU::S_NOP)) 697 .addImm(Arg); 698 } 699 } 700 701 bool SIInstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const { 702 MachineBasicBlock &MBB = *MI->getParent(); 703 DebugLoc DL = MBB.findDebugLoc(MI); 704 switch (MI->getOpcode()) { 705 default: return AMDGPUInstrInfo::expandPostRAPseudo(MI); 706 707 case AMDGPU::SI_CONSTDATA_PTR: { 708 unsigned Reg = MI->getOperand(0).getReg(); 709 unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 710 unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 711 712 BuildMI(MBB, MI, DL, get(AMDGPU::S_GETPC_B64), Reg); 713 714 // Add 32-bit offset from this instruction to the start of the constant data. 715 BuildMI(MBB, MI, DL, get(AMDGPU::S_ADD_U32), RegLo) 716 .addReg(RegLo) 717 .addTargetIndex(AMDGPU::TI_CONSTDATA_START) 718 .addReg(AMDGPU::SCC, RegState::Define | RegState::Implicit); 719 BuildMI(MBB, MI, DL, get(AMDGPU::S_ADDC_U32), RegHi) 720 .addReg(RegHi) 721 .addImm(0) 722 .addReg(AMDGPU::SCC, RegState::Define | RegState::Implicit) 723 .addReg(AMDGPU::SCC, RegState::Implicit); 724 MI->eraseFromParent(); 725 break; 726 } 727 case AMDGPU::SGPR_USE: 728 // This is just a placeholder for register allocation. 729 MI->eraseFromParent(); 730 break; 731 732 case AMDGPU::V_MOV_B64_PSEUDO: { 733 unsigned Dst = MI->getOperand(0).getReg(); 734 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 735 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 736 737 const MachineOperand &SrcOp = MI->getOperand(1); 738 // FIXME: Will this work for 64-bit floating point immediates? 739 assert(!SrcOp.isFPImm()); 740 if (SrcOp.isImm()) { 741 APInt Imm(64, SrcOp.getImm()); 742 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 743 .addImm(Imm.getLoBits(32).getZExtValue()) 744 .addReg(Dst, RegState::Implicit); 745 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 746 .addImm(Imm.getHiBits(32).getZExtValue()) 747 .addReg(Dst, RegState::Implicit); 748 } else { 749 assert(SrcOp.isReg()); 750 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 751 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 752 .addReg(Dst, RegState::Implicit); 753 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 754 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 755 .addReg(Dst, RegState::Implicit); 756 } 757 MI->eraseFromParent(); 758 break; 759 } 760 761 case AMDGPU::V_CNDMASK_B64_PSEUDO: { 762 unsigned Dst = MI->getOperand(0).getReg(); 763 unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 764 unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 765 unsigned Src0 = MI->getOperand(1).getReg(); 766 unsigned Src1 = MI->getOperand(2).getReg(); 767 const MachineOperand &SrcCond = MI->getOperand(3); 768 769 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstLo) 770 .addReg(RI.getSubReg(Src0, AMDGPU::sub0)) 771 .addReg(RI.getSubReg(Src1, AMDGPU::sub0)) 772 .addOperand(SrcCond); 773 BuildMI(MBB, MI, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstHi) 774 .addReg(RI.getSubReg(Src0, AMDGPU::sub1)) 775 .addReg(RI.getSubReg(Src1, AMDGPU::sub1)) 776 .addOperand(SrcCond); 777 MI->eraseFromParent(); 778 break; 779 } 780 } 781 return true; 782 } 783 784 /// Commutes the operands in the given instruction. 785 /// The commutable operands are specified by their indices OpIdx0 and OpIdx1. 786 /// 787 /// Do not call this method for a non-commutable instruction or for 788 /// non-commutable pair of operand indices OpIdx0 and OpIdx1. 789 /// Even though the instruction is commutable, the method may still 790 /// fail to commute the operands, null pointer is returned in such cases. 791 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr *MI, 792 bool NewMI, 793 unsigned OpIdx0, 794 unsigned OpIdx1) const { 795 int CommutedOpcode = commuteOpcode(*MI); 796 if (CommutedOpcode == -1) 797 return nullptr; 798 799 int Src0Idx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 800 AMDGPU::OpName::src0); 801 MachineOperand &Src0 = MI->getOperand(Src0Idx); 802 if (!Src0.isReg()) 803 return nullptr; 804 805 int Src1Idx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), 806 AMDGPU::OpName::src1); 807 808 if ((OpIdx0 != static_cast<unsigned>(Src0Idx) || 809 OpIdx1 != static_cast<unsigned>(Src1Idx)) && 810 (OpIdx0 != static_cast<unsigned>(Src1Idx) || 811 OpIdx1 != static_cast<unsigned>(Src0Idx))) 812 return nullptr; 813 814 MachineOperand &Src1 = MI->getOperand(Src1Idx); 815 816 // Make sure it's legal to commute operands for VOP2. 817 if (isVOP2(*MI) && 818 (!isOperandLegal(MI, Src0Idx, &Src1) || 819 !isOperandLegal(MI, Src1Idx, &Src0))) { 820 return nullptr; 821 } 822 823 if (!Src1.isReg()) { 824 // Allow commuting instructions with Imm operands. 825 if (NewMI || !Src1.isImm() || 826 (!isVOP2(*MI) && !isVOP3(*MI))) { 827 return nullptr; 828 } 829 830 // Be sure to copy the source modifiers to the right place. 831 if (MachineOperand *Src0Mods 832 = getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) { 833 MachineOperand *Src1Mods 834 = getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers); 835 836 int Src0ModsVal = Src0Mods->getImm(); 837 if (!Src1Mods && Src0ModsVal != 0) 838 return nullptr; 839 840 // XXX - This assert might be a lie. It might be useful to have a neg 841 // modifier with 0.0. 842 int Src1ModsVal = Src1Mods->getImm(); 843 assert((Src1ModsVal == 0) && "Not expecting modifiers with immediates"); 844 845 Src1Mods->setImm(Src0ModsVal); 846 Src0Mods->setImm(Src1ModsVal); 847 } 848 849 unsigned Reg = Src0.getReg(); 850 unsigned SubReg = Src0.getSubReg(); 851 if (Src1.isImm()) 852 Src0.ChangeToImmediate(Src1.getImm()); 853 else 854 llvm_unreachable("Should only have immediates"); 855 856 Src1.ChangeToRegister(Reg, false); 857 Src1.setSubReg(SubReg); 858 } else { 859 MI = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx0, OpIdx1); 860 } 861 862 if (MI) 863 MI->setDesc(get(CommutedOpcode)); 864 865 return MI; 866 } 867 868 // This needs to be implemented because the source modifiers may be inserted 869 // between the true commutable operands, and the base 870 // TargetInstrInfo::commuteInstruction uses it. 871 bool SIInstrInfo::findCommutedOpIndices(MachineInstr *MI, 872 unsigned &SrcOpIdx0, 873 unsigned &SrcOpIdx1) const { 874 const MCInstrDesc &MCID = MI->getDesc(); 875 if (!MCID.isCommutable()) 876 return false; 877 878 unsigned Opc = MI->getOpcode(); 879 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 880 if (Src0Idx == -1) 881 return false; 882 883 // FIXME: Workaround TargetInstrInfo::commuteInstruction asserting on 884 // immediate. Also, immediate src0 operand is not handled in 885 // SIInstrInfo::commuteInstruction(); 886 if (!MI->getOperand(Src0Idx).isReg()) 887 return false; 888 889 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 890 if (Src1Idx == -1) 891 return false; 892 893 MachineOperand &Src1 = MI->getOperand(Src1Idx); 894 if (Src1.isImm()) { 895 // SIInstrInfo::commuteInstruction() does support commuting the immediate 896 // operand src1 in 2 and 3 operand instructions. 897 if (!isVOP2(MI->getOpcode()) && !isVOP3(MI->getOpcode())) 898 return false; 899 } else if (Src1.isReg()) { 900 // If any source modifiers are set, the generic instruction commuting won't 901 // understand how to copy the source modifiers. 902 if (hasModifiersSet(*MI, AMDGPU::OpName::src0_modifiers) || 903 hasModifiersSet(*MI, AMDGPU::OpName::src1_modifiers)) 904 return false; 905 } else 906 return false; 907 908 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 909 } 910 911 MachineInstr *SIInstrInfo::buildMovInstr(MachineBasicBlock *MBB, 912 MachineBasicBlock::iterator I, 913 unsigned DstReg, 914 unsigned SrcReg) const { 915 return BuildMI(*MBB, I, MBB->findDebugLoc(I), get(AMDGPU::V_MOV_B32_e32), 916 DstReg) .addReg(SrcReg); 917 } 918 919 bool SIInstrInfo::isMov(unsigned Opcode) const { 920 switch(Opcode) { 921 default: return false; 922 case AMDGPU::S_MOV_B32: 923 case AMDGPU::S_MOV_B64: 924 case AMDGPU::V_MOV_B32_e32: 925 case AMDGPU::V_MOV_B32_e64: 926 return true; 927 } 928 } 929 930 static void removeModOperands(MachineInstr &MI) { 931 unsigned Opc = MI.getOpcode(); 932 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, 933 AMDGPU::OpName::src0_modifiers); 934 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, 935 AMDGPU::OpName::src1_modifiers); 936 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, 937 AMDGPU::OpName::src2_modifiers); 938 939 MI.RemoveOperand(Src2ModIdx); 940 MI.RemoveOperand(Src1ModIdx); 941 MI.RemoveOperand(Src0ModIdx); 942 } 943 944 bool SIInstrInfo::FoldImmediate(MachineInstr *UseMI, MachineInstr *DefMI, 945 unsigned Reg, MachineRegisterInfo *MRI) const { 946 if (!MRI->hasOneNonDBGUse(Reg)) 947 return false; 948 949 unsigned Opc = UseMI->getOpcode(); 950 if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64) { 951 // Don't fold if we are using source modifiers. The new VOP2 instructions 952 // don't have them. 953 if (hasModifiersSet(*UseMI, AMDGPU::OpName::src0_modifiers) || 954 hasModifiersSet(*UseMI, AMDGPU::OpName::src1_modifiers) || 955 hasModifiersSet(*UseMI, AMDGPU::OpName::src2_modifiers)) { 956 return false; 957 } 958 959 MachineOperand *Src0 = getNamedOperand(*UseMI, AMDGPU::OpName::src0); 960 MachineOperand *Src1 = getNamedOperand(*UseMI, AMDGPU::OpName::src1); 961 MachineOperand *Src2 = getNamedOperand(*UseMI, AMDGPU::OpName::src2); 962 963 // Multiplied part is the constant: Use v_madmk_f32 964 // We should only expect these to be on src0 due to canonicalizations. 965 if (Src0->isReg() && Src0->getReg() == Reg) { 966 if (!Src1->isReg() || 967 (Src1->isReg() && RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 968 return false; 969 970 if (!Src2->isReg() || 971 (Src2->isReg() && RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))) 972 return false; 973 974 // We need to do some weird looking operand shuffling since the madmk 975 // operands are out of the normal expected order with the multiplied 976 // constant as the last operand. 977 // 978 // v_mad_f32 src0, src1, src2 -> v_madmk_f32 src0 * src2K + src1 979 // src0 -> src2 K 980 // src1 -> src0 981 // src2 -> src1 982 983 const int64_t Imm = DefMI->getOperand(1).getImm(); 984 985 // FIXME: This would be a lot easier if we could return a new instruction 986 // instead of having to modify in place. 987 988 // Remove these first since they are at the end. 989 UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, 990 AMDGPU::OpName::omod)); 991 UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, 992 AMDGPU::OpName::clamp)); 993 994 unsigned Src1Reg = Src1->getReg(); 995 unsigned Src1SubReg = Src1->getSubReg(); 996 unsigned Src2Reg = Src2->getReg(); 997 unsigned Src2SubReg = Src2->getSubReg(); 998 Src0->setReg(Src1Reg); 999 Src0->setSubReg(Src1SubReg); 1000 Src0->setIsKill(Src1->isKill()); 1001 1002 Src1->setReg(Src2Reg); 1003 Src1->setSubReg(Src2SubReg); 1004 Src1->setIsKill(Src2->isKill()); 1005 1006 if (Opc == AMDGPU::V_MAC_F32_e64) { 1007 UseMI->untieRegOperand( 1008 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 1009 } 1010 1011 Src2->ChangeToImmediate(Imm); 1012 1013 removeModOperands(*UseMI); 1014 UseMI->setDesc(get(AMDGPU::V_MADMK_F32)); 1015 1016 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 1017 if (DeleteDef) 1018 DefMI->eraseFromParent(); 1019 1020 return true; 1021 } 1022 1023 // Added part is the constant: Use v_madak_f32 1024 if (Src2->isReg() && Src2->getReg() == Reg) { 1025 // Not allowed to use constant bus for another operand. 1026 // We can however allow an inline immediate as src0. 1027 if (!Src0->isImm() && 1028 (Src0->isReg() && RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))) 1029 return false; 1030 1031 if (!Src1->isReg() || 1032 (Src1->isReg() && RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 1033 return false; 1034 1035 const int64_t Imm = DefMI->getOperand(1).getImm(); 1036 1037 // FIXME: This would be a lot easier if we could return a new instruction 1038 // instead of having to modify in place. 1039 1040 // Remove these first since they are at the end. 1041 UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, 1042 AMDGPU::OpName::omod)); 1043 UseMI->RemoveOperand(AMDGPU::getNamedOperandIdx(Opc, 1044 AMDGPU::OpName::clamp)); 1045 1046 if (Opc == AMDGPU::V_MAC_F32_e64) { 1047 UseMI->untieRegOperand( 1048 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 1049 } 1050 1051 // ChangingToImmediate adds Src2 back to the instruction. 1052 Src2->ChangeToImmediate(Imm); 1053 1054 // These come before src2. 1055 removeModOperands(*UseMI); 1056 UseMI->setDesc(get(AMDGPU::V_MADAK_F32)); 1057 1058 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 1059 if (DeleteDef) 1060 DefMI->eraseFromParent(); 1061 1062 return true; 1063 } 1064 } 1065 1066 return false; 1067 } 1068 1069 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 1070 int WidthB, int OffsetB) { 1071 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 1072 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 1073 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 1074 return LowOffset + LowWidth <= HighOffset; 1075 } 1076 1077 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr *MIa, 1078 MachineInstr *MIb) const { 1079 unsigned BaseReg0, Offset0; 1080 unsigned BaseReg1, Offset1; 1081 1082 if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) && 1083 getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) { 1084 assert(MIa->hasOneMemOperand() && MIb->hasOneMemOperand() && 1085 "read2 / write2 not expected here yet"); 1086 unsigned Width0 = (*MIa->memoperands_begin())->getSize(); 1087 unsigned Width1 = (*MIb->memoperands_begin())->getSize(); 1088 if (BaseReg0 == BaseReg1 && 1089 offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) { 1090 return true; 1091 } 1092 } 1093 1094 return false; 1095 } 1096 1097 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(MachineInstr *MIa, 1098 MachineInstr *MIb, 1099 AliasAnalysis *AA) const { 1100 assert(MIa && (MIa->mayLoad() || MIa->mayStore()) && 1101 "MIa must load from or modify a memory location"); 1102 assert(MIb && (MIb->mayLoad() || MIb->mayStore()) && 1103 "MIb must load from or modify a memory location"); 1104 1105 if (MIa->hasUnmodeledSideEffects() || MIb->hasUnmodeledSideEffects()) 1106 return false; 1107 1108 // XXX - Can we relax this between address spaces? 1109 if (MIa->hasOrderedMemoryRef() || MIb->hasOrderedMemoryRef()) 1110 return false; 1111 1112 // TODO: Should we check the address space from the MachineMemOperand? That 1113 // would allow us to distinguish objects we know don't alias based on the 1114 // underlying address space, even if it was lowered to a different one, 1115 // e.g. private accesses lowered to use MUBUF instructions on a scratch 1116 // buffer. 1117 if (isDS(*MIa)) { 1118 if (isDS(*MIb)) 1119 return checkInstOffsetsDoNotOverlap(MIa, MIb); 1120 1121 return !isFLAT(*MIb); 1122 } 1123 1124 if (isMUBUF(*MIa) || isMTBUF(*MIa)) { 1125 if (isMUBUF(*MIb) || isMTBUF(*MIb)) 1126 return checkInstOffsetsDoNotOverlap(MIa, MIb); 1127 1128 return !isFLAT(*MIb) && !isSMRD(*MIb); 1129 } 1130 1131 if (isSMRD(*MIa)) { 1132 if (isSMRD(*MIb)) 1133 return checkInstOffsetsDoNotOverlap(MIa, MIb); 1134 1135 return !isFLAT(*MIb) && !isMUBUF(*MIa) && !isMTBUF(*MIa); 1136 } 1137 1138 if (isFLAT(*MIa)) { 1139 if (isFLAT(*MIb)) 1140 return checkInstOffsetsDoNotOverlap(MIa, MIb); 1141 1142 return false; 1143 } 1144 1145 return false; 1146 } 1147 1148 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB, 1149 MachineBasicBlock::iterator &MI, 1150 LiveVariables *LV) const { 1151 1152 switch (MI->getOpcode()) { 1153 default: return nullptr; 1154 case AMDGPU::V_MAC_F32_e64: break; 1155 case AMDGPU::V_MAC_F32_e32: { 1156 const MachineOperand *Src0 = getNamedOperand(*MI, AMDGPU::OpName::src0); 1157 if (Src0->isImm() && !isInlineConstant(*Src0, 4)) 1158 return nullptr; 1159 break; 1160 } 1161 } 1162 1163 const MachineOperand *Dst = getNamedOperand(*MI, AMDGPU::OpName::dst); 1164 const MachineOperand *Src0 = getNamedOperand(*MI, AMDGPU::OpName::src0); 1165 const MachineOperand *Src1 = getNamedOperand(*MI, AMDGPU::OpName::src1); 1166 const MachineOperand *Src2 = getNamedOperand(*MI, AMDGPU::OpName::src2); 1167 1168 return BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::V_MAD_F32)) 1169 .addOperand(*Dst) 1170 .addImm(0) // Src0 mods 1171 .addOperand(*Src0) 1172 .addImm(0) // Src1 mods 1173 .addOperand(*Src1) 1174 .addImm(0) // Src mods 1175 .addOperand(*Src2) 1176 .addImm(0) // clamp 1177 .addImm(0); // omod 1178 } 1179 1180 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 1181 int64_t SVal = Imm.getSExtValue(); 1182 if (SVal >= -16 && SVal <= 64) 1183 return true; 1184 1185 if (Imm.getBitWidth() == 64) { 1186 uint64_t Val = Imm.getZExtValue(); 1187 return (DoubleToBits(0.0) == Val) || 1188 (DoubleToBits(1.0) == Val) || 1189 (DoubleToBits(-1.0) == Val) || 1190 (DoubleToBits(0.5) == Val) || 1191 (DoubleToBits(-0.5) == Val) || 1192 (DoubleToBits(2.0) == Val) || 1193 (DoubleToBits(-2.0) == Val) || 1194 (DoubleToBits(4.0) == Val) || 1195 (DoubleToBits(-4.0) == Val); 1196 } 1197 1198 // The actual type of the operand does not seem to matter as long 1199 // as the bits match one of the inline immediate values. For example: 1200 // 1201 // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal, 1202 // so it is a legal inline immediate. 1203 // 1204 // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in 1205 // floating-point, so it is a legal inline immediate. 1206 uint32_t Val = Imm.getZExtValue(); 1207 1208 return (FloatToBits(0.0f) == Val) || 1209 (FloatToBits(1.0f) == Val) || 1210 (FloatToBits(-1.0f) == Val) || 1211 (FloatToBits(0.5f) == Val) || 1212 (FloatToBits(-0.5f) == Val) || 1213 (FloatToBits(2.0f) == Val) || 1214 (FloatToBits(-2.0f) == Val) || 1215 (FloatToBits(4.0f) == Val) || 1216 (FloatToBits(-4.0f) == Val); 1217 } 1218 1219 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 1220 unsigned OpSize) const { 1221 if (MO.isImm()) { 1222 // MachineOperand provides no way to tell the true operand size, since it 1223 // only records a 64-bit value. We need to know the size to determine if a 1224 // 32-bit floating point immediate bit pattern is legal for an integer 1225 // immediate. It would be for any 32-bit integer operand, but would not be 1226 // for a 64-bit one. 1227 1228 unsigned BitSize = 8 * OpSize; 1229 return isInlineConstant(APInt(BitSize, MO.getImm(), true)); 1230 } 1231 1232 return false; 1233 } 1234 1235 bool SIInstrInfo::isLiteralConstant(const MachineOperand &MO, 1236 unsigned OpSize) const { 1237 return MO.isImm() && !isInlineConstant(MO, OpSize); 1238 } 1239 1240 static bool compareMachineOp(const MachineOperand &Op0, 1241 const MachineOperand &Op1) { 1242 if (Op0.getType() != Op1.getType()) 1243 return false; 1244 1245 switch (Op0.getType()) { 1246 case MachineOperand::MO_Register: 1247 return Op0.getReg() == Op1.getReg(); 1248 case MachineOperand::MO_Immediate: 1249 return Op0.getImm() == Op1.getImm(); 1250 default: 1251 llvm_unreachable("Didn't expect to be comparing these operand types"); 1252 } 1253 } 1254 1255 bool SIInstrInfo::isImmOperandLegal(const MachineInstr *MI, unsigned OpNo, 1256 const MachineOperand &MO) const { 1257 const MCOperandInfo &OpInfo = get(MI->getOpcode()).OpInfo[OpNo]; 1258 1259 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI()); 1260 1261 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 1262 return true; 1263 1264 if (OpInfo.RegClass < 0) 1265 return false; 1266 1267 unsigned OpSize = RI.getRegClass(OpInfo.RegClass)->getSize(); 1268 if (isLiteralConstant(MO, OpSize)) 1269 return RI.opCanUseLiteralConstant(OpInfo.OperandType); 1270 1271 return RI.opCanUseInlineConstant(OpInfo.OperandType); 1272 } 1273 1274 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 1275 int Op32 = AMDGPU::getVOPe32(Opcode); 1276 if (Op32 == -1) 1277 return false; 1278 1279 return pseudoToMCOpcode(Op32) != -1; 1280 } 1281 1282 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 1283 // The src0_modifier operand is present on all instructions 1284 // that have modifiers. 1285 1286 return AMDGPU::getNamedOperandIdx(Opcode, 1287 AMDGPU::OpName::src0_modifiers) != -1; 1288 } 1289 1290 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 1291 unsigned OpName) const { 1292 const MachineOperand *Mods = getNamedOperand(MI, OpName); 1293 return Mods && Mods->getImm(); 1294 } 1295 1296 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 1297 const MachineOperand &MO, 1298 unsigned OpSize) const { 1299 // Literal constants use the constant bus. 1300 if (isLiteralConstant(MO, OpSize)) 1301 return true; 1302 1303 if (!MO.isReg() || !MO.isUse()) 1304 return false; 1305 1306 if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) 1307 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 1308 1309 // FLAT_SCR is just an SGPR pair. 1310 if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR)) 1311 return true; 1312 1313 // EXEC register uses the constant bus. 1314 if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC) 1315 return true; 1316 1317 // SGPRs use the constant bus 1318 if (MO.getReg() == AMDGPU::M0 || MO.getReg() == AMDGPU::VCC || 1319 (!MO.isImplicit() && 1320 (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) || 1321 AMDGPU::SGPR_64RegClass.contains(MO.getReg())))) { 1322 return true; 1323 } 1324 1325 return false; 1326 } 1327 1328 static unsigned findImplicitSGPRRead(const MachineInstr &MI) { 1329 for (const MachineOperand &MO : MI.implicit_operands()) { 1330 // We only care about reads. 1331 if (MO.isDef()) 1332 continue; 1333 1334 switch (MO.getReg()) { 1335 case AMDGPU::VCC: 1336 case AMDGPU::M0: 1337 case AMDGPU::FLAT_SCR: 1338 return MO.getReg(); 1339 1340 default: 1341 break; 1342 } 1343 } 1344 1345 return AMDGPU::NoRegister; 1346 } 1347 1348 bool SIInstrInfo::verifyInstruction(const MachineInstr *MI, 1349 StringRef &ErrInfo) const { 1350 uint16_t Opcode = MI->getOpcode(); 1351 const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); 1352 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 1353 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 1354 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 1355 1356 // Make sure the number of operands is correct. 1357 const MCInstrDesc &Desc = get(Opcode); 1358 if (!Desc.isVariadic() && 1359 Desc.getNumOperands() != MI->getNumExplicitOperands()) { 1360 ErrInfo = "Instruction has wrong number of operands."; 1361 return false; 1362 } 1363 1364 // Make sure the register classes are correct 1365 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 1366 if (MI->getOperand(i).isFPImm()) { 1367 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 1368 "all fp values to integers."; 1369 return false; 1370 } 1371 1372 int RegClass = Desc.OpInfo[i].RegClass; 1373 1374 switch (Desc.OpInfo[i].OperandType) { 1375 case MCOI::OPERAND_REGISTER: 1376 if (MI->getOperand(i).isImm()) { 1377 ErrInfo = "Illegal immediate value for operand."; 1378 return false; 1379 } 1380 break; 1381 case AMDGPU::OPERAND_REG_IMM32: 1382 break; 1383 case AMDGPU::OPERAND_REG_INLINE_C: 1384 if (isLiteralConstant(MI->getOperand(i), 1385 RI.getRegClass(RegClass)->getSize())) { 1386 ErrInfo = "Illegal immediate value for operand."; 1387 return false; 1388 } 1389 break; 1390 case MCOI::OPERAND_IMMEDIATE: 1391 // Check if this operand is an immediate. 1392 // FrameIndex operands will be replaced by immediates, so they are 1393 // allowed. 1394 if (!MI->getOperand(i).isImm() && !MI->getOperand(i).isFI()) { 1395 ErrInfo = "Expected immediate, but got non-immediate"; 1396 return false; 1397 } 1398 // Fall-through 1399 default: 1400 continue; 1401 } 1402 1403 if (!MI->getOperand(i).isReg()) 1404 continue; 1405 1406 if (RegClass != -1) { 1407 unsigned Reg = MI->getOperand(i).getReg(); 1408 if (TargetRegisterInfo::isVirtualRegister(Reg)) 1409 continue; 1410 1411 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 1412 if (!RC->contains(Reg)) { 1413 ErrInfo = "Operand has incorrect register class."; 1414 return false; 1415 } 1416 } 1417 } 1418 1419 1420 // Verify VOP* 1421 if (isVOP1(*MI) || isVOP2(*MI) || isVOP3(*MI) || isVOPC(*MI)) { 1422 // Only look at the true operands. Only a real operand can use the constant 1423 // bus, and we don't want to check pseudo-operands like the source modifier 1424 // flags. 1425 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx }; 1426 1427 unsigned ConstantBusCount = 0; 1428 unsigned SGPRUsed = findImplicitSGPRRead(*MI); 1429 if (SGPRUsed != AMDGPU::NoRegister) 1430 ++ConstantBusCount; 1431 1432 for (int OpIdx : OpIndices) { 1433 if (OpIdx == -1) 1434 break; 1435 const MachineOperand &MO = MI->getOperand(OpIdx); 1436 if (usesConstantBus(MRI, MO, getOpSize(Opcode, OpIdx))) { 1437 if (MO.isReg()) { 1438 if (MO.getReg() != SGPRUsed) 1439 ++ConstantBusCount; 1440 SGPRUsed = MO.getReg(); 1441 } else { 1442 ++ConstantBusCount; 1443 } 1444 } 1445 } 1446 if (ConstantBusCount > 1) { 1447 ErrInfo = "VOP* instruction uses the constant bus more than once"; 1448 return false; 1449 } 1450 } 1451 1452 // Verify misc. restrictions on specific instructions. 1453 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 || 1454 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) { 1455 const MachineOperand &Src0 = MI->getOperand(Src0Idx); 1456 const MachineOperand &Src1 = MI->getOperand(Src1Idx); 1457 const MachineOperand &Src2 = MI->getOperand(Src2Idx); 1458 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 1459 if (!compareMachineOp(Src0, Src1) && 1460 !compareMachineOp(Src0, Src2)) { 1461 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 1462 return false; 1463 } 1464 } 1465 } 1466 1467 // Make sure we aren't losing exec uses in the td files. This mostly requires 1468 // being careful when using let Uses to try to add other use registers. 1469 if (!isGenericOpcode(Opcode) && !isSALU(Opcode) && !isSMRD(Opcode)) { 1470 const MachineOperand *Exec = MI->findRegisterUseOperand(AMDGPU::EXEC); 1471 if (!Exec || !Exec->isImplicit()) { 1472 ErrInfo = "VALU instruction does not implicitly read exec mask"; 1473 return false; 1474 } 1475 } 1476 1477 return true; 1478 } 1479 1480 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) { 1481 switch (MI.getOpcode()) { 1482 default: return AMDGPU::INSTRUCTION_LIST_END; 1483 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 1484 case AMDGPU::COPY: return AMDGPU::COPY; 1485 case AMDGPU::PHI: return AMDGPU::PHI; 1486 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 1487 case AMDGPU::S_MOV_B32: 1488 return MI.getOperand(1).isReg() ? 1489 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 1490 case AMDGPU::S_ADD_I32: 1491 case AMDGPU::S_ADD_U32: return AMDGPU::V_ADD_I32_e32; 1492 case AMDGPU::S_ADDC_U32: return AMDGPU::V_ADDC_U32_e32; 1493 case AMDGPU::S_SUB_I32: 1494 case AMDGPU::S_SUB_U32: return AMDGPU::V_SUB_I32_e32; 1495 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 1496 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32; 1497 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e32; 1498 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e32; 1499 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e32; 1500 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e32; 1501 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e32; 1502 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e32; 1503 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e32; 1504 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 1505 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64; 1506 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 1507 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64; 1508 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 1509 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64; 1510 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32; 1511 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32; 1512 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32; 1513 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32; 1514 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 1515 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 1516 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 1517 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 1518 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; 1519 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; 1520 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; 1521 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; 1522 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; 1523 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; 1524 case AMDGPU::S_LOAD_DWORD_IMM: 1525 case AMDGPU::S_LOAD_DWORD_SGPR: 1526 case AMDGPU::S_LOAD_DWORD_IMM_ci: 1527 return AMDGPU::BUFFER_LOAD_DWORD_ADDR64; 1528 case AMDGPU::S_LOAD_DWORDX2_IMM: 1529 case AMDGPU::S_LOAD_DWORDX2_SGPR: 1530 case AMDGPU::S_LOAD_DWORDX2_IMM_ci: 1531 return AMDGPU::BUFFER_LOAD_DWORDX2_ADDR64; 1532 case AMDGPU::S_LOAD_DWORDX4_IMM: 1533 case AMDGPU::S_LOAD_DWORDX4_SGPR: 1534 case AMDGPU::S_LOAD_DWORDX4_IMM_ci: 1535 return AMDGPU::BUFFER_LOAD_DWORDX4_ADDR64; 1536 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 1537 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 1538 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 1539 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 1540 } 1541 } 1542 1543 bool SIInstrInfo::isSALUOpSupportedOnVALU(const MachineInstr &MI) const { 1544 return getVALUOp(MI) != AMDGPU::INSTRUCTION_LIST_END; 1545 } 1546 1547 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 1548 unsigned OpNo) const { 1549 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 1550 const MCInstrDesc &Desc = get(MI.getOpcode()); 1551 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 1552 Desc.OpInfo[OpNo].RegClass == -1) { 1553 unsigned Reg = MI.getOperand(OpNo).getReg(); 1554 1555 if (TargetRegisterInfo::isVirtualRegister(Reg)) 1556 return MRI.getRegClass(Reg); 1557 return RI.getPhysRegClass(Reg); 1558 } 1559 1560 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 1561 return RI.getRegClass(RCID); 1562 } 1563 1564 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const { 1565 switch (MI.getOpcode()) { 1566 case AMDGPU::COPY: 1567 case AMDGPU::REG_SEQUENCE: 1568 case AMDGPU::PHI: 1569 case AMDGPU::INSERT_SUBREG: 1570 return RI.hasVGPRs(getOpRegClass(MI, 0)); 1571 default: 1572 return RI.hasVGPRs(getOpRegClass(MI, OpNo)); 1573 } 1574 } 1575 1576 void SIInstrInfo::legalizeOpWithMove(MachineInstr *MI, unsigned OpIdx) const { 1577 MachineBasicBlock::iterator I = MI; 1578 MachineBasicBlock *MBB = MI->getParent(); 1579 MachineOperand &MO = MI->getOperand(OpIdx); 1580 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1581 unsigned RCID = get(MI->getOpcode()).OpInfo[OpIdx].RegClass; 1582 const TargetRegisterClass *RC = RI.getRegClass(RCID); 1583 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1584 if (MO.isReg()) 1585 Opcode = AMDGPU::COPY; 1586 else if (RI.isSGPRClass(RC)) 1587 Opcode = AMDGPU::S_MOV_B32; 1588 1589 1590 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 1591 if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC)) 1592 VRC = &AMDGPU::VReg_64RegClass; 1593 else 1594 VRC = &AMDGPU::VGPR_32RegClass; 1595 1596 unsigned Reg = MRI.createVirtualRegister(VRC); 1597 DebugLoc DL = MBB->findDebugLoc(I); 1598 BuildMI(*MI->getParent(), I, DL, get(Opcode), Reg) 1599 .addOperand(MO); 1600 MO.ChangeToRegister(Reg, false); 1601 } 1602 1603 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI, 1604 MachineRegisterInfo &MRI, 1605 MachineOperand &SuperReg, 1606 const TargetRegisterClass *SuperRC, 1607 unsigned SubIdx, 1608 const TargetRegisterClass *SubRC) 1609 const { 1610 MachineBasicBlock *MBB = MI->getParent(); 1611 DebugLoc DL = MI->getDebugLoc(); 1612 unsigned SubReg = MRI.createVirtualRegister(SubRC); 1613 1614 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 1615 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 1616 .addReg(SuperReg.getReg(), 0, SubIdx); 1617 return SubReg; 1618 } 1619 1620 // Just in case the super register is itself a sub-register, copy it to a new 1621 // value so we don't need to worry about merging its subreg index with the 1622 // SubIdx passed to this function. The register coalescer should be able to 1623 // eliminate this extra copy. 1624 unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC); 1625 1626 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 1627 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 1628 1629 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 1630 .addReg(NewSuperReg, 0, SubIdx); 1631 1632 return SubReg; 1633 } 1634 1635 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 1636 MachineBasicBlock::iterator MII, 1637 MachineRegisterInfo &MRI, 1638 MachineOperand &Op, 1639 const TargetRegisterClass *SuperRC, 1640 unsigned SubIdx, 1641 const TargetRegisterClass *SubRC) const { 1642 if (Op.isImm()) { 1643 // XXX - Is there a better way to do this? 1644 if (SubIdx == AMDGPU::sub0) 1645 return MachineOperand::CreateImm(Op.getImm() & 0xFFFFFFFF); 1646 if (SubIdx == AMDGPU::sub1) 1647 return MachineOperand::CreateImm(Op.getImm() >> 32); 1648 1649 llvm_unreachable("Unhandled register index for immediate"); 1650 } 1651 1652 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 1653 SubIdx, SubRC); 1654 return MachineOperand::CreateReg(SubReg, false); 1655 } 1656 1657 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 1658 void SIInstrInfo::swapOperands(MachineBasicBlock::iterator Inst) const { 1659 assert(Inst->getNumExplicitOperands() == 3); 1660 MachineOperand Op1 = Inst->getOperand(1); 1661 Inst->RemoveOperand(1); 1662 Inst->addOperand(Op1); 1663 } 1664 1665 bool SIInstrInfo::isOperandLegal(const MachineInstr *MI, unsigned OpIdx, 1666 const MachineOperand *MO) const { 1667 const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); 1668 const MCInstrDesc &InstDesc = get(MI->getOpcode()); 1669 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx]; 1670 const TargetRegisterClass *DefinedRC = 1671 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 1672 if (!MO) 1673 MO = &MI->getOperand(OpIdx); 1674 1675 if (isVALU(*MI) && 1676 usesConstantBus(MRI, *MO, DefinedRC->getSize())) { 1677 unsigned SGPRUsed = 1678 MO->isReg() ? MO->getReg() : (unsigned)AMDGPU::NoRegister; 1679 for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { 1680 if (i == OpIdx) 1681 continue; 1682 const MachineOperand &Op = MI->getOperand(i); 1683 if (Op.isReg() && Op.getReg() != SGPRUsed && 1684 usesConstantBus(MRI, Op, getOpSize(*MI, i))) { 1685 return false; 1686 } 1687 } 1688 } 1689 1690 if (MO->isReg()) { 1691 assert(DefinedRC); 1692 const TargetRegisterClass *RC = 1693 TargetRegisterInfo::isVirtualRegister(MO->getReg()) ? 1694 MRI.getRegClass(MO->getReg()) : 1695 RI.getPhysRegClass(MO->getReg()); 1696 1697 // In order to be legal, the common sub-class must be equal to the 1698 // class of the current operand. For example: 1699 // 1700 // v_mov_b32 s0 ; Operand defined as vsrc_32 1701 // ; RI.getCommonSubClass(s0,vsrc_32) = sgpr ; LEGAL 1702 // 1703 // s_sendmsg 0, s0 ; Operand defined as m0reg 1704 // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL 1705 1706 return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC; 1707 } 1708 1709 1710 // Handle non-register types that are treated like immediates. 1711 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI()); 1712 1713 if (!DefinedRC) { 1714 // This operand expects an immediate. 1715 return true; 1716 } 1717 1718 return isImmOperandLegal(MI, OpIdx, *MO); 1719 } 1720 1721 // Legalize VOP3 operands. Because all operand types are supported for any 1722 // operand, and since literal constants are not allowed and should never be 1723 // seen, we only need to worry about inserting copies if we use multiple SGPR 1724 // operands. 1725 void SIInstrInfo::legalizeOperandsVOP3( 1726 MachineRegisterInfo &MRI, 1727 MachineInstr *MI) const { 1728 unsigned Opc = MI->getOpcode(); 1729 1730 int VOP3Idx[3] = { 1731 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 1732 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 1733 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 1734 }; 1735 1736 // Find the one SGPR operand we are allowed to use. 1737 unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx); 1738 1739 for (unsigned i = 0; i < 3; ++i) { 1740 int Idx = VOP3Idx[i]; 1741 if (Idx == -1) 1742 break; 1743 MachineOperand &MO = MI->getOperand(Idx); 1744 1745 // We should never see a VOP3 instruction with an illegal immediate operand. 1746 if (!MO.isReg()) 1747 continue; 1748 1749 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg()))) 1750 continue; // VGPRs are legal 1751 1752 if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) { 1753 SGPRReg = MO.getReg(); 1754 // We can use one SGPR in each VOP3 instruction. 1755 continue; 1756 } 1757 1758 // If we make it this far, then the operand is not legal and we must 1759 // legalize it. 1760 legalizeOpWithMove(MI, Idx); 1761 } 1762 } 1763 1764 void SIInstrInfo::legalizeOperands(MachineInstr *MI) const { 1765 MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); 1766 unsigned Opc = MI->getOpcode(); 1767 1768 // Legalize VOP2 1769 if (isVOP2(*MI)) { 1770 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 1771 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 1772 1773 // Legalize src0 1774 if (!isOperandLegal(MI, Src0Idx)) 1775 legalizeOpWithMove(MI, Src0Idx); 1776 1777 // Legalize src1 1778 if (isOperandLegal(MI, Src1Idx)) 1779 return; 1780 1781 // Usually src0 of VOP2 instructions allow more types of inputs 1782 // than src1, so try to commute the instruction to decrease our 1783 // chances of having to insert a MOV instruction to legalize src1. 1784 if (MI->isCommutable()) { 1785 if (commuteInstruction(MI)) 1786 // If we are successful in commuting, then we know MI is legal, so 1787 // we are done. 1788 return; 1789 } 1790 1791 legalizeOpWithMove(MI, Src1Idx); 1792 return; 1793 } 1794 1795 // Legalize VOP3 1796 if (isVOP3(*MI)) { 1797 legalizeOperandsVOP3(MRI, MI); 1798 return; 1799 } 1800 1801 // Legalize REG_SEQUENCE and PHI 1802 // The register class of the operands much be the same type as the register 1803 // class of the output. 1804 if (MI->getOpcode() == AMDGPU::PHI) { 1805 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 1806 for (unsigned i = 1, e = MI->getNumOperands(); i != e; i+=2) { 1807 if (!MI->getOperand(i).isReg() || 1808 !TargetRegisterInfo::isVirtualRegister(MI->getOperand(i).getReg())) 1809 continue; 1810 const TargetRegisterClass *OpRC = 1811 MRI.getRegClass(MI->getOperand(i).getReg()); 1812 if (RI.hasVGPRs(OpRC)) { 1813 VRC = OpRC; 1814 } else { 1815 SRC = OpRC; 1816 } 1817 } 1818 1819 // If any of the operands are VGPR registers, then they all most be 1820 // otherwise we will create illegal VGPR->SGPR copies when legalizing 1821 // them. 1822 if (VRC || !RI.isSGPRClass(getOpRegClass(*MI, 0))) { 1823 if (!VRC) { 1824 assert(SRC); 1825 VRC = RI.getEquivalentVGPRClass(SRC); 1826 } 1827 RC = VRC; 1828 } else { 1829 RC = SRC; 1830 } 1831 1832 // Update all the operands so they have the same type. 1833 for (unsigned I = 1, E = MI->getNumOperands(); I != E; I += 2) { 1834 MachineOperand &Op = MI->getOperand(I); 1835 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg())) 1836 continue; 1837 unsigned DstReg = MRI.createVirtualRegister(RC); 1838 1839 // MI is a PHI instruction. 1840 MachineBasicBlock *InsertBB = MI->getOperand(I + 1).getMBB(); 1841 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 1842 1843 BuildMI(*InsertBB, Insert, MI->getDebugLoc(), get(AMDGPU::COPY), DstReg) 1844 .addOperand(Op); 1845 Op.setReg(DstReg); 1846 } 1847 } 1848 1849 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 1850 // VGPR dest type and SGPR sources, insert copies so all operands are 1851 // VGPRs. This seems to help operand folding / the register coalescer. 1852 if (MI->getOpcode() == AMDGPU::REG_SEQUENCE) { 1853 MachineBasicBlock *MBB = MI->getParent(); 1854 const TargetRegisterClass *DstRC = getOpRegClass(*MI, 0); 1855 if (RI.hasVGPRs(DstRC)) { 1856 // Update all the operands so they are VGPR register classes. These may 1857 // not be the same register class because REG_SEQUENCE supports mixing 1858 // subregister index types e.g. sub0_sub1 + sub2 + sub3 1859 for (unsigned I = 1, E = MI->getNumOperands(); I != E; I += 2) { 1860 MachineOperand &Op = MI->getOperand(I); 1861 if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg())) 1862 continue; 1863 1864 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 1865 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 1866 if (VRC == OpRC) 1867 continue; 1868 1869 unsigned DstReg = MRI.createVirtualRegister(VRC); 1870 1871 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::COPY), DstReg) 1872 .addOperand(Op); 1873 1874 Op.setReg(DstReg); 1875 Op.setIsKill(); 1876 } 1877 } 1878 1879 return; 1880 } 1881 1882 // Legalize INSERT_SUBREG 1883 // src0 must have the same register class as dst 1884 if (MI->getOpcode() == AMDGPU::INSERT_SUBREG) { 1885 unsigned Dst = MI->getOperand(0).getReg(); 1886 unsigned Src0 = MI->getOperand(1).getReg(); 1887 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 1888 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 1889 if (DstRC != Src0RC) { 1890 MachineBasicBlock &MBB = *MI->getParent(); 1891 unsigned NewSrc0 = MRI.createVirtualRegister(DstRC); 1892 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::COPY), NewSrc0) 1893 .addReg(Src0); 1894 MI->getOperand(1).setReg(NewSrc0); 1895 } 1896 return; 1897 } 1898 1899 // Legalize MUBUF* instructions 1900 // FIXME: If we start using the non-addr64 instructions for compute, we 1901 // may need to legalize them here. 1902 int SRsrcIdx = 1903 AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::srsrc); 1904 if (SRsrcIdx != -1) { 1905 // We have an MUBUF instruction 1906 MachineOperand *SRsrc = &MI->getOperand(SRsrcIdx); 1907 unsigned SRsrcRC = get(MI->getOpcode()).OpInfo[SRsrcIdx].RegClass; 1908 if (RI.getCommonSubClass(MRI.getRegClass(SRsrc->getReg()), 1909 RI.getRegClass(SRsrcRC))) { 1910 // The operands are legal. 1911 // FIXME: We may need to legalize operands besided srsrc. 1912 return; 1913 } 1914 1915 MachineBasicBlock &MBB = *MI->getParent(); 1916 1917 // Extract the ptr from the resource descriptor. 1918 unsigned SRsrcPtr = buildExtractSubReg(MI, MRI, *SRsrc, 1919 &AMDGPU::VReg_128RegClass, AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 1920 1921 // Create an empty resource descriptor 1922 unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 1923 unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 1924 unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 1925 unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass); 1926 uint64_t RsrcDataFormat = getDefaultRsrcDataFormat(); 1927 1928 // Zero64 = 0 1929 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B64), 1930 Zero64) 1931 .addImm(0); 1932 1933 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 1934 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), 1935 SRsrcFormatLo) 1936 .addImm(RsrcDataFormat & 0xFFFFFFFF); 1937 1938 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 1939 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), 1940 SRsrcFormatHi) 1941 .addImm(RsrcDataFormat >> 32); 1942 1943 // NewSRsrc = {Zero64, SRsrcFormat} 1944 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewSRsrc) 1945 .addReg(Zero64) 1946 .addImm(AMDGPU::sub0_sub1) 1947 .addReg(SRsrcFormatLo) 1948 .addImm(AMDGPU::sub2) 1949 .addReg(SRsrcFormatHi) 1950 .addImm(AMDGPU::sub3); 1951 1952 MachineOperand *VAddr = getNamedOperand(*MI, AMDGPU::OpName::vaddr); 1953 unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 1954 if (VAddr) { 1955 // This is already an ADDR64 instruction so we need to add the pointer 1956 // extracted from the resource descriptor to the current value of VAddr. 1957 unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1958 unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 1959 1960 // NewVaddrLo = SRsrcPtr:sub0 + VAddr:sub0 1961 DebugLoc DL = MI->getDebugLoc(); 1962 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo) 1963 .addReg(SRsrcPtr, 0, AMDGPU::sub0) 1964 .addReg(VAddr->getReg(), 0, AMDGPU::sub0); 1965 1966 // NewVaddrHi = SRsrcPtr:sub1 + VAddr:sub1 1967 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi) 1968 .addReg(SRsrcPtr, 0, AMDGPU::sub1) 1969 .addReg(VAddr->getReg(), 0, AMDGPU::sub1); 1970 1971 // NewVaddr = {NewVaddrHi, NewVaddrLo} 1972 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 1973 .addReg(NewVAddrLo) 1974 .addImm(AMDGPU::sub0) 1975 .addReg(NewVAddrHi) 1976 .addImm(AMDGPU::sub1); 1977 } else { 1978 // This instructions is the _OFFSET variant, so we need to convert it to 1979 // ADDR64. 1980 MachineOperand *VData = getNamedOperand(*MI, AMDGPU::OpName::vdata); 1981 MachineOperand *Offset = getNamedOperand(*MI, AMDGPU::OpName::offset); 1982 MachineOperand *SOffset = getNamedOperand(*MI, AMDGPU::OpName::soffset); 1983 1984 // Create the new instruction. 1985 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI->getOpcode()); 1986 MachineInstr *Addr64 = 1987 BuildMI(MBB, MI, MI->getDebugLoc(), get(Addr64Opcode)) 1988 .addOperand(*VData) 1989 .addReg(AMDGPU::NoRegister) // Dummy value for vaddr. 1990 // This will be replaced later 1991 // with the new value of vaddr. 1992 .addOperand(*SRsrc) 1993 .addOperand(*SOffset) 1994 .addOperand(*Offset) 1995 .addImm(0) // glc 1996 .addImm(0) // slc 1997 .addImm(0) // tfe 1998 .setMemRefs(MI->memoperands_begin(), MI->memoperands_end()); 1999 2000 MI->removeFromParent(); 2001 MI = Addr64; 2002 2003 // NewVaddr = {NewVaddrHi, NewVaddrLo} 2004 BuildMI(MBB, MI, MI->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 2005 .addReg(SRsrcPtr, 0, AMDGPU::sub0) 2006 .addImm(AMDGPU::sub0) 2007 .addReg(SRsrcPtr, 0, AMDGPU::sub1) 2008 .addImm(AMDGPU::sub1); 2009 2010 VAddr = getNamedOperand(*MI, AMDGPU::OpName::vaddr); 2011 SRsrc = getNamedOperand(*MI, AMDGPU::OpName::srsrc); 2012 } 2013 2014 // Update the instruction to use NewVaddr 2015 VAddr->setReg(NewVAddr); 2016 // Update the instruction to use NewSRsrc 2017 SRsrc->setReg(NewSRsrc); 2018 } 2019 } 2020 2021 void SIInstrInfo::splitSMRD(MachineInstr *MI, 2022 const TargetRegisterClass *HalfRC, 2023 unsigned HalfImmOp, unsigned HalfSGPROp, 2024 MachineInstr *&Lo, MachineInstr *&Hi) const { 2025 2026 DebugLoc DL = MI->getDebugLoc(); 2027 MachineBasicBlock *MBB = MI->getParent(); 2028 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 2029 unsigned RegLo = MRI.createVirtualRegister(HalfRC); 2030 unsigned RegHi = MRI.createVirtualRegister(HalfRC); 2031 unsigned HalfSize = HalfRC->getSize(); 2032 const MachineOperand *OffOp = 2033 getNamedOperand(*MI, AMDGPU::OpName::offset); 2034 const MachineOperand *SBase = getNamedOperand(*MI, AMDGPU::OpName::sbase); 2035 2036 // The SMRD has an 8-bit offset in dwords on SI and a 20-bit offset in bytes 2037 // on VI. 2038 2039 bool IsKill = SBase->isKill(); 2040 if (OffOp) { 2041 bool isVI = 2042 MBB->getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() >= 2043 AMDGPUSubtarget::VOLCANIC_ISLANDS; 2044 unsigned OffScale = isVI ? 1 : 4; 2045 // Handle the _IMM variant 2046 unsigned LoOffset = OffOp->getImm() * OffScale; 2047 unsigned HiOffset = LoOffset + HalfSize; 2048 Lo = BuildMI(*MBB, MI, DL, get(HalfImmOp), RegLo) 2049 // Use addReg instead of addOperand 2050 // to make sure kill flag is cleared. 2051 .addReg(SBase->getReg(), 0, SBase->getSubReg()) 2052 .addImm(LoOffset / OffScale); 2053 2054 if (!isUInt<20>(HiOffset) || (!isVI && !isUInt<8>(HiOffset / OffScale))) { 2055 unsigned OffsetSGPR = 2056 MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 2057 BuildMI(*MBB, MI, DL, get(AMDGPU::S_MOV_B32), OffsetSGPR) 2058 .addImm(HiOffset); // The offset in register is in bytes. 2059 Hi = BuildMI(*MBB, MI, DL, get(HalfSGPROp), RegHi) 2060 .addReg(SBase->getReg(), getKillRegState(IsKill), 2061 SBase->getSubReg()) 2062 .addReg(OffsetSGPR); 2063 } else { 2064 Hi = BuildMI(*MBB, MI, DL, get(HalfImmOp), RegHi) 2065 .addReg(SBase->getReg(), getKillRegState(IsKill), 2066 SBase->getSubReg()) 2067 .addImm(HiOffset / OffScale); 2068 } 2069 } else { 2070 // Handle the _SGPR variant 2071 MachineOperand *SOff = getNamedOperand(*MI, AMDGPU::OpName::soff); 2072 Lo = BuildMI(*MBB, MI, DL, get(HalfSGPROp), RegLo) 2073 .addReg(SBase->getReg(), 0, SBase->getSubReg()) 2074 .addOperand(*SOff); 2075 unsigned OffsetSGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 2076 BuildMI(*MBB, MI, DL, get(AMDGPU::S_ADD_I32), OffsetSGPR) 2077 .addReg(SOff->getReg(), 0, SOff->getSubReg()) 2078 .addImm(HalfSize); 2079 Hi = BuildMI(*MBB, MI, DL, get(HalfSGPROp), RegHi) 2080 .addReg(SBase->getReg(), getKillRegState(IsKill), 2081 SBase->getSubReg()) 2082 .addReg(OffsetSGPR); 2083 } 2084 2085 unsigned SubLo, SubHi; 2086 const TargetRegisterClass *NewDstRC; 2087 switch (HalfSize) { 2088 case 4: 2089 SubLo = AMDGPU::sub0; 2090 SubHi = AMDGPU::sub1; 2091 NewDstRC = &AMDGPU::VReg_64RegClass; 2092 break; 2093 case 8: 2094 SubLo = AMDGPU::sub0_sub1; 2095 SubHi = AMDGPU::sub2_sub3; 2096 NewDstRC = &AMDGPU::VReg_128RegClass; 2097 break; 2098 case 16: 2099 SubLo = AMDGPU::sub0_sub1_sub2_sub3; 2100 SubHi = AMDGPU::sub4_sub5_sub6_sub7; 2101 NewDstRC = &AMDGPU::VReg_256RegClass; 2102 break; 2103 case 32: 2104 SubLo = AMDGPU::sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7; 2105 SubHi = AMDGPU::sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15; 2106 NewDstRC = &AMDGPU::VReg_512RegClass; 2107 break; 2108 default: 2109 llvm_unreachable("Unhandled HalfSize"); 2110 } 2111 2112 unsigned OldDst = MI->getOperand(0).getReg(); 2113 unsigned NewDst = MRI.createVirtualRegister(NewDstRC); 2114 2115 MRI.replaceRegWith(OldDst, NewDst); 2116 2117 BuildMI(*MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewDst) 2118 .addReg(RegLo) 2119 .addImm(SubLo) 2120 .addReg(RegHi) 2121 .addImm(SubHi); 2122 } 2123 2124 void SIInstrInfo::moveSMRDToVALU(MachineInstr *MI, 2125 MachineRegisterInfo &MRI, 2126 SmallVectorImpl<MachineInstr *> &Worklist) const { 2127 MachineBasicBlock *MBB = MI->getParent(); 2128 int DstIdx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::dst); 2129 assert(DstIdx != -1); 2130 unsigned DstRCID = get(MI->getOpcode()).OpInfo[DstIdx].RegClass; 2131 switch(RI.getRegClass(DstRCID)->getSize()) { 2132 case 4: 2133 case 8: 2134 case 16: { 2135 unsigned NewOpcode = getVALUOp(*MI); 2136 unsigned RegOffset; 2137 unsigned ImmOffset; 2138 2139 if (MI->getOperand(2).isReg()) { 2140 RegOffset = MI->getOperand(2).getReg(); 2141 ImmOffset = 0; 2142 } else { 2143 assert(MI->getOperand(2).isImm()); 2144 // SMRD instructions take a dword offsets on SI and byte offset on VI 2145 // and MUBUF instructions always take a byte offset. 2146 ImmOffset = MI->getOperand(2).getImm(); 2147 if (MBB->getParent()->getSubtarget<AMDGPUSubtarget>().getGeneration() <= 2148 AMDGPUSubtarget::SEA_ISLANDS) 2149 ImmOffset <<= 2; 2150 RegOffset = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 2151 2152 if (isUInt<12>(ImmOffset)) { 2153 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), 2154 RegOffset) 2155 .addImm(0); 2156 } else { 2157 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), 2158 RegOffset) 2159 .addImm(ImmOffset); 2160 ImmOffset = 0; 2161 } 2162 } 2163 2164 unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass); 2165 unsigned DWord0 = RegOffset; 2166 unsigned DWord1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 2167 unsigned DWord2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 2168 unsigned DWord3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 2169 uint64_t RsrcDataFormat = getDefaultRsrcDataFormat(); 2170 2171 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), DWord1) 2172 .addImm(0); 2173 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), DWord2) 2174 .addImm(RsrcDataFormat & 0xFFFFFFFF); 2175 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::S_MOV_B32), DWord3) 2176 .addImm(RsrcDataFormat >> 32); 2177 BuildMI(*MBB, MI, MI->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), SRsrc) 2178 .addReg(DWord0) 2179 .addImm(AMDGPU::sub0) 2180 .addReg(DWord1) 2181 .addImm(AMDGPU::sub1) 2182 .addReg(DWord2) 2183 .addImm(AMDGPU::sub2) 2184 .addReg(DWord3) 2185 .addImm(AMDGPU::sub3); 2186 2187 const MCInstrDesc &NewInstDesc = get(NewOpcode); 2188 const TargetRegisterClass *NewDstRC 2189 = RI.getRegClass(NewInstDesc.OpInfo[0].RegClass); 2190 unsigned NewDstReg = MRI.createVirtualRegister(NewDstRC); 2191 unsigned DstReg = MI->getOperand(0).getReg(); 2192 MRI.replaceRegWith(DstReg, NewDstReg); 2193 2194 MachineInstr *NewInst = 2195 BuildMI(*MBB, MI, MI->getDebugLoc(), NewInstDesc, NewDstReg) 2196 .addOperand(MI->getOperand(1)) // sbase 2197 .addReg(SRsrc) 2198 .addImm(0) 2199 .addImm(ImmOffset) 2200 .addImm(0) // glc 2201 .addImm(0) // slc 2202 .addImm(0) // tfe 2203 .setMemRefs(MI->memoperands_begin(), MI->memoperands_end()); 2204 MI->eraseFromParent(); 2205 2206 legalizeOperands(NewInst); 2207 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 2208 break; 2209 } 2210 case 32: { 2211 MachineInstr *Lo, *Hi; 2212 splitSMRD(MI, &AMDGPU::SReg_128RegClass, AMDGPU::S_LOAD_DWORDX4_IMM, 2213 AMDGPU::S_LOAD_DWORDX4_SGPR, Lo, Hi); 2214 MI->eraseFromParent(); 2215 moveSMRDToVALU(Lo, MRI, Worklist); 2216 moveSMRDToVALU(Hi, MRI, Worklist); 2217 break; 2218 } 2219 2220 case 64: { 2221 MachineInstr *Lo, *Hi; 2222 splitSMRD(MI, &AMDGPU::SReg_256RegClass, AMDGPU::S_LOAD_DWORDX8_IMM, 2223 AMDGPU::S_LOAD_DWORDX8_SGPR, Lo, Hi); 2224 MI->eraseFromParent(); 2225 moveSMRDToVALU(Lo, MRI, Worklist); 2226 moveSMRDToVALU(Hi, MRI, Worklist); 2227 break; 2228 } 2229 } 2230 } 2231 2232 void SIInstrInfo::moveToVALU(MachineInstr &TopInst) const { 2233 SmallVector<MachineInstr *, 128> Worklist; 2234 Worklist.push_back(&TopInst); 2235 2236 while (!Worklist.empty()) { 2237 MachineInstr *Inst = Worklist.pop_back_val(); 2238 MachineBasicBlock *MBB = Inst->getParent(); 2239 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 2240 2241 unsigned Opcode = Inst->getOpcode(); 2242 unsigned NewOpcode = getVALUOp(*Inst); 2243 2244 // Handle some special cases 2245 switch (Opcode) { 2246 default: 2247 if (isSMRD(*Inst)) { 2248 moveSMRDToVALU(Inst, MRI, Worklist); 2249 continue; 2250 } 2251 break; 2252 case AMDGPU::S_AND_B64: 2253 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_AND_B32_e64); 2254 Inst->eraseFromParent(); 2255 continue; 2256 2257 case AMDGPU::S_OR_B64: 2258 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_OR_B32_e64); 2259 Inst->eraseFromParent(); 2260 continue; 2261 2262 case AMDGPU::S_XOR_B64: 2263 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_XOR_B32_e64); 2264 Inst->eraseFromParent(); 2265 continue; 2266 2267 case AMDGPU::S_NOT_B64: 2268 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::V_NOT_B32_e32); 2269 Inst->eraseFromParent(); 2270 continue; 2271 2272 case AMDGPU::S_BCNT1_I32_B64: 2273 splitScalar64BitBCNT(Worklist, Inst); 2274 Inst->eraseFromParent(); 2275 continue; 2276 2277 case AMDGPU::S_BFE_I64: { 2278 splitScalar64BitBFE(Worklist, Inst); 2279 Inst->eraseFromParent(); 2280 continue; 2281 } 2282 2283 case AMDGPU::S_LSHL_B32: 2284 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2285 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 2286 swapOperands(Inst); 2287 } 2288 break; 2289 case AMDGPU::S_ASHR_I32: 2290 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2291 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 2292 swapOperands(Inst); 2293 } 2294 break; 2295 case AMDGPU::S_LSHR_B32: 2296 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2297 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 2298 swapOperands(Inst); 2299 } 2300 break; 2301 case AMDGPU::S_LSHL_B64: 2302 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2303 NewOpcode = AMDGPU::V_LSHLREV_B64; 2304 swapOperands(Inst); 2305 } 2306 break; 2307 case AMDGPU::S_ASHR_I64: 2308 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2309 NewOpcode = AMDGPU::V_ASHRREV_I64; 2310 swapOperands(Inst); 2311 } 2312 break; 2313 case AMDGPU::S_LSHR_B64: 2314 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 2315 NewOpcode = AMDGPU::V_LSHRREV_B64; 2316 swapOperands(Inst); 2317 } 2318 break; 2319 2320 case AMDGPU::S_BFE_U64: 2321 case AMDGPU::S_BFM_B64: 2322 llvm_unreachable("Moving this op to VALU not implemented"); 2323 } 2324 2325 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 2326 // We cannot move this instruction to the VALU, so we should try to 2327 // legalize its operands instead. 2328 legalizeOperands(Inst); 2329 continue; 2330 } 2331 2332 // Use the new VALU Opcode. 2333 const MCInstrDesc &NewDesc = get(NewOpcode); 2334 Inst->setDesc(NewDesc); 2335 2336 // Remove any references to SCC. Vector instructions can't read from it, and 2337 // We're just about to add the implicit use / defs of VCC, and we don't want 2338 // both. 2339 for (unsigned i = Inst->getNumOperands() - 1; i > 0; --i) { 2340 MachineOperand &Op = Inst->getOperand(i); 2341 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) 2342 Inst->RemoveOperand(i); 2343 } 2344 2345 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 2346 // We are converting these to a BFE, so we need to add the missing 2347 // operands for the size and offset. 2348 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 2349 Inst->addOperand(MachineOperand::CreateImm(0)); 2350 Inst->addOperand(MachineOperand::CreateImm(Size)); 2351 2352 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 2353 // The VALU version adds the second operand to the result, so insert an 2354 // extra 0 operand. 2355 Inst->addOperand(MachineOperand::CreateImm(0)); 2356 } 2357 2358 Inst->addImplicitDefUseOperands(*Inst->getParent()->getParent()); 2359 2360 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 2361 const MachineOperand &OffsetWidthOp = Inst->getOperand(2); 2362 // If we need to move this to VGPRs, we need to unpack the second operand 2363 // back into the 2 separate ones for bit offset and width. 2364 assert(OffsetWidthOp.isImm() && 2365 "Scalar BFE is only implemented for constant width and offset"); 2366 uint32_t Imm = OffsetWidthOp.getImm(); 2367 2368 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 2369 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 2370 Inst->RemoveOperand(2); // Remove old immediate. 2371 Inst->addOperand(MachineOperand::CreateImm(Offset)); 2372 Inst->addOperand(MachineOperand::CreateImm(BitWidth)); 2373 } 2374 2375 // Update the destination register class. 2376 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(*Inst); 2377 if (!NewDstRC) 2378 continue; 2379 2380 unsigned DstReg = Inst->getOperand(0).getReg(); 2381 unsigned NewDstReg = MRI.createVirtualRegister(NewDstRC); 2382 MRI.replaceRegWith(DstReg, NewDstReg); 2383 2384 // Legalize the operands 2385 legalizeOperands(Inst); 2386 2387 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 2388 } 2389 } 2390 2391 //===----------------------------------------------------------------------===// 2392 // Indirect addressing callbacks 2393 //===----------------------------------------------------------------------===// 2394 2395 unsigned SIInstrInfo::calculateIndirectAddress(unsigned RegIndex, 2396 unsigned Channel) const { 2397 assert(Channel == 0); 2398 return RegIndex; 2399 } 2400 2401 const TargetRegisterClass *SIInstrInfo::getIndirectAddrRegClass() const { 2402 return &AMDGPU::VGPR_32RegClass; 2403 } 2404 2405 void SIInstrInfo::splitScalar64BitUnaryOp( 2406 SmallVectorImpl<MachineInstr *> &Worklist, 2407 MachineInstr *Inst, 2408 unsigned Opcode) const { 2409 MachineBasicBlock &MBB = *Inst->getParent(); 2410 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2411 2412 MachineOperand &Dest = Inst->getOperand(0); 2413 MachineOperand &Src0 = Inst->getOperand(1); 2414 DebugLoc DL = Inst->getDebugLoc(); 2415 2416 MachineBasicBlock::iterator MII = Inst; 2417 2418 const MCInstrDesc &InstDesc = get(Opcode); 2419 const TargetRegisterClass *Src0RC = Src0.isReg() ? 2420 MRI.getRegClass(Src0.getReg()) : 2421 &AMDGPU::SGPR_32RegClass; 2422 2423 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 2424 2425 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 2426 AMDGPU::sub0, Src0SubRC); 2427 2428 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 2429 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 2430 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 2431 2432 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 2433 BuildMI(MBB, MII, DL, InstDesc, DestSub0) 2434 .addOperand(SrcReg0Sub0); 2435 2436 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 2437 AMDGPU::sub1, Src0SubRC); 2438 2439 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 2440 BuildMI(MBB, MII, DL, InstDesc, DestSub1) 2441 .addOperand(SrcReg0Sub1); 2442 2443 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC); 2444 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 2445 .addReg(DestSub0) 2446 .addImm(AMDGPU::sub0) 2447 .addReg(DestSub1) 2448 .addImm(AMDGPU::sub1); 2449 2450 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 2451 2452 // We don't need to legalizeOperands here because for a single operand, src0 2453 // will support any kind of input. 2454 2455 // Move all users of this moved value. 2456 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 2457 } 2458 2459 void SIInstrInfo::splitScalar64BitBinaryOp( 2460 SmallVectorImpl<MachineInstr *> &Worklist, 2461 MachineInstr *Inst, 2462 unsigned Opcode) const { 2463 MachineBasicBlock &MBB = *Inst->getParent(); 2464 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2465 2466 MachineOperand &Dest = Inst->getOperand(0); 2467 MachineOperand &Src0 = Inst->getOperand(1); 2468 MachineOperand &Src1 = Inst->getOperand(2); 2469 DebugLoc DL = Inst->getDebugLoc(); 2470 2471 MachineBasicBlock::iterator MII = Inst; 2472 2473 const MCInstrDesc &InstDesc = get(Opcode); 2474 const TargetRegisterClass *Src0RC = Src0.isReg() ? 2475 MRI.getRegClass(Src0.getReg()) : 2476 &AMDGPU::SGPR_32RegClass; 2477 2478 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 2479 const TargetRegisterClass *Src1RC = Src1.isReg() ? 2480 MRI.getRegClass(Src1.getReg()) : 2481 &AMDGPU::SGPR_32RegClass; 2482 2483 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 2484 2485 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 2486 AMDGPU::sub0, Src0SubRC); 2487 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 2488 AMDGPU::sub0, Src1SubRC); 2489 2490 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 2491 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 2492 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 2493 2494 unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 2495 MachineInstr *LoHalf = BuildMI(MBB, MII, DL, InstDesc, DestSub0) 2496 .addOperand(SrcReg0Sub0) 2497 .addOperand(SrcReg1Sub0); 2498 2499 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 2500 AMDGPU::sub1, Src0SubRC); 2501 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 2502 AMDGPU::sub1, Src1SubRC); 2503 2504 unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 2505 MachineInstr *HiHalf = BuildMI(MBB, MII, DL, InstDesc, DestSub1) 2506 .addOperand(SrcReg0Sub1) 2507 .addOperand(SrcReg1Sub1); 2508 2509 unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC); 2510 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 2511 .addReg(DestSub0) 2512 .addImm(AMDGPU::sub0) 2513 .addReg(DestSub1) 2514 .addImm(AMDGPU::sub1); 2515 2516 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 2517 2518 // Try to legalize the operands in case we need to swap the order to keep it 2519 // valid. 2520 legalizeOperands(LoHalf); 2521 legalizeOperands(HiHalf); 2522 2523 // Move all users of this moved vlaue. 2524 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 2525 } 2526 2527 void SIInstrInfo::splitScalar64BitBCNT(SmallVectorImpl<MachineInstr *> &Worklist, 2528 MachineInstr *Inst) const { 2529 MachineBasicBlock &MBB = *Inst->getParent(); 2530 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2531 2532 MachineBasicBlock::iterator MII = Inst; 2533 DebugLoc DL = Inst->getDebugLoc(); 2534 2535 MachineOperand &Dest = Inst->getOperand(0); 2536 MachineOperand &Src = Inst->getOperand(1); 2537 2538 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 2539 const TargetRegisterClass *SrcRC = Src.isReg() ? 2540 MRI.getRegClass(Src.getReg()) : 2541 &AMDGPU::SGPR_32RegClass; 2542 2543 unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2544 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2545 2546 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); 2547 2548 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 2549 AMDGPU::sub0, SrcSubRC); 2550 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 2551 AMDGPU::sub1, SrcSubRC); 2552 2553 BuildMI(MBB, MII, DL, InstDesc, MidReg) 2554 .addOperand(SrcRegSub0) 2555 .addImm(0); 2556 2557 BuildMI(MBB, MII, DL, InstDesc, ResultReg) 2558 .addOperand(SrcRegSub1) 2559 .addReg(MidReg); 2560 2561 MRI.replaceRegWith(Dest.getReg(), ResultReg); 2562 2563 // We don't need to legalize operands here. src0 for etiher instruction can be 2564 // an SGPR, and the second input is unused or determined here. 2565 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 2566 } 2567 2568 void SIInstrInfo::splitScalar64BitBFE(SmallVectorImpl<MachineInstr *> &Worklist, 2569 MachineInstr *Inst) const { 2570 MachineBasicBlock &MBB = *Inst->getParent(); 2571 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2572 MachineBasicBlock::iterator MII = Inst; 2573 DebugLoc DL = Inst->getDebugLoc(); 2574 2575 MachineOperand &Dest = Inst->getOperand(0); 2576 uint32_t Imm = Inst->getOperand(2).getImm(); 2577 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 2578 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 2579 2580 (void) Offset; 2581 2582 // Only sext_inreg cases handled. 2583 assert(Inst->getOpcode() == AMDGPU::S_BFE_I64 && 2584 BitWidth <= 32 && 2585 Offset == 0 && 2586 "Not implemented"); 2587 2588 if (BitWidth < 32) { 2589 unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2590 unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2591 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 2592 2593 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo) 2594 .addReg(Inst->getOperand(1).getReg(), 0, AMDGPU::sub0) 2595 .addImm(0) 2596 .addImm(BitWidth); 2597 2598 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 2599 .addImm(31) 2600 .addReg(MidRegLo); 2601 2602 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 2603 .addReg(MidRegLo) 2604 .addImm(AMDGPU::sub0) 2605 .addReg(MidRegHi) 2606 .addImm(AMDGPU::sub1); 2607 2608 MRI.replaceRegWith(Dest.getReg(), ResultReg); 2609 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 2610 return; 2611 } 2612 2613 MachineOperand &Src = Inst->getOperand(1); 2614 unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2615 unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 2616 2617 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 2618 .addImm(31) 2619 .addReg(Src.getReg(), 0, AMDGPU::sub0); 2620 2621 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 2622 .addReg(Src.getReg(), 0, AMDGPU::sub0) 2623 .addImm(AMDGPU::sub0) 2624 .addReg(TmpReg) 2625 .addImm(AMDGPU::sub1); 2626 2627 MRI.replaceRegWith(Dest.getReg(), ResultReg); 2628 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 2629 } 2630 2631 void SIInstrInfo::addUsersToMoveToVALUWorklist( 2632 unsigned DstReg, 2633 MachineRegisterInfo &MRI, 2634 SmallVectorImpl<MachineInstr *> &Worklist) const { 2635 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 2636 E = MRI.use_end(); I != E; ++I) { 2637 MachineInstr &UseMI = *I->getParent(); 2638 if (!canReadVGPR(UseMI, I.getOperandNo())) { 2639 Worklist.push_back(&UseMI); 2640 } 2641 } 2642 } 2643 2644 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 2645 const MachineInstr &Inst) const { 2646 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 2647 2648 switch (Inst.getOpcode()) { 2649 // For target instructions, getOpRegClass just returns the virtual register 2650 // class associated with the operand, so we need to find an equivalent VGPR 2651 // register class in order to move the instruction to the VALU. 2652 case AMDGPU::COPY: 2653 case AMDGPU::PHI: 2654 case AMDGPU::REG_SEQUENCE: 2655 case AMDGPU::INSERT_SUBREG: 2656 if (RI.hasVGPRs(NewDstRC)) 2657 return nullptr; 2658 2659 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 2660 if (!NewDstRC) 2661 return nullptr; 2662 return NewDstRC; 2663 default: 2664 return NewDstRC; 2665 } 2666 } 2667 2668 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr *MI, 2669 int OpIndices[3]) const { 2670 const MCInstrDesc &Desc = MI->getDesc(); 2671 2672 // Find the one SGPR operand we are allowed to use. 2673 // 2674 // First we need to consider the instruction's operand requirements before 2675 // legalizing. Some operands are required to be SGPRs, such as implicit uses 2676 // of VCC, but we are still bound by the constant bus requirement to only use 2677 // one. 2678 // 2679 // If the operand's class is an SGPR, we can never move it. 2680 2681 unsigned SGPRReg = findImplicitSGPRRead(*MI); 2682 if (SGPRReg != AMDGPU::NoRegister) 2683 return SGPRReg; 2684 2685 unsigned UsedSGPRs[3] = { AMDGPU::NoRegister }; 2686 const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); 2687 2688 for (unsigned i = 0; i < 3; ++i) { 2689 int Idx = OpIndices[i]; 2690 if (Idx == -1) 2691 break; 2692 2693 const MachineOperand &MO = MI->getOperand(Idx); 2694 if (RI.isSGPRClassID(Desc.OpInfo[Idx].RegClass)) 2695 SGPRReg = MO.getReg(); 2696 2697 if (MO.isReg() && RI.isSGPRClass(MRI.getRegClass(MO.getReg()))) 2698 UsedSGPRs[i] = MO.getReg(); 2699 } 2700 2701 if (SGPRReg != AMDGPU::NoRegister) 2702 return SGPRReg; 2703 2704 // We don't have a required SGPR operand, so we have a bit more freedom in 2705 // selecting operands to move. 2706 2707 // Try to select the most used SGPR. If an SGPR is equal to one of the 2708 // others, we choose that. 2709 // 2710 // e.g. 2711 // V_FMA_F32 v0, s0, s0, s0 -> No moves 2712 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 2713 2714 if (UsedSGPRs[0] != AMDGPU::NoRegister) { 2715 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 2716 SGPRReg = UsedSGPRs[0]; 2717 } 2718 2719 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { 2720 if (UsedSGPRs[1] == UsedSGPRs[2]) 2721 SGPRReg = UsedSGPRs[1]; 2722 } 2723 2724 return SGPRReg; 2725 } 2726 2727 MachineInstrBuilder SIInstrInfo::buildIndirectWrite( 2728 MachineBasicBlock *MBB, 2729 MachineBasicBlock::iterator I, 2730 unsigned ValueReg, 2731 unsigned Address, unsigned OffsetReg) const { 2732 const DebugLoc &DL = MBB->findDebugLoc(I); 2733 unsigned IndirectBaseReg = AMDGPU::VGPR_32RegClass.getRegister( 2734 getIndirectIndexBegin(*MBB->getParent())); 2735 2736 return BuildMI(*MBB, I, DL, get(AMDGPU::SI_INDIRECT_DST_V1)) 2737 .addReg(IndirectBaseReg, RegState::Define) 2738 .addOperand(I->getOperand(0)) 2739 .addReg(IndirectBaseReg) 2740 .addReg(OffsetReg) 2741 .addImm(0) 2742 .addReg(ValueReg); 2743 } 2744 2745 MachineInstrBuilder SIInstrInfo::buildIndirectRead( 2746 MachineBasicBlock *MBB, 2747 MachineBasicBlock::iterator I, 2748 unsigned ValueReg, 2749 unsigned Address, unsigned OffsetReg) const { 2750 const DebugLoc &DL = MBB->findDebugLoc(I); 2751 unsigned IndirectBaseReg = AMDGPU::VGPR_32RegClass.getRegister( 2752 getIndirectIndexBegin(*MBB->getParent())); 2753 2754 return BuildMI(*MBB, I, DL, get(AMDGPU::SI_INDIRECT_SRC_V1)) 2755 .addOperand(I->getOperand(0)) 2756 .addOperand(I->getOperand(1)) 2757 .addReg(IndirectBaseReg) 2758 .addReg(OffsetReg) 2759 .addImm(0); 2760 2761 } 2762 2763 void SIInstrInfo::reserveIndirectRegisters(BitVector &Reserved, 2764 const MachineFunction &MF) const { 2765 int End = getIndirectIndexEnd(MF); 2766 int Begin = getIndirectIndexBegin(MF); 2767 2768 if (End == -1) 2769 return; 2770 2771 2772 for (int Index = Begin; Index <= End; ++Index) 2773 Reserved.set(AMDGPU::VGPR_32RegClass.getRegister(Index)); 2774 2775 for (int Index = std::max(0, Begin - 1); Index <= End; ++Index) 2776 Reserved.set(AMDGPU::VReg_64RegClass.getRegister(Index)); 2777 2778 for (int Index = std::max(0, Begin - 2); Index <= End; ++Index) 2779 Reserved.set(AMDGPU::VReg_96RegClass.getRegister(Index)); 2780 2781 for (int Index = std::max(0, Begin - 3); Index <= End; ++Index) 2782 Reserved.set(AMDGPU::VReg_128RegClass.getRegister(Index)); 2783 2784 for (int Index = std::max(0, Begin - 7); Index <= End; ++Index) 2785 Reserved.set(AMDGPU::VReg_256RegClass.getRegister(Index)); 2786 2787 for (int Index = std::max(0, Begin - 15); Index <= End; ++Index) 2788 Reserved.set(AMDGPU::VReg_512RegClass.getRegister(Index)); 2789 } 2790 2791 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 2792 unsigned OperandName) const { 2793 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 2794 if (Idx == -1) 2795 return nullptr; 2796 2797 return &MI.getOperand(Idx); 2798 } 2799 2800 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 2801 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 2802 if (ST.isAmdHsaOS()) { 2803 RsrcDataFormat |= (1ULL << 56); 2804 2805 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 2806 // Set MTYPE = 2 2807 RsrcDataFormat |= (2ULL << 59); 2808 } 2809 2810 return RsrcDataFormat; 2811 } 2812 2813 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 2814 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 2815 AMDGPU::RSRC_TID_ENABLE | 2816 0xffffffff; // Size; 2817 2818 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 2819 // Clear them unless we want a huge stride. 2820 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 2821 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 2822 2823 return Rsrc23; 2824 } 2825