1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/CodeGen/LiveVariables.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/RegisterScavenging.h" 25 #include "llvm/CodeGen/ScheduleDAG.h" 26 #include "llvm/IR/DiagnosticInfo.h" 27 #include "llvm/IR/IntrinsicsAMDGPU.h" 28 #include "llvm/MC/MCContext.h" 29 #include "llvm/Support/CommandLine.h" 30 #include "llvm/Target/TargetMachine.h" 31 32 using namespace llvm; 33 34 #define DEBUG_TYPE "si-instr-info" 35 36 #define GET_INSTRINFO_CTOR_DTOR 37 #include "AMDGPUGenInstrInfo.inc" 38 39 namespace llvm { 40 41 class AAResults; 42 43 namespace AMDGPU { 44 #define GET_D16ImageDimIntrinsics_IMPL 45 #define GET_ImageDimIntrinsicTable_IMPL 46 #define GET_RsrcIntrinsics_IMPL 47 #include "AMDGPUGenSearchableTables.inc" 48 } 49 } 50 51 52 // Must be at least 4 to be able to branch over minimum unconditional branch 53 // code. This is only for making it possible to write reasonably small tests for 54 // long branches. 55 static cl::opt<unsigned> 56 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 57 cl::desc("Restrict range of branch instructions (DEBUG)")); 58 59 static cl::opt<bool> Fix16BitCopies( 60 "amdgpu-fix-16-bit-physreg-copies", 61 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 62 cl::init(true), 63 cl::ReallyHidden); 64 65 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 66 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 67 RI(ST), ST(ST) { 68 SchedModel.init(&ST); 69 } 70 71 //===----------------------------------------------------------------------===// 72 // TargetInstrInfo callbacks 73 //===----------------------------------------------------------------------===// 74 75 static unsigned getNumOperandsNoGlue(SDNode *Node) { 76 unsigned N = Node->getNumOperands(); 77 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 78 --N; 79 return N; 80 } 81 82 /// Returns true if both nodes have the same value for the given 83 /// operand \p Op, or if both nodes do not have this operand. 84 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 85 unsigned Opc0 = N0->getMachineOpcode(); 86 unsigned Opc1 = N1->getMachineOpcode(); 87 88 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 89 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 90 91 if (Op0Idx == -1 && Op1Idx == -1) 92 return true; 93 94 95 if ((Op0Idx == -1 && Op1Idx != -1) || 96 (Op1Idx == -1 && Op0Idx != -1)) 97 return false; 98 99 // getNamedOperandIdx returns the index for the MachineInstr's operands, 100 // which includes the result as the first operand. We are indexing into the 101 // MachineSDNode's operands, so we need to skip the result operand to get 102 // the real index. 103 --Op0Idx; 104 --Op1Idx; 105 106 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 107 } 108 109 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, 110 AAResults *AA) const { 111 if (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isSDWA(MI)) { 112 // Normally VALU use of exec would block the rematerialization, but that 113 // is OK in this case to have an implicit exec read as all VALU do. 114 // We really want all of the generic logic for this except for this. 115 116 // Another potential implicit use is mode register. The core logic of 117 // the RA will not attempt rematerialization if mode is set anywhere 118 // in the function, otherwise it is safe since mode is not changed. 119 return !MI.hasImplicitDef() && 120 MI.getNumImplicitOperands() == MI.getDesc().getNumImplicitUses() && 121 !MI.mayRaiseFPException(); 122 } 123 124 return false; 125 } 126 127 bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const { 128 // Any implicit use of exec by VALU is not a real register read. 129 return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() && 130 isVALU(*MO.getParent()); 131 } 132 133 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 134 int64_t &Offset0, 135 int64_t &Offset1) const { 136 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 137 return false; 138 139 unsigned Opc0 = Load0->getMachineOpcode(); 140 unsigned Opc1 = Load1->getMachineOpcode(); 141 142 // Make sure both are actually loads. 143 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 144 return false; 145 146 if (isDS(Opc0) && isDS(Opc1)) { 147 148 // FIXME: Handle this case: 149 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 150 return false; 151 152 // Check base reg. 153 if (Load0->getOperand(0) != Load1->getOperand(0)) 154 return false; 155 156 // Skip read2 / write2 variants for simplicity. 157 // TODO: We should report true if the used offsets are adjacent (excluded 158 // st64 versions). 159 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 160 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 161 if (Offset0Idx == -1 || Offset1Idx == -1) 162 return false; 163 164 // XXX - be careful of datalesss loads 165 // getNamedOperandIdx returns the index for MachineInstrs. Since they 166 // include the output in the operand list, but SDNodes don't, we need to 167 // subtract the index by one. 168 Offset0Idx -= get(Opc0).NumDefs; 169 Offset1Idx -= get(Opc1).NumDefs; 170 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue(); 171 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue(); 172 return true; 173 } 174 175 if (isSMRD(Opc0) && isSMRD(Opc1)) { 176 // Skip time and cache invalidation instructions. 177 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || 178 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) 179 return false; 180 181 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1)); 182 183 // Check base reg. 184 if (Load0->getOperand(0) != Load1->getOperand(0)) 185 return false; 186 187 const ConstantSDNode *Load0Offset = 188 dyn_cast<ConstantSDNode>(Load0->getOperand(1)); 189 const ConstantSDNode *Load1Offset = 190 dyn_cast<ConstantSDNode>(Load1->getOperand(1)); 191 192 if (!Load0Offset || !Load1Offset) 193 return false; 194 195 Offset0 = Load0Offset->getZExtValue(); 196 Offset1 = Load1Offset->getZExtValue(); 197 return true; 198 } 199 200 // MUBUF and MTBUF can access the same addresses. 201 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 202 203 // MUBUF and MTBUF have vaddr at different indices. 204 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 205 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 206 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 207 return false; 208 209 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 210 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 211 212 if (OffIdx0 == -1 || OffIdx1 == -1) 213 return false; 214 215 // getNamedOperandIdx returns the index for MachineInstrs. Since they 216 // include the output in the operand list, but SDNodes don't, we need to 217 // subtract the index by one. 218 OffIdx0 -= get(Opc0).NumDefs; 219 OffIdx1 -= get(Opc1).NumDefs; 220 221 SDValue Off0 = Load0->getOperand(OffIdx0); 222 SDValue Off1 = Load1->getOperand(OffIdx1); 223 224 // The offset might be a FrameIndexSDNode. 225 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 226 return false; 227 228 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue(); 229 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue(); 230 return true; 231 } 232 233 return false; 234 } 235 236 static bool isStride64(unsigned Opc) { 237 switch (Opc) { 238 case AMDGPU::DS_READ2ST64_B32: 239 case AMDGPU::DS_READ2ST64_B64: 240 case AMDGPU::DS_WRITE2ST64_B32: 241 case AMDGPU::DS_WRITE2ST64_B64: 242 return true; 243 default: 244 return false; 245 } 246 } 247 248 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 249 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 250 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 251 const TargetRegisterInfo *TRI) const { 252 if (!LdSt.mayLoadOrStore()) 253 return false; 254 255 unsigned Opc = LdSt.getOpcode(); 256 OffsetIsScalable = false; 257 const MachineOperand *BaseOp, *OffsetOp; 258 int DataOpIdx; 259 260 if (isDS(LdSt)) { 261 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 262 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 263 if (OffsetOp) { 264 // Normal, single offset LDS instruction. 265 if (!BaseOp) { 266 // DS_CONSUME/DS_APPEND use M0 for the base address. 267 // TODO: find the implicit use operand for M0 and use that as BaseOp? 268 return false; 269 } 270 BaseOps.push_back(BaseOp); 271 Offset = OffsetOp->getImm(); 272 // Get appropriate operand, and compute width accordingly. 273 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 274 if (DataOpIdx == -1) 275 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 276 Width = getOpSize(LdSt, DataOpIdx); 277 } else { 278 // The 2 offset instructions use offset0 and offset1 instead. We can treat 279 // these as a load with a single offset if the 2 offsets are consecutive. 280 // We will use this for some partially aligned loads. 281 const MachineOperand *Offset0Op = 282 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 283 const MachineOperand *Offset1Op = 284 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 285 286 unsigned Offset0 = Offset0Op->getImm(); 287 unsigned Offset1 = Offset1Op->getImm(); 288 if (Offset0 + 1 != Offset1) 289 return false; 290 291 // Each of these offsets is in element sized units, so we need to convert 292 // to bytes of the individual reads. 293 294 unsigned EltSize; 295 if (LdSt.mayLoad()) 296 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 297 else { 298 assert(LdSt.mayStore()); 299 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 300 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 301 } 302 303 if (isStride64(Opc)) 304 EltSize *= 64; 305 306 BaseOps.push_back(BaseOp); 307 Offset = EltSize * Offset0; 308 // Get appropriate operand(s), and compute width accordingly. 309 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 310 if (DataOpIdx == -1) { 311 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 312 Width = getOpSize(LdSt, DataOpIdx); 313 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 314 Width += getOpSize(LdSt, DataOpIdx); 315 } else { 316 Width = getOpSize(LdSt, DataOpIdx); 317 } 318 } 319 return true; 320 } 321 322 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 323 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 324 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL 325 return false; 326 BaseOps.push_back(RSrc); 327 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 328 if (BaseOp && !BaseOp->isFI()) 329 BaseOps.push_back(BaseOp); 330 const MachineOperand *OffsetImm = 331 getNamedOperand(LdSt, AMDGPU::OpName::offset); 332 Offset = OffsetImm->getImm(); 333 const MachineOperand *SOffset = 334 getNamedOperand(LdSt, AMDGPU::OpName::soffset); 335 if (SOffset) { 336 if (SOffset->isReg()) 337 BaseOps.push_back(SOffset); 338 else 339 Offset += SOffset->getImm(); 340 } 341 // Get appropriate operand, and compute width accordingly. 342 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 343 if (DataOpIdx == -1) 344 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 345 Width = getOpSize(LdSt, DataOpIdx); 346 return true; 347 } 348 349 if (isMIMG(LdSt)) { 350 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 351 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 352 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 353 if (VAddr0Idx >= 0) { 354 // GFX10 possible NSA encoding. 355 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 356 BaseOps.push_back(&LdSt.getOperand(I)); 357 } else { 358 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 359 } 360 Offset = 0; 361 // Get appropriate operand, and compute width accordingly. 362 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 363 Width = getOpSize(LdSt, DataOpIdx); 364 return true; 365 } 366 367 if (isSMRD(LdSt)) { 368 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 369 if (!BaseOp) // e.g. S_MEMTIME 370 return false; 371 BaseOps.push_back(BaseOp); 372 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 373 Offset = OffsetOp ? OffsetOp->getImm() : 0; 374 // Get appropriate operand, and compute width accordingly. 375 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 376 Width = getOpSize(LdSt, DataOpIdx); 377 return true; 378 } 379 380 if (isFLAT(LdSt)) { 381 // Instructions have either vaddr or saddr or both or none. 382 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 383 if (BaseOp) 384 BaseOps.push_back(BaseOp); 385 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 386 if (BaseOp) 387 BaseOps.push_back(BaseOp); 388 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 389 // Get appropriate operand, and compute width accordingly. 390 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 391 if (DataOpIdx == -1) 392 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 393 Width = getOpSize(LdSt, DataOpIdx); 394 return true; 395 } 396 397 return false; 398 } 399 400 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 401 ArrayRef<const MachineOperand *> BaseOps1, 402 const MachineInstr &MI2, 403 ArrayRef<const MachineOperand *> BaseOps2) { 404 // Only examine the first "base" operand of each instruction, on the 405 // assumption that it represents the real base address of the memory access. 406 // Other operands are typically offsets or indices from this base address. 407 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 408 return true; 409 410 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 411 return false; 412 413 auto MO1 = *MI1.memoperands_begin(); 414 auto MO2 = *MI2.memoperands_begin(); 415 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 416 return false; 417 418 auto Base1 = MO1->getValue(); 419 auto Base2 = MO2->getValue(); 420 if (!Base1 || !Base2) 421 return false; 422 Base1 = getUnderlyingObject(Base1); 423 Base2 = getUnderlyingObject(Base2); 424 425 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 426 return false; 427 428 return Base1 == Base2; 429 } 430 431 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 432 ArrayRef<const MachineOperand *> BaseOps2, 433 unsigned NumLoads, 434 unsigned NumBytes) const { 435 // If the mem ops (to be clustered) do not have the same base ptr, then they 436 // should not be clustered 437 if (!BaseOps1.empty() && !BaseOps2.empty()) { 438 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 439 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 440 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 441 return false; 442 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 443 // If only one base op is empty, they do not have the same base ptr 444 return false; 445 } 446 447 // In order to avoid regester pressure, on an average, the number of DWORDS 448 // loaded together by all clustered mem ops should not exceed 8. This is an 449 // empirical value based on certain observations and performance related 450 // experiments. 451 // The good thing about this heuristic is - it avoids clustering of too many 452 // sub-word loads, and also avoids clustering of wide loads. Below is the 453 // brief summary of how the heuristic behaves for various `LoadSize`. 454 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 455 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 456 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 457 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 458 // (5) LoadSize >= 17: do not cluster 459 const unsigned LoadSize = NumBytes / NumLoads; 460 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads; 461 return NumDWORDs <= 8; 462 } 463 464 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 465 // the first 16 loads will be interleaved with the stores, and the next 16 will 466 // be clustered as expected. It should really split into 2 16 store batches. 467 // 468 // Loads are clustered until this returns false, rather than trying to schedule 469 // groups of stores. This also means we have to deal with saying different 470 // address space loads should be clustered, and ones which might cause bank 471 // conflicts. 472 // 473 // This might be deprecated so it might not be worth that much effort to fix. 474 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 475 int64_t Offset0, int64_t Offset1, 476 unsigned NumLoads) const { 477 assert(Offset1 > Offset0 && 478 "Second offset should be larger than first offset!"); 479 // If we have less than 16 loads in a row, and the offsets are within 64 480 // bytes, then schedule together. 481 482 // A cacheline is 64 bytes (for global memory). 483 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 484 } 485 486 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 487 MachineBasicBlock::iterator MI, 488 const DebugLoc &DL, MCRegister DestReg, 489 MCRegister SrcReg, bool KillSrc, 490 const char *Msg = "illegal SGPR to VGPR copy") { 491 MachineFunction *MF = MBB.getParent(); 492 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 493 LLVMContext &C = MF->getFunction().getContext(); 494 C.diagnose(IllegalCopy); 495 496 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 497 .addReg(SrcReg, getKillRegState(KillSrc)); 498 } 499 500 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR. It is not possible 501 /// to directly copy, so an intermediate VGPR needs to be used. 502 static void indirectCopyToAGPR(const SIInstrInfo &TII, 503 MachineBasicBlock &MBB, 504 MachineBasicBlock::iterator MI, 505 const DebugLoc &DL, MCRegister DestReg, 506 MCRegister SrcReg, bool KillSrc, 507 RegScavenger &RS, 508 Register ImpDefSuperReg = Register(), 509 Register ImpUseSuperReg = Register()) { 510 const SIRegisterInfo &RI = TII.getRegisterInfo(); 511 512 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) || 513 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 514 515 // First try to find defining accvgpr_write to avoid temporary registers. 516 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 517 --Def; 518 if (!Def->definesRegister(SrcReg, &RI)) 519 continue; 520 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 521 break; 522 523 MachineOperand &DefOp = Def->getOperand(1); 524 assert(DefOp.isReg() || DefOp.isImm()); 525 526 if (DefOp.isReg()) { 527 // Check that register source operand if not clobbered before MI. 528 // Immediate operands are always safe to propagate. 529 bool SafeToPropagate = true; 530 for (auto I = Def; I != MI && SafeToPropagate; ++I) 531 if (I->modifiesRegister(DefOp.getReg(), &RI)) 532 SafeToPropagate = false; 533 534 if (!SafeToPropagate) 535 break; 536 537 DefOp.setIsKill(false); 538 } 539 540 MachineInstrBuilder Builder = 541 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 542 .add(DefOp); 543 if (ImpDefSuperReg) 544 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 545 546 if (ImpUseSuperReg) { 547 Builder.addReg(ImpUseSuperReg, 548 getKillRegState(KillSrc) | RegState::Implicit); 549 } 550 551 return; 552 } 553 554 RS.enterBasicBlock(MBB); 555 RS.forward(MI); 556 557 // Ideally we want to have three registers for a long reg_sequence copy 558 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 559 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 560 *MBB.getParent()); 561 562 // Registers in the sequence are allocated contiguously so we can just 563 // use register number to pick one of three round-robin temps. 564 unsigned RegNo = DestReg % 3; 565 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 566 if (!Tmp) 567 report_fatal_error("Cannot scavenge VGPR to copy to AGPR"); 568 RS.setRegUsed(Tmp); 569 570 if (!TII.getSubtarget().hasGFX90AInsts()) { 571 // Only loop through if there are any free registers left, otherwise 572 // scavenger may report a fatal error without emergency spill slot 573 // or spill with the slot. 574 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { 575 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 576 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 577 break; 578 Tmp = Tmp2; 579 RS.setRegUsed(Tmp); 580 } 581 } 582 583 // Insert copy to temporary VGPR. 584 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 585 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 586 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 587 } else { 588 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 589 } 590 591 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 592 .addReg(SrcReg, getKillRegState(KillSrc)); 593 if (ImpUseSuperReg) { 594 UseBuilder.addReg(ImpUseSuperReg, 595 getKillRegState(KillSrc) | RegState::Implicit); 596 } 597 598 MachineInstrBuilder DefBuilder 599 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 600 .addReg(Tmp, RegState::Kill); 601 602 if (ImpDefSuperReg) 603 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 604 } 605 606 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 607 MachineBasicBlock::iterator MI, const DebugLoc &DL, 608 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 609 const TargetRegisterClass *RC, bool Forward) { 610 const SIRegisterInfo &RI = TII.getRegisterInfo(); 611 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 612 MachineBasicBlock::iterator I = MI; 613 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 614 615 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 616 int16_t SubIdx = BaseIndices[Idx]; 617 Register Reg = RI.getSubReg(DestReg, SubIdx); 618 unsigned Opcode = AMDGPU::S_MOV_B32; 619 620 // Is SGPR aligned? If so try to combine with next. 621 Register Src = RI.getSubReg(SrcReg, SubIdx); 622 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; 623 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; 624 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 625 // Can use SGPR64 copy 626 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 627 SubIdx = RI.getSubRegFromChannel(Channel, 2); 628 Opcode = AMDGPU::S_MOV_B64; 629 Idx++; 630 } 631 632 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx)) 633 .addReg(RI.getSubReg(SrcReg, SubIdx)) 634 .addReg(SrcReg, RegState::Implicit); 635 636 if (!FirstMI) 637 FirstMI = LastMI; 638 639 if (!Forward) 640 I--; 641 } 642 643 assert(FirstMI && LastMI); 644 if (!Forward) 645 std::swap(FirstMI, LastMI); 646 647 FirstMI->addOperand( 648 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 649 650 if (KillSrc) 651 LastMI->addRegisterKilled(SrcReg, &RI); 652 } 653 654 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 655 MachineBasicBlock::iterator MI, 656 const DebugLoc &DL, MCRegister DestReg, 657 MCRegister SrcReg, bool KillSrc) const { 658 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg); 659 660 // FIXME: This is hack to resolve copies between 16 bit and 32 bit 661 // registers until all patterns are fixed. 662 if (Fix16BitCopies && 663 ((RI.getRegSizeInBits(*RC) == 16) ^ 664 (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) { 665 MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg; 666 MCRegister Super = RI.get32BitRegister(RegToFix); 667 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); 668 RegToFix = Super; 669 670 if (DestReg == SrcReg) { 671 // Insert empty bundle since ExpandPostRA expects an instruction here. 672 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 673 return; 674 } 675 676 RC = RI.getPhysRegClass(DestReg); 677 } 678 679 if (RC == &AMDGPU::VGPR_32RegClass) { 680 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 681 AMDGPU::SReg_32RegClass.contains(SrcReg) || 682 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 683 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 684 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 685 BuildMI(MBB, MI, DL, get(Opc), DestReg) 686 .addReg(SrcReg, getKillRegState(KillSrc)); 687 return; 688 } 689 690 if (RC == &AMDGPU::SReg_32_XM0RegClass || 691 RC == &AMDGPU::SReg_32RegClass) { 692 if (SrcReg == AMDGPU::SCC) { 693 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 694 .addImm(1) 695 .addImm(0); 696 return; 697 } 698 699 if (DestReg == AMDGPU::VCC_LO) { 700 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 701 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 702 .addReg(SrcReg, getKillRegState(KillSrc)); 703 } else { 704 // FIXME: Hack until VReg_1 removed. 705 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 706 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 707 .addImm(0) 708 .addReg(SrcReg, getKillRegState(KillSrc)); 709 } 710 711 return; 712 } 713 714 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 715 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 716 return; 717 } 718 719 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 720 .addReg(SrcReg, getKillRegState(KillSrc)); 721 return; 722 } 723 724 if (RC == &AMDGPU::SReg_64RegClass) { 725 if (SrcReg == AMDGPU::SCC) { 726 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 727 .addImm(1) 728 .addImm(0); 729 return; 730 } 731 732 if (DestReg == AMDGPU::VCC) { 733 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 734 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 735 .addReg(SrcReg, getKillRegState(KillSrc)); 736 } else { 737 // FIXME: Hack until VReg_1 removed. 738 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 739 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 740 .addImm(0) 741 .addReg(SrcReg, getKillRegState(KillSrc)); 742 } 743 744 return; 745 } 746 747 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 748 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 749 return; 750 } 751 752 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 753 .addReg(SrcReg, getKillRegState(KillSrc)); 754 return; 755 } 756 757 if (DestReg == AMDGPU::SCC) { 758 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 759 // but SelectionDAG emits such copies for i1 sources. 760 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 761 // This copy can only be produced by patterns 762 // with explicit SCC, which are known to be enabled 763 // only for subtargets with S_CMP_LG_U64 present. 764 assert(ST.hasScalarCompareEq64()); 765 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 766 .addReg(SrcReg, getKillRegState(KillSrc)) 767 .addImm(0); 768 } else { 769 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 770 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 771 .addReg(SrcReg, getKillRegState(KillSrc)) 772 .addImm(0); 773 } 774 775 return; 776 } 777 778 if (RC == &AMDGPU::AGPR_32RegClass) { 779 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) { 780 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 781 .addReg(SrcReg, getKillRegState(KillSrc)); 782 return; 783 } 784 785 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { 786 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) 787 .addReg(SrcReg, getKillRegState(KillSrc)); 788 return; 789 } 790 791 // FIXME: Pass should maintain scavenger to avoid scan through the block on 792 // every AGPR spill. 793 RegScavenger RS; 794 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS); 795 return; 796 } 797 798 const unsigned Size = RI.getRegSizeInBits(*RC); 799 if (Size == 16) { 800 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 801 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || 802 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 803 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 804 805 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 806 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 807 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 808 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 809 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || 810 AMDGPU::SReg_LO16RegClass.contains(DestReg) || 811 AMDGPU::AGPR_LO16RegClass.contains(DestReg); 812 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 813 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 814 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 815 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 816 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 817 818 if (IsSGPRDst) { 819 if (!IsSGPRSrc) { 820 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 821 return; 822 } 823 824 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 825 .addReg(NewSrcReg, getKillRegState(KillSrc)); 826 return; 827 } 828 829 if (IsAGPRDst || IsAGPRSrc) { 830 if (!DstLow || !SrcLow) { 831 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 832 "Cannot use hi16 subreg with an AGPR!"); 833 } 834 835 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 836 return; 837 } 838 839 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 840 if (!DstLow || !SrcLow) { 841 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 842 "Cannot use hi16 subreg on VI!"); 843 } 844 845 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 846 .addReg(NewSrcReg, getKillRegState(KillSrc)); 847 return; 848 } 849 850 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 851 .addImm(0) // src0_modifiers 852 .addReg(NewSrcReg) 853 .addImm(0) // clamp 854 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 855 : AMDGPU::SDWA::SdwaSel::WORD_1) 856 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 857 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 858 : AMDGPU::SDWA::SdwaSel::WORD_1) 859 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 860 // First implicit operand is $exec. 861 MIB->tieOperands(0, MIB->getNumOperands() - 1); 862 return; 863 } 864 865 const TargetRegisterClass *SrcRC = RI.getPhysRegClass(SrcReg); 866 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) { 867 if (ST.hasPackedFP32Ops()) { 868 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) 869 .addImm(SISrcMods::OP_SEL_1) 870 .addReg(SrcReg) 871 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 872 .addReg(SrcReg) 873 .addImm(0) // op_sel_lo 874 .addImm(0) // op_sel_hi 875 .addImm(0) // neg_lo 876 .addImm(0) // neg_hi 877 .addImm(0) // clamp 878 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit); 879 return; 880 } 881 } 882 883 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 884 if (RI.isSGPRClass(RC)) { 885 if (!RI.isSGPRClass(SrcRC)) { 886 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 887 return; 888 } 889 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RC, Forward); 890 return; 891 } 892 893 unsigned EltSize = 4; 894 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 895 if (RI.hasAGPRs(RC)) { 896 Opcode = (RI.hasVGPRs(SrcRC)) ? 897 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 898 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(SrcRC)) { 899 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 900 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && 901 (RI.isProperlyAlignedRC(*RC) && 902 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) { 903 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov. 904 if (ST.hasPackedFP32Ops()) { 905 Opcode = AMDGPU::V_PK_MOV_B32; 906 EltSize = 8; 907 } 908 } 909 910 // For the cases where we need an intermediate instruction/temporary register 911 // (destination is an AGPR), we need a scavenger. 912 // 913 // FIXME: The pass should maintain this for us so we don't have to re-scan the 914 // whole block for every handled copy. 915 std::unique_ptr<RegScavenger> RS; 916 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 917 RS.reset(new RegScavenger()); 918 919 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize); 920 921 // If there is an overlap, we can't kill the super-register on the last 922 // instruction, since it will also kill the components made live by this def. 923 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 924 925 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 926 unsigned SubIdx; 927 if (Forward) 928 SubIdx = SubIndices[Idx]; 929 else 930 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 931 932 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 933 934 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 935 Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register(); 936 Register ImpUseSuper = SrcReg; 937 indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx), 938 RI.getSubReg(SrcReg, SubIdx), UseKill, *RS, 939 ImpDefSuper, ImpUseSuper); 940 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { 941 Register DstSubReg = RI.getSubReg(DestReg, SubIdx); 942 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 943 MachineInstrBuilder MIB = 944 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg) 945 .addImm(SISrcMods::OP_SEL_1) 946 .addReg(SrcSubReg) 947 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 948 .addReg(SrcSubReg) 949 .addImm(0) // op_sel_lo 950 .addImm(0) // op_sel_hi 951 .addImm(0) // neg_lo 952 .addImm(0) // neg_hi 953 .addImm(0) // clamp 954 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 955 if (Idx == 0) 956 MIB.addReg(DestReg, RegState::Define | RegState::Implicit); 957 } else { 958 MachineInstrBuilder Builder = 959 BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx)) 960 .addReg(RI.getSubReg(SrcReg, SubIdx)); 961 if (Idx == 0) 962 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 963 964 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 965 } 966 } 967 } 968 969 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 970 int NewOpc; 971 972 // Try to map original to commuted opcode 973 NewOpc = AMDGPU::getCommuteRev(Opcode); 974 if (NewOpc != -1) 975 // Check if the commuted (REV) opcode exists on the target. 976 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 977 978 // Try to map commuted to original opcode 979 NewOpc = AMDGPU::getCommuteOrig(Opcode); 980 if (NewOpc != -1) 981 // Check if the original (non-REV) opcode exists on the target. 982 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 983 984 return Opcode; 985 } 986 987 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 988 MachineBasicBlock::iterator MI, 989 const DebugLoc &DL, unsigned DestReg, 990 int64_t Value) const { 991 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 992 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 993 if (RegClass == &AMDGPU::SReg_32RegClass || 994 RegClass == &AMDGPU::SGPR_32RegClass || 995 RegClass == &AMDGPU::SReg_32_XM0RegClass || 996 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 997 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 998 .addImm(Value); 999 return; 1000 } 1001 1002 if (RegClass == &AMDGPU::SReg_64RegClass || 1003 RegClass == &AMDGPU::SGPR_64RegClass || 1004 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 1005 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 1006 .addImm(Value); 1007 return; 1008 } 1009 1010 if (RegClass == &AMDGPU::VGPR_32RegClass) { 1011 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 1012 .addImm(Value); 1013 return; 1014 } 1015 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { 1016 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 1017 .addImm(Value); 1018 return; 1019 } 1020 1021 unsigned EltSize = 4; 1022 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1023 if (RI.isSGPRClass(RegClass)) { 1024 if (RI.getRegSizeInBits(*RegClass) > 32) { 1025 Opcode = AMDGPU::S_MOV_B64; 1026 EltSize = 8; 1027 } else { 1028 Opcode = AMDGPU::S_MOV_B32; 1029 EltSize = 4; 1030 } 1031 } 1032 1033 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 1034 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1035 int64_t IdxValue = Idx == 0 ? Value : 0; 1036 1037 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 1038 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 1039 Builder.addImm(IdxValue); 1040 } 1041 } 1042 1043 const TargetRegisterClass * 1044 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1045 return &AMDGPU::VGPR_32RegClass; 1046 } 1047 1048 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1049 MachineBasicBlock::iterator I, 1050 const DebugLoc &DL, Register DstReg, 1051 ArrayRef<MachineOperand> Cond, 1052 Register TrueReg, 1053 Register FalseReg) const { 1054 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1055 const TargetRegisterClass *BoolXExecRC = 1056 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1057 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1058 "Not a VGPR32 reg"); 1059 1060 if (Cond.size() == 1) { 1061 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1062 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1063 .add(Cond[0]); 1064 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1065 .addImm(0) 1066 .addReg(FalseReg) 1067 .addImm(0) 1068 .addReg(TrueReg) 1069 .addReg(SReg); 1070 } else if (Cond.size() == 2) { 1071 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1072 switch (Cond[0].getImm()) { 1073 case SIInstrInfo::SCC_TRUE: { 1074 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1075 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1076 : AMDGPU::S_CSELECT_B64), SReg) 1077 .addImm(1) 1078 .addImm(0); 1079 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1080 .addImm(0) 1081 .addReg(FalseReg) 1082 .addImm(0) 1083 .addReg(TrueReg) 1084 .addReg(SReg); 1085 break; 1086 } 1087 case SIInstrInfo::SCC_FALSE: { 1088 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1089 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1090 : AMDGPU::S_CSELECT_B64), SReg) 1091 .addImm(0) 1092 .addImm(1); 1093 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1094 .addImm(0) 1095 .addReg(FalseReg) 1096 .addImm(0) 1097 .addReg(TrueReg) 1098 .addReg(SReg); 1099 break; 1100 } 1101 case SIInstrInfo::VCCNZ: { 1102 MachineOperand RegOp = Cond[1]; 1103 RegOp.setImplicit(false); 1104 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1105 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1106 .add(RegOp); 1107 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1108 .addImm(0) 1109 .addReg(FalseReg) 1110 .addImm(0) 1111 .addReg(TrueReg) 1112 .addReg(SReg); 1113 break; 1114 } 1115 case SIInstrInfo::VCCZ: { 1116 MachineOperand RegOp = Cond[1]; 1117 RegOp.setImplicit(false); 1118 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1119 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1120 .add(RegOp); 1121 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1122 .addImm(0) 1123 .addReg(TrueReg) 1124 .addImm(0) 1125 .addReg(FalseReg) 1126 .addReg(SReg); 1127 break; 1128 } 1129 case SIInstrInfo::EXECNZ: { 1130 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1131 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1132 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1133 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1134 .addImm(0); 1135 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1136 : AMDGPU::S_CSELECT_B64), SReg) 1137 .addImm(1) 1138 .addImm(0); 1139 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1140 .addImm(0) 1141 .addReg(FalseReg) 1142 .addImm(0) 1143 .addReg(TrueReg) 1144 .addReg(SReg); 1145 break; 1146 } 1147 case SIInstrInfo::EXECZ: { 1148 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1149 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1150 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1151 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1152 .addImm(0); 1153 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1154 : AMDGPU::S_CSELECT_B64), SReg) 1155 .addImm(0) 1156 .addImm(1); 1157 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1158 .addImm(0) 1159 .addReg(FalseReg) 1160 .addImm(0) 1161 .addReg(TrueReg) 1162 .addReg(SReg); 1163 llvm_unreachable("Unhandled branch predicate EXECZ"); 1164 break; 1165 } 1166 default: 1167 llvm_unreachable("invalid branch predicate"); 1168 } 1169 } else { 1170 llvm_unreachable("Can only handle Cond size 1 or 2"); 1171 } 1172 } 1173 1174 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1175 MachineBasicBlock::iterator I, 1176 const DebugLoc &DL, 1177 Register SrcReg, int Value) const { 1178 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1179 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1180 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1181 .addImm(Value) 1182 .addReg(SrcReg); 1183 1184 return Reg; 1185 } 1186 1187 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1188 MachineBasicBlock::iterator I, 1189 const DebugLoc &DL, 1190 Register SrcReg, int Value) const { 1191 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1192 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1193 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1194 .addImm(Value) 1195 .addReg(SrcReg); 1196 1197 return Reg; 1198 } 1199 1200 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1201 1202 if (RI.hasAGPRs(DstRC)) 1203 return AMDGPU::COPY; 1204 if (RI.getRegSizeInBits(*DstRC) == 32) { 1205 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1206 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1207 return AMDGPU::S_MOV_B64; 1208 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1209 return AMDGPU::V_MOV_B64_PSEUDO; 1210 } 1211 return AMDGPU::COPY; 1212 } 1213 1214 const MCInstrDesc & 1215 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1216 bool IsIndirectSrc) const { 1217 if (IsIndirectSrc) { 1218 if (VecSize <= 32) // 4 bytes 1219 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1220 if (VecSize <= 64) // 8 bytes 1221 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1222 if (VecSize <= 96) // 12 bytes 1223 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1224 if (VecSize <= 128) // 16 bytes 1225 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1226 if (VecSize <= 160) // 20 bytes 1227 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1228 if (VecSize <= 256) // 32 bytes 1229 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1230 if (VecSize <= 512) // 64 bytes 1231 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1232 if (VecSize <= 1024) // 128 bytes 1233 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1234 1235 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1236 } 1237 1238 if (VecSize <= 32) // 4 bytes 1239 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1240 if (VecSize <= 64) // 8 bytes 1241 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1242 if (VecSize <= 96) // 12 bytes 1243 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1244 if (VecSize <= 128) // 16 bytes 1245 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1246 if (VecSize <= 160) // 20 bytes 1247 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1248 if (VecSize <= 256) // 32 bytes 1249 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1250 if (VecSize <= 512) // 64 bytes 1251 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1252 if (VecSize <= 1024) // 128 bytes 1253 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1254 1255 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1256 } 1257 1258 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1259 if (VecSize <= 32) // 4 bytes 1260 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1261 if (VecSize <= 64) // 8 bytes 1262 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1263 if (VecSize <= 96) // 12 bytes 1264 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1265 if (VecSize <= 128) // 16 bytes 1266 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1267 if (VecSize <= 160) // 20 bytes 1268 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1269 if (VecSize <= 256) // 32 bytes 1270 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1271 if (VecSize <= 512) // 64 bytes 1272 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1273 if (VecSize <= 1024) // 128 bytes 1274 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1275 1276 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1277 } 1278 1279 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1280 if (VecSize <= 32) // 4 bytes 1281 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1282 if (VecSize <= 64) // 8 bytes 1283 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1284 if (VecSize <= 96) // 12 bytes 1285 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1286 if (VecSize <= 128) // 16 bytes 1287 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1288 if (VecSize <= 160) // 20 bytes 1289 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1290 if (VecSize <= 256) // 32 bytes 1291 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1292 if (VecSize <= 512) // 64 bytes 1293 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1294 if (VecSize <= 1024) // 128 bytes 1295 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1296 1297 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1298 } 1299 1300 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1301 if (VecSize <= 64) // 8 bytes 1302 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1303 if (VecSize <= 128) // 16 bytes 1304 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1305 if (VecSize <= 256) // 32 bytes 1306 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1307 if (VecSize <= 512) // 64 bytes 1308 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1309 if (VecSize <= 1024) // 128 bytes 1310 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1311 1312 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1313 } 1314 1315 const MCInstrDesc & 1316 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1317 bool IsSGPR) const { 1318 if (IsSGPR) { 1319 switch (EltSize) { 1320 case 32: 1321 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1322 case 64: 1323 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1324 default: 1325 llvm_unreachable("invalid reg indexing elt size"); 1326 } 1327 } 1328 1329 assert(EltSize == 32 && "invalid reg indexing elt size"); 1330 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1331 } 1332 1333 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1334 switch (Size) { 1335 case 4: 1336 return AMDGPU::SI_SPILL_S32_SAVE; 1337 case 8: 1338 return AMDGPU::SI_SPILL_S64_SAVE; 1339 case 12: 1340 return AMDGPU::SI_SPILL_S96_SAVE; 1341 case 16: 1342 return AMDGPU::SI_SPILL_S128_SAVE; 1343 case 20: 1344 return AMDGPU::SI_SPILL_S160_SAVE; 1345 case 24: 1346 return AMDGPU::SI_SPILL_S192_SAVE; 1347 case 28: 1348 return AMDGPU::SI_SPILL_S224_SAVE; 1349 case 32: 1350 return AMDGPU::SI_SPILL_S256_SAVE; 1351 case 64: 1352 return AMDGPU::SI_SPILL_S512_SAVE; 1353 case 128: 1354 return AMDGPU::SI_SPILL_S1024_SAVE; 1355 default: 1356 llvm_unreachable("unknown register size"); 1357 } 1358 } 1359 1360 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1361 switch (Size) { 1362 case 4: 1363 return AMDGPU::SI_SPILL_V32_SAVE; 1364 case 8: 1365 return AMDGPU::SI_SPILL_V64_SAVE; 1366 case 12: 1367 return AMDGPU::SI_SPILL_V96_SAVE; 1368 case 16: 1369 return AMDGPU::SI_SPILL_V128_SAVE; 1370 case 20: 1371 return AMDGPU::SI_SPILL_V160_SAVE; 1372 case 24: 1373 return AMDGPU::SI_SPILL_V192_SAVE; 1374 case 28: 1375 return AMDGPU::SI_SPILL_V224_SAVE; 1376 case 32: 1377 return AMDGPU::SI_SPILL_V256_SAVE; 1378 case 64: 1379 return AMDGPU::SI_SPILL_V512_SAVE; 1380 case 128: 1381 return AMDGPU::SI_SPILL_V1024_SAVE; 1382 default: 1383 llvm_unreachable("unknown register size"); 1384 } 1385 } 1386 1387 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1388 switch (Size) { 1389 case 4: 1390 return AMDGPU::SI_SPILL_A32_SAVE; 1391 case 8: 1392 return AMDGPU::SI_SPILL_A64_SAVE; 1393 case 12: 1394 return AMDGPU::SI_SPILL_A96_SAVE; 1395 case 16: 1396 return AMDGPU::SI_SPILL_A128_SAVE; 1397 case 20: 1398 return AMDGPU::SI_SPILL_A160_SAVE; 1399 case 24: 1400 return AMDGPU::SI_SPILL_A192_SAVE; 1401 case 28: 1402 return AMDGPU::SI_SPILL_A224_SAVE; 1403 case 32: 1404 return AMDGPU::SI_SPILL_A256_SAVE; 1405 case 64: 1406 return AMDGPU::SI_SPILL_A512_SAVE; 1407 case 128: 1408 return AMDGPU::SI_SPILL_A1024_SAVE; 1409 default: 1410 llvm_unreachable("unknown register size"); 1411 } 1412 } 1413 1414 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 1415 MachineBasicBlock::iterator MI, 1416 Register SrcReg, bool isKill, 1417 int FrameIndex, 1418 const TargetRegisterClass *RC, 1419 const TargetRegisterInfo *TRI) const { 1420 MachineFunction *MF = MBB.getParent(); 1421 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1422 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1423 const DebugLoc &DL = MBB.findDebugLoc(MI); 1424 1425 MachinePointerInfo PtrInfo 1426 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1427 MachineMemOperand *MMO = MF->getMachineMemOperand( 1428 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1429 FrameInfo.getObjectAlign(FrameIndex)); 1430 unsigned SpillSize = TRI->getSpillSize(*RC); 1431 1432 if (RI.isSGPRClass(RC)) { 1433 MFI->setHasSpilledSGPRs(); 1434 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1435 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1436 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1437 1438 // We are only allowed to create one new instruction when spilling 1439 // registers, so we need to use pseudo instruction for spilling SGPRs. 1440 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1441 1442 // The SGPR spill/restore instructions only work on number sgprs, so we need 1443 // to make sure we are using the correct register class. 1444 if (SrcReg.isVirtual() && SpillSize == 4) { 1445 MachineRegisterInfo &MRI = MF->getRegInfo(); 1446 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1447 } 1448 1449 BuildMI(MBB, MI, DL, OpDesc) 1450 .addReg(SrcReg, getKillRegState(isKill)) // data 1451 .addFrameIndex(FrameIndex) // addr 1452 .addMemOperand(MMO) 1453 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1454 1455 if (RI.spillSGPRToVGPR()) 1456 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1457 return; 1458 } 1459 1460 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize) 1461 : getVGPRSpillSaveOpcode(SpillSize); 1462 MFI->setHasSpilledVGPRs(); 1463 1464 BuildMI(MBB, MI, DL, get(Opcode)) 1465 .addReg(SrcReg, getKillRegState(isKill)) // data 1466 .addFrameIndex(FrameIndex) // addr 1467 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1468 .addImm(0) // offset 1469 .addMemOperand(MMO); 1470 } 1471 1472 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1473 switch (Size) { 1474 case 4: 1475 return AMDGPU::SI_SPILL_S32_RESTORE; 1476 case 8: 1477 return AMDGPU::SI_SPILL_S64_RESTORE; 1478 case 12: 1479 return AMDGPU::SI_SPILL_S96_RESTORE; 1480 case 16: 1481 return AMDGPU::SI_SPILL_S128_RESTORE; 1482 case 20: 1483 return AMDGPU::SI_SPILL_S160_RESTORE; 1484 case 24: 1485 return AMDGPU::SI_SPILL_S192_RESTORE; 1486 case 28: 1487 return AMDGPU::SI_SPILL_S224_RESTORE; 1488 case 32: 1489 return AMDGPU::SI_SPILL_S256_RESTORE; 1490 case 64: 1491 return AMDGPU::SI_SPILL_S512_RESTORE; 1492 case 128: 1493 return AMDGPU::SI_SPILL_S1024_RESTORE; 1494 default: 1495 llvm_unreachable("unknown register size"); 1496 } 1497 } 1498 1499 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1500 switch (Size) { 1501 case 4: 1502 return AMDGPU::SI_SPILL_V32_RESTORE; 1503 case 8: 1504 return AMDGPU::SI_SPILL_V64_RESTORE; 1505 case 12: 1506 return AMDGPU::SI_SPILL_V96_RESTORE; 1507 case 16: 1508 return AMDGPU::SI_SPILL_V128_RESTORE; 1509 case 20: 1510 return AMDGPU::SI_SPILL_V160_RESTORE; 1511 case 24: 1512 return AMDGPU::SI_SPILL_V192_RESTORE; 1513 case 28: 1514 return AMDGPU::SI_SPILL_V224_RESTORE; 1515 case 32: 1516 return AMDGPU::SI_SPILL_V256_RESTORE; 1517 case 64: 1518 return AMDGPU::SI_SPILL_V512_RESTORE; 1519 case 128: 1520 return AMDGPU::SI_SPILL_V1024_RESTORE; 1521 default: 1522 llvm_unreachable("unknown register size"); 1523 } 1524 } 1525 1526 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1527 switch (Size) { 1528 case 4: 1529 return AMDGPU::SI_SPILL_A32_RESTORE; 1530 case 8: 1531 return AMDGPU::SI_SPILL_A64_RESTORE; 1532 case 12: 1533 return AMDGPU::SI_SPILL_A96_RESTORE; 1534 case 16: 1535 return AMDGPU::SI_SPILL_A128_RESTORE; 1536 case 20: 1537 return AMDGPU::SI_SPILL_A160_RESTORE; 1538 case 24: 1539 return AMDGPU::SI_SPILL_A192_RESTORE; 1540 case 28: 1541 return AMDGPU::SI_SPILL_A224_RESTORE; 1542 case 32: 1543 return AMDGPU::SI_SPILL_A256_RESTORE; 1544 case 64: 1545 return AMDGPU::SI_SPILL_A512_RESTORE; 1546 case 128: 1547 return AMDGPU::SI_SPILL_A1024_RESTORE; 1548 default: 1549 llvm_unreachable("unknown register size"); 1550 } 1551 } 1552 1553 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1554 MachineBasicBlock::iterator MI, 1555 Register DestReg, int FrameIndex, 1556 const TargetRegisterClass *RC, 1557 const TargetRegisterInfo *TRI) const { 1558 MachineFunction *MF = MBB.getParent(); 1559 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1560 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1561 const DebugLoc &DL = MBB.findDebugLoc(MI); 1562 unsigned SpillSize = TRI->getSpillSize(*RC); 1563 1564 MachinePointerInfo PtrInfo 1565 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1566 1567 MachineMemOperand *MMO = MF->getMachineMemOperand( 1568 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1569 FrameInfo.getObjectAlign(FrameIndex)); 1570 1571 if (RI.isSGPRClass(RC)) { 1572 MFI->setHasSpilledSGPRs(); 1573 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1574 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1575 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1576 1577 // FIXME: Maybe this should not include a memoperand because it will be 1578 // lowered to non-memory instructions. 1579 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1580 if (DestReg.isVirtual() && SpillSize == 4) { 1581 MachineRegisterInfo &MRI = MF->getRegInfo(); 1582 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1583 } 1584 1585 if (RI.spillSGPRToVGPR()) 1586 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1587 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1588 .addFrameIndex(FrameIndex) // addr 1589 .addMemOperand(MMO) 1590 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1591 1592 return; 1593 } 1594 1595 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize) 1596 : getVGPRSpillRestoreOpcode(SpillSize); 1597 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1598 .addFrameIndex(FrameIndex) // vaddr 1599 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1600 .addImm(0) // offset 1601 .addMemOperand(MMO); 1602 } 1603 1604 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1605 MachineBasicBlock::iterator MI) const { 1606 insertNoops(MBB, MI, 1); 1607 } 1608 1609 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 1610 MachineBasicBlock::iterator MI, 1611 unsigned Quantity) const { 1612 DebugLoc DL = MBB.findDebugLoc(MI); 1613 while (Quantity > 0) { 1614 unsigned Arg = std::min(Quantity, 8u); 1615 Quantity -= Arg; 1616 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 1617 } 1618 } 1619 1620 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 1621 auto MF = MBB.getParent(); 1622 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 1623 1624 assert(Info->isEntryFunction()); 1625 1626 if (MBB.succ_empty()) { 1627 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 1628 if (HasNoTerminator) { 1629 if (Info->returnsVoid()) { 1630 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 1631 } else { 1632 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 1633 } 1634 } 1635 } 1636 } 1637 1638 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 1639 switch (MI.getOpcode()) { 1640 default: return 1; // FIXME: Do wait states equal cycles? 1641 1642 case AMDGPU::S_NOP: 1643 return MI.getOperand(0).getImm() + 1; 1644 1645 // FIXME: Any other pseudo instruction? 1646 // SI_RETURN_TO_EPILOG is a fallthrough to code outside of the function. The 1647 // hazard, even if one exist, won't really be visible. Should we handle it? 1648 case AMDGPU::SI_MASKED_UNREACHABLE: 1649 case AMDGPU::WAVE_BARRIER: 1650 return 0; 1651 } 1652 } 1653 1654 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1655 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1656 MachineBasicBlock &MBB = *MI.getParent(); 1657 DebugLoc DL = MBB.findDebugLoc(MI); 1658 switch (MI.getOpcode()) { 1659 default: return TargetInstrInfo::expandPostRAPseudo(MI); 1660 case AMDGPU::S_MOV_B64_term: 1661 // This is only a terminator to get the correct spill code placement during 1662 // register allocation. 1663 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1664 break; 1665 1666 case AMDGPU::S_MOV_B32_term: 1667 // This is only a terminator to get the correct spill code placement during 1668 // register allocation. 1669 MI.setDesc(get(AMDGPU::S_MOV_B32)); 1670 break; 1671 1672 case AMDGPU::S_XOR_B64_term: 1673 // This is only a terminator to get the correct spill code placement during 1674 // register allocation. 1675 MI.setDesc(get(AMDGPU::S_XOR_B64)); 1676 break; 1677 1678 case AMDGPU::S_XOR_B32_term: 1679 // This is only a terminator to get the correct spill code placement during 1680 // register allocation. 1681 MI.setDesc(get(AMDGPU::S_XOR_B32)); 1682 break; 1683 case AMDGPU::S_OR_B64_term: 1684 // This is only a terminator to get the correct spill code placement during 1685 // register allocation. 1686 MI.setDesc(get(AMDGPU::S_OR_B64)); 1687 break; 1688 case AMDGPU::S_OR_B32_term: 1689 // This is only a terminator to get the correct spill code placement during 1690 // register allocation. 1691 MI.setDesc(get(AMDGPU::S_OR_B32)); 1692 break; 1693 1694 case AMDGPU::S_ANDN2_B64_term: 1695 // This is only a terminator to get the correct spill code placement during 1696 // register allocation. 1697 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 1698 break; 1699 1700 case AMDGPU::S_ANDN2_B32_term: 1701 // This is only a terminator to get the correct spill code placement during 1702 // register allocation. 1703 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 1704 break; 1705 1706 case AMDGPU::S_AND_B64_term: 1707 // This is only a terminator to get the correct spill code placement during 1708 // register allocation. 1709 MI.setDesc(get(AMDGPU::S_AND_B64)); 1710 break; 1711 1712 case AMDGPU::S_AND_B32_term: 1713 // This is only a terminator to get the correct spill code placement during 1714 // register allocation. 1715 MI.setDesc(get(AMDGPU::S_AND_B32)); 1716 break; 1717 1718 case AMDGPU::V_MOV_B64_PSEUDO: { 1719 Register Dst = MI.getOperand(0).getReg(); 1720 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1721 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1722 1723 const MachineOperand &SrcOp = MI.getOperand(1); 1724 // FIXME: Will this work for 64-bit floating point immediates? 1725 assert(!SrcOp.isFPImm()); 1726 if (SrcOp.isImm()) { 1727 APInt Imm(64, SrcOp.getImm()); 1728 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 1729 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 1730 if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) { 1731 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1732 .addImm(SISrcMods::OP_SEL_1) 1733 .addImm(Lo.getSExtValue()) 1734 .addImm(SISrcMods::OP_SEL_1) 1735 .addImm(Lo.getSExtValue()) 1736 .addImm(0) // op_sel_lo 1737 .addImm(0) // op_sel_hi 1738 .addImm(0) // neg_lo 1739 .addImm(0) // neg_hi 1740 .addImm(0); // clamp 1741 } else { 1742 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1743 .addImm(Lo.getSExtValue()) 1744 .addReg(Dst, RegState::Implicit | RegState::Define); 1745 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1746 .addImm(Hi.getSExtValue()) 1747 .addReg(Dst, RegState::Implicit | RegState::Define); 1748 } 1749 } else { 1750 assert(SrcOp.isReg()); 1751 if (ST.hasPackedFP32Ops() && 1752 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) { 1753 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1754 .addImm(SISrcMods::OP_SEL_1) // src0_mod 1755 .addReg(SrcOp.getReg()) 1756 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod 1757 .addReg(SrcOp.getReg()) 1758 .addImm(0) // op_sel_lo 1759 .addImm(0) // op_sel_hi 1760 .addImm(0) // neg_lo 1761 .addImm(0) // neg_hi 1762 .addImm(0); // clamp 1763 } else { 1764 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1765 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 1766 .addReg(Dst, RegState::Implicit | RegState::Define); 1767 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1768 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 1769 .addReg(Dst, RegState::Implicit | RegState::Define); 1770 } 1771 } 1772 MI.eraseFromParent(); 1773 break; 1774 } 1775 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 1776 expandMovDPP64(MI); 1777 break; 1778 } 1779 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { 1780 const MachineOperand &SrcOp = MI.getOperand(1); 1781 assert(!SrcOp.isFPImm()); 1782 APInt Imm(64, SrcOp.getImm()); 1783 if (Imm.isIntN(32) || isInlineConstant(Imm)) { 1784 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1785 break; 1786 } 1787 1788 Register Dst = MI.getOperand(0).getReg(); 1789 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1790 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1791 1792 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 1793 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 1794 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) 1795 .addImm(Lo.getSExtValue()) 1796 .addReg(Dst, RegState::Implicit | RegState::Define); 1797 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) 1798 .addImm(Hi.getSExtValue()) 1799 .addReg(Dst, RegState::Implicit | RegState::Define); 1800 MI.eraseFromParent(); 1801 break; 1802 } 1803 case AMDGPU::V_SET_INACTIVE_B32: { 1804 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1805 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1806 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 1807 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 1808 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 1809 .add(MI.getOperand(2)); 1810 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1811 .addReg(Exec); 1812 MI.eraseFromParent(); 1813 break; 1814 } 1815 case AMDGPU::V_SET_INACTIVE_B64: { 1816 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1817 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1818 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 1819 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 1820 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 1821 MI.getOperand(0).getReg()) 1822 .add(MI.getOperand(2)); 1823 expandPostRAPseudo(*Copy); 1824 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1825 .addReg(Exec); 1826 MI.eraseFromParent(); 1827 break; 1828 } 1829 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1830 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1831 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1832 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1833 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1834 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1835 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1836 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1837 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1838 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1839 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1840 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1841 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1842 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1843 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1844 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1845 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 1846 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 1847 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 1848 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 1849 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 1850 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 1851 1852 unsigned Opc; 1853 if (RI.hasVGPRs(EltRC)) { 1854 Opc = AMDGPU::V_MOVRELD_B32_e32; 1855 } else { 1856 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 1857 : AMDGPU::S_MOVRELD_B32; 1858 } 1859 1860 const MCInstrDesc &OpDesc = get(Opc); 1861 Register VecReg = MI.getOperand(0).getReg(); 1862 bool IsUndef = MI.getOperand(1).isUndef(); 1863 unsigned SubReg = MI.getOperand(3).getImm(); 1864 assert(VecReg == MI.getOperand(1).getReg()); 1865 1866 MachineInstrBuilder MIB = 1867 BuildMI(MBB, MI, DL, OpDesc) 1868 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1869 .add(MI.getOperand(2)) 1870 .addReg(VecReg, RegState::ImplicitDefine) 1871 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1872 1873 const int ImpDefIdx = 1874 OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1875 const int ImpUseIdx = ImpDefIdx + 1; 1876 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1877 MI.eraseFromParent(); 1878 break; 1879 } 1880 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 1881 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 1882 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 1883 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 1884 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 1885 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 1886 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 1887 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 1888 assert(ST.useVGPRIndexMode()); 1889 Register VecReg = MI.getOperand(0).getReg(); 1890 bool IsUndef = MI.getOperand(1).isUndef(); 1891 Register Idx = MI.getOperand(3).getReg(); 1892 Register SubReg = MI.getOperand(4).getImm(); 1893 1894 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1895 .addReg(Idx) 1896 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 1897 SetOn->getOperand(3).setIsUndef(); 1898 1899 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect); 1900 MachineInstrBuilder MIB = 1901 BuildMI(MBB, MI, DL, OpDesc) 1902 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1903 .add(MI.getOperand(2)) 1904 .addReg(VecReg, RegState::ImplicitDefine) 1905 .addReg(VecReg, 1906 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1907 1908 const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1909 const int ImpUseIdx = ImpDefIdx + 1; 1910 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1911 1912 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1913 1914 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1915 1916 MI.eraseFromParent(); 1917 break; 1918 } 1919 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 1920 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 1921 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 1922 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 1923 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 1924 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 1925 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 1926 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 1927 assert(ST.useVGPRIndexMode()); 1928 Register Dst = MI.getOperand(0).getReg(); 1929 Register VecReg = MI.getOperand(1).getReg(); 1930 bool IsUndef = MI.getOperand(1).isUndef(); 1931 Register Idx = MI.getOperand(2).getReg(); 1932 Register SubReg = MI.getOperand(3).getImm(); 1933 1934 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1935 .addReg(Idx) 1936 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 1937 SetOn->getOperand(3).setIsUndef(); 1938 1939 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32)) 1940 .addDef(Dst) 1941 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1942 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)) 1943 .addReg(AMDGPU::M0, RegState::Implicit); 1944 1945 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1946 1947 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1948 1949 MI.eraseFromParent(); 1950 break; 1951 } 1952 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 1953 MachineFunction &MF = *MBB.getParent(); 1954 Register Reg = MI.getOperand(0).getReg(); 1955 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 1956 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 1957 1958 // Create a bundle so these instructions won't be re-ordered by the 1959 // post-RA scheduler. 1960 MIBundleBuilder Bundler(MBB, MI); 1961 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 1962 1963 // Add 32-bit offset from this instruction to the start of the 1964 // constant data. 1965 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) 1966 .addReg(RegLo) 1967 .add(MI.getOperand(1))); 1968 1969 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 1970 .addReg(RegHi); 1971 MIB.add(MI.getOperand(2)); 1972 1973 Bundler.append(MIB); 1974 finalizeBundle(MBB, Bundler.begin()); 1975 1976 MI.eraseFromParent(); 1977 break; 1978 } 1979 case AMDGPU::ENTER_STRICT_WWM: { 1980 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1981 // Whole Wave Mode is entered. 1982 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1983 : AMDGPU::S_OR_SAVEEXEC_B64)); 1984 break; 1985 } 1986 case AMDGPU::ENTER_STRICT_WQM: { 1987 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1988 // STRICT_WQM is entered. 1989 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1990 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; 1991 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 1992 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec); 1993 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec); 1994 1995 MI.eraseFromParent(); 1996 break; 1997 } 1998 case AMDGPU::EXIT_STRICT_WWM: 1999 case AMDGPU::EXIT_STRICT_WQM: { 2000 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 2001 // WWM/STICT_WQM is exited. 2002 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 2003 break; 2004 } 2005 } 2006 return true; 2007 } 2008 2009 std::pair<MachineInstr*, MachineInstr*> 2010 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 2011 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 2012 2013 MachineBasicBlock &MBB = *MI.getParent(); 2014 DebugLoc DL = MBB.findDebugLoc(MI); 2015 MachineFunction *MF = MBB.getParent(); 2016 MachineRegisterInfo &MRI = MF->getRegInfo(); 2017 Register Dst = MI.getOperand(0).getReg(); 2018 unsigned Part = 0; 2019 MachineInstr *Split[2]; 2020 2021 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 2022 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 2023 if (Dst.isPhysical()) { 2024 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 2025 } else { 2026 assert(MRI.isSSA()); 2027 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2028 MovDPP.addDef(Tmp); 2029 } 2030 2031 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 2032 const MachineOperand &SrcOp = MI.getOperand(I); 2033 assert(!SrcOp.isFPImm()); 2034 if (SrcOp.isImm()) { 2035 APInt Imm(64, SrcOp.getImm()); 2036 Imm.ashrInPlace(Part * 32); 2037 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 2038 } else { 2039 assert(SrcOp.isReg()); 2040 Register Src = SrcOp.getReg(); 2041 if (Src.isPhysical()) 2042 MovDPP.addReg(RI.getSubReg(Src, Sub)); 2043 else 2044 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 2045 } 2046 } 2047 2048 for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I) 2049 MovDPP.addImm(MI.getOperand(I).getImm()); 2050 2051 Split[Part] = MovDPP; 2052 ++Part; 2053 } 2054 2055 if (Dst.isVirtual()) 2056 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 2057 .addReg(Split[0]->getOperand(0).getReg()) 2058 .addImm(AMDGPU::sub0) 2059 .addReg(Split[1]->getOperand(0).getReg()) 2060 .addImm(AMDGPU::sub1); 2061 2062 MI.eraseFromParent(); 2063 return std::make_pair(Split[0], Split[1]); 2064 } 2065 2066 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 2067 MachineOperand &Src0, 2068 unsigned Src0OpName, 2069 MachineOperand &Src1, 2070 unsigned Src1OpName) const { 2071 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 2072 if (!Src0Mods) 2073 return false; 2074 2075 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 2076 assert(Src1Mods && 2077 "All commutable instructions have both src0 and src1 modifiers"); 2078 2079 int Src0ModsVal = Src0Mods->getImm(); 2080 int Src1ModsVal = Src1Mods->getImm(); 2081 2082 Src1Mods->setImm(Src0ModsVal); 2083 Src0Mods->setImm(Src1ModsVal); 2084 return true; 2085 } 2086 2087 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 2088 MachineOperand &RegOp, 2089 MachineOperand &NonRegOp) { 2090 Register Reg = RegOp.getReg(); 2091 unsigned SubReg = RegOp.getSubReg(); 2092 bool IsKill = RegOp.isKill(); 2093 bool IsDead = RegOp.isDead(); 2094 bool IsUndef = RegOp.isUndef(); 2095 bool IsDebug = RegOp.isDebug(); 2096 2097 if (NonRegOp.isImm()) 2098 RegOp.ChangeToImmediate(NonRegOp.getImm()); 2099 else if (NonRegOp.isFI()) 2100 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 2101 else if (NonRegOp.isGlobal()) { 2102 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 2103 NonRegOp.getTargetFlags()); 2104 } else 2105 return nullptr; 2106 2107 // Make sure we don't reinterpret a subreg index in the target flags. 2108 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 2109 2110 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 2111 NonRegOp.setSubReg(SubReg); 2112 2113 return &MI; 2114 } 2115 2116 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2117 unsigned Src0Idx, 2118 unsigned Src1Idx) const { 2119 assert(!NewMI && "this should never be used"); 2120 2121 unsigned Opc = MI.getOpcode(); 2122 int CommutedOpcode = commuteOpcode(Opc); 2123 if (CommutedOpcode == -1) 2124 return nullptr; 2125 2126 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 2127 static_cast<int>(Src0Idx) && 2128 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 2129 static_cast<int>(Src1Idx) && 2130 "inconsistency with findCommutedOpIndices"); 2131 2132 MachineOperand &Src0 = MI.getOperand(Src0Idx); 2133 MachineOperand &Src1 = MI.getOperand(Src1Idx); 2134 2135 MachineInstr *CommutedMI = nullptr; 2136 if (Src0.isReg() && Src1.isReg()) { 2137 if (isOperandLegal(MI, Src1Idx, &Src0)) { 2138 // Be sure to copy the source modifiers to the right place. 2139 CommutedMI 2140 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2141 } 2142 2143 } else if (Src0.isReg() && !Src1.isReg()) { 2144 // src0 should always be able to support any operand type, so no need to 2145 // check operand legality. 2146 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2147 } else if (!Src0.isReg() && Src1.isReg()) { 2148 if (isOperandLegal(MI, Src1Idx, &Src0)) 2149 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2150 } else { 2151 // FIXME: Found two non registers to commute. This does happen. 2152 return nullptr; 2153 } 2154 2155 if (CommutedMI) { 2156 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2157 Src1, AMDGPU::OpName::src1_modifiers); 2158 2159 CommutedMI->setDesc(get(CommutedOpcode)); 2160 } 2161 2162 return CommutedMI; 2163 } 2164 2165 // This needs to be implemented because the source modifiers may be inserted 2166 // between the true commutable operands, and the base 2167 // TargetInstrInfo::commuteInstruction uses it. 2168 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2169 unsigned &SrcOpIdx0, 2170 unsigned &SrcOpIdx1) const { 2171 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2172 } 2173 2174 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0, 2175 unsigned &SrcOpIdx1) const { 2176 if (!Desc.isCommutable()) 2177 return false; 2178 2179 unsigned Opc = Desc.getOpcode(); 2180 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2181 if (Src0Idx == -1) 2182 return false; 2183 2184 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2185 if (Src1Idx == -1) 2186 return false; 2187 2188 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2189 } 2190 2191 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2192 int64_t BrOffset) const { 2193 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2194 // block is unanalyzable. 2195 assert(BranchOp != AMDGPU::S_SETPC_B64); 2196 2197 // Convert to dwords. 2198 BrOffset /= 4; 2199 2200 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2201 // from the next instruction. 2202 BrOffset -= 1; 2203 2204 return isIntN(BranchOffsetBits, BrOffset); 2205 } 2206 2207 MachineBasicBlock *SIInstrInfo::getBranchDestBlock( 2208 const MachineInstr &MI) const { 2209 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { 2210 // This would be a difficult analysis to perform, but can always be legal so 2211 // there's no need to analyze it. 2212 return nullptr; 2213 } 2214 2215 return MI.getOperand(0).getMBB(); 2216 } 2217 2218 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2219 MachineBasicBlock &DestBB, 2220 const DebugLoc &DL, 2221 int64_t BrOffset, 2222 RegScavenger *RS) const { 2223 assert(RS && "RegScavenger required for long branching"); 2224 assert(MBB.empty() && 2225 "new block should be inserted for expanding unconditional branch"); 2226 assert(MBB.pred_size() == 1); 2227 2228 MachineFunction *MF = MBB.getParent(); 2229 MachineRegisterInfo &MRI = MF->getRegInfo(); 2230 2231 // FIXME: Virtual register workaround for RegScavenger not working with empty 2232 // blocks. 2233 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2234 2235 auto I = MBB.end(); 2236 2237 // We need to compute the offset relative to the instruction immediately after 2238 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2239 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2240 2241 auto &MCCtx = MF->getContext(); 2242 MCSymbol *PostGetPCLabel = 2243 MCCtx.createTempSymbol("post_getpc", /*AlwaysAddSuffix=*/true); 2244 GetPC->setPostInstrSymbol(*MF, PostGetPCLabel); 2245 2246 MCSymbol *OffsetLo = 2247 MCCtx.createTempSymbol("offset_lo", /*AlwaysAddSuffix=*/true); 2248 MCSymbol *OffsetHi = 2249 MCCtx.createTempSymbol("offset_hi", /*AlwaysAddSuffix=*/true); 2250 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2251 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2252 .addReg(PCReg, 0, AMDGPU::sub0) 2253 .addSym(OffsetLo, MO_FAR_BRANCH_OFFSET); 2254 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2255 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2256 .addReg(PCReg, 0, AMDGPU::sub1) 2257 .addSym(OffsetHi, MO_FAR_BRANCH_OFFSET); 2258 2259 // Insert the indirect branch after the other terminator. 2260 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2261 .addReg(PCReg); 2262 2263 auto ComputeBlockSize = [](const TargetInstrInfo *TII, 2264 const MachineBasicBlock &MBB) { 2265 unsigned Size = 0; 2266 for (const MachineInstr &MI : MBB) 2267 Size += TII->getInstSizeInBytes(MI); 2268 return Size; 2269 }; 2270 2271 // FIXME: If spilling is necessary, this will fail because this scavenger has 2272 // no emergency stack slots. It is non-trivial to spill in this situation, 2273 // because the restore code needs to be specially placed after the 2274 // jump. BranchRelaxation then needs to be made aware of the newly inserted 2275 // block. 2276 // 2277 // If a spill is needed for the pc register pair, we need to insert a spill 2278 // restore block right before the destination block, and insert a short branch 2279 // into the old destination block's fallthrough predecessor. 2280 // e.g.: 2281 // 2282 // s_cbranch_scc0 skip_long_branch: 2283 // 2284 // long_branch_bb: 2285 // spill s[8:9] 2286 // s_getpc_b64 s[8:9] 2287 // s_add_u32 s8, s8, restore_bb 2288 // s_addc_u32 s9, s9, 0 2289 // s_setpc_b64 s[8:9] 2290 // 2291 // skip_long_branch: 2292 // foo; 2293 // 2294 // ..... 2295 // 2296 // dest_bb_fallthrough_predecessor: 2297 // bar; 2298 // s_branch dest_bb 2299 // 2300 // restore_bb: 2301 // restore s[8:9] 2302 // fallthrough dest_bb 2303 /// 2304 // dest_bb: 2305 // buzz; 2306 2307 RS->enterBasicBlockEnd(MBB); 2308 Register Scav = RS->scavengeRegisterBackwards( 2309 AMDGPU::SReg_64RegClass, 2310 MachineBasicBlock::iterator(GetPC), false, 0); 2311 MRI.replaceRegWith(PCReg, Scav); 2312 MRI.clearVirtRegs(); 2313 RS->setRegUsed(Scav); 2314 2315 // Now, the distance could be defined. 2316 auto *Offset = MCBinaryExpr::createSub( 2317 MCSymbolRefExpr::create(DestBB.getSymbol(), MCCtx), 2318 MCSymbolRefExpr::create(PostGetPCLabel, MCCtx), MCCtx); 2319 // Add offset assignments. 2320 auto *Mask = MCConstantExpr::create(0xFFFFFFFFULL, MCCtx); 2321 OffsetLo->setVariableValue(MCBinaryExpr::createAnd(Offset, Mask, MCCtx)); 2322 auto *ShAmt = MCConstantExpr::create(32, MCCtx); 2323 OffsetHi->setVariableValue(MCBinaryExpr::createAShr(Offset, ShAmt, MCCtx)); 2324 return ComputeBlockSize(this, MBB); 2325 } 2326 2327 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2328 switch (Cond) { 2329 case SIInstrInfo::SCC_TRUE: 2330 return AMDGPU::S_CBRANCH_SCC1; 2331 case SIInstrInfo::SCC_FALSE: 2332 return AMDGPU::S_CBRANCH_SCC0; 2333 case SIInstrInfo::VCCNZ: 2334 return AMDGPU::S_CBRANCH_VCCNZ; 2335 case SIInstrInfo::VCCZ: 2336 return AMDGPU::S_CBRANCH_VCCZ; 2337 case SIInstrInfo::EXECNZ: 2338 return AMDGPU::S_CBRANCH_EXECNZ; 2339 case SIInstrInfo::EXECZ: 2340 return AMDGPU::S_CBRANCH_EXECZ; 2341 default: 2342 llvm_unreachable("invalid branch predicate"); 2343 } 2344 } 2345 2346 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2347 switch (Opcode) { 2348 case AMDGPU::S_CBRANCH_SCC0: 2349 return SCC_FALSE; 2350 case AMDGPU::S_CBRANCH_SCC1: 2351 return SCC_TRUE; 2352 case AMDGPU::S_CBRANCH_VCCNZ: 2353 return VCCNZ; 2354 case AMDGPU::S_CBRANCH_VCCZ: 2355 return VCCZ; 2356 case AMDGPU::S_CBRANCH_EXECNZ: 2357 return EXECNZ; 2358 case AMDGPU::S_CBRANCH_EXECZ: 2359 return EXECZ; 2360 default: 2361 return INVALID_BR; 2362 } 2363 } 2364 2365 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2366 MachineBasicBlock::iterator I, 2367 MachineBasicBlock *&TBB, 2368 MachineBasicBlock *&FBB, 2369 SmallVectorImpl<MachineOperand> &Cond, 2370 bool AllowModify) const { 2371 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2372 // Unconditional Branch 2373 TBB = I->getOperand(0).getMBB(); 2374 return false; 2375 } 2376 2377 MachineBasicBlock *CondBB = nullptr; 2378 2379 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2380 CondBB = I->getOperand(1).getMBB(); 2381 Cond.push_back(I->getOperand(0)); 2382 } else { 2383 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 2384 if (Pred == INVALID_BR) 2385 return true; 2386 2387 CondBB = I->getOperand(0).getMBB(); 2388 Cond.push_back(MachineOperand::CreateImm(Pred)); 2389 Cond.push_back(I->getOperand(1)); // Save the branch register. 2390 } 2391 ++I; 2392 2393 if (I == MBB.end()) { 2394 // Conditional branch followed by fall-through. 2395 TBB = CondBB; 2396 return false; 2397 } 2398 2399 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2400 TBB = CondBB; 2401 FBB = I->getOperand(0).getMBB(); 2402 return false; 2403 } 2404 2405 return true; 2406 } 2407 2408 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 2409 MachineBasicBlock *&FBB, 2410 SmallVectorImpl<MachineOperand> &Cond, 2411 bool AllowModify) const { 2412 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2413 auto E = MBB.end(); 2414 if (I == E) 2415 return false; 2416 2417 // Skip over the instructions that are artificially terminators for special 2418 // exec management. 2419 while (I != E && !I->isBranch() && !I->isReturn()) { 2420 switch (I->getOpcode()) { 2421 case AMDGPU::S_MOV_B64_term: 2422 case AMDGPU::S_XOR_B64_term: 2423 case AMDGPU::S_OR_B64_term: 2424 case AMDGPU::S_ANDN2_B64_term: 2425 case AMDGPU::S_AND_B64_term: 2426 case AMDGPU::S_MOV_B32_term: 2427 case AMDGPU::S_XOR_B32_term: 2428 case AMDGPU::S_OR_B32_term: 2429 case AMDGPU::S_ANDN2_B32_term: 2430 case AMDGPU::S_AND_B32_term: 2431 break; 2432 case AMDGPU::SI_IF: 2433 case AMDGPU::SI_ELSE: 2434 case AMDGPU::SI_KILL_I1_TERMINATOR: 2435 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 2436 // FIXME: It's messy that these need to be considered here at all. 2437 return true; 2438 default: 2439 llvm_unreachable("unexpected non-branch terminator inst"); 2440 } 2441 2442 ++I; 2443 } 2444 2445 if (I == E) 2446 return false; 2447 2448 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 2449 } 2450 2451 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 2452 int *BytesRemoved) const { 2453 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2454 2455 unsigned Count = 0; 2456 unsigned RemovedSize = 0; 2457 while (I != MBB.end()) { 2458 MachineBasicBlock::iterator Next = std::next(I); 2459 RemovedSize += getInstSizeInBytes(*I); 2460 I->eraseFromParent(); 2461 ++Count; 2462 I = Next; 2463 } 2464 2465 if (BytesRemoved) 2466 *BytesRemoved = RemovedSize; 2467 2468 return Count; 2469 } 2470 2471 // Copy the flags onto the implicit condition register operand. 2472 static void preserveCondRegFlags(MachineOperand &CondReg, 2473 const MachineOperand &OrigCond) { 2474 CondReg.setIsUndef(OrigCond.isUndef()); 2475 CondReg.setIsKill(OrigCond.isKill()); 2476 } 2477 2478 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 2479 MachineBasicBlock *TBB, 2480 MachineBasicBlock *FBB, 2481 ArrayRef<MachineOperand> Cond, 2482 const DebugLoc &DL, 2483 int *BytesAdded) const { 2484 if (!FBB && Cond.empty()) { 2485 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2486 .addMBB(TBB); 2487 if (BytesAdded) 2488 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2489 return 1; 2490 } 2491 2492 if(Cond.size() == 1 && Cond[0].isReg()) { 2493 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 2494 .add(Cond[0]) 2495 .addMBB(TBB); 2496 return 1; 2497 } 2498 2499 assert(TBB && Cond[0].isImm()); 2500 2501 unsigned Opcode 2502 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 2503 2504 if (!FBB) { 2505 Cond[1].isUndef(); 2506 MachineInstr *CondBr = 2507 BuildMI(&MBB, DL, get(Opcode)) 2508 .addMBB(TBB); 2509 2510 // Copy the flags onto the implicit condition register operand. 2511 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 2512 fixImplicitOperands(*CondBr); 2513 2514 if (BytesAdded) 2515 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2516 return 1; 2517 } 2518 2519 assert(TBB && FBB); 2520 2521 MachineInstr *CondBr = 2522 BuildMI(&MBB, DL, get(Opcode)) 2523 .addMBB(TBB); 2524 fixImplicitOperands(*CondBr); 2525 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2526 .addMBB(FBB); 2527 2528 MachineOperand &CondReg = CondBr->getOperand(1); 2529 CondReg.setIsUndef(Cond[1].isUndef()); 2530 CondReg.setIsKill(Cond[1].isKill()); 2531 2532 if (BytesAdded) 2533 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 2534 2535 return 2; 2536 } 2537 2538 bool SIInstrInfo::reverseBranchCondition( 2539 SmallVectorImpl<MachineOperand> &Cond) const { 2540 if (Cond.size() != 2) { 2541 return true; 2542 } 2543 2544 if (Cond[0].isImm()) { 2545 Cond[0].setImm(-Cond[0].getImm()); 2546 return false; 2547 } 2548 2549 return true; 2550 } 2551 2552 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 2553 ArrayRef<MachineOperand> Cond, 2554 Register DstReg, Register TrueReg, 2555 Register FalseReg, int &CondCycles, 2556 int &TrueCycles, int &FalseCycles) const { 2557 switch (Cond[0].getImm()) { 2558 case VCCNZ: 2559 case VCCZ: { 2560 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2561 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2562 if (MRI.getRegClass(FalseReg) != RC) 2563 return false; 2564 2565 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2566 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2567 2568 // Limit to equal cost for branch vs. N v_cndmask_b32s. 2569 return RI.hasVGPRs(RC) && NumInsts <= 6; 2570 } 2571 case SCC_TRUE: 2572 case SCC_FALSE: { 2573 // FIXME: We could insert for VGPRs if we could replace the original compare 2574 // with a vector one. 2575 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2576 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2577 if (MRI.getRegClass(FalseReg) != RC) 2578 return false; 2579 2580 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2581 2582 // Multiples of 8 can do s_cselect_b64 2583 if (NumInsts % 2 == 0) 2584 NumInsts /= 2; 2585 2586 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2587 return RI.isSGPRClass(RC); 2588 } 2589 default: 2590 return false; 2591 } 2592 } 2593 2594 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 2595 MachineBasicBlock::iterator I, const DebugLoc &DL, 2596 Register DstReg, ArrayRef<MachineOperand> Cond, 2597 Register TrueReg, Register FalseReg) const { 2598 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 2599 if (Pred == VCCZ || Pred == SCC_FALSE) { 2600 Pred = static_cast<BranchPredicate>(-Pred); 2601 std::swap(TrueReg, FalseReg); 2602 } 2603 2604 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2605 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 2606 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 2607 2608 if (DstSize == 32) { 2609 MachineInstr *Select; 2610 if (Pred == SCC_TRUE) { 2611 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 2612 .addReg(TrueReg) 2613 .addReg(FalseReg); 2614 } else { 2615 // Instruction's operands are backwards from what is expected. 2616 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 2617 .addReg(FalseReg) 2618 .addReg(TrueReg); 2619 } 2620 2621 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2622 return; 2623 } 2624 2625 if (DstSize == 64 && Pred == SCC_TRUE) { 2626 MachineInstr *Select = 2627 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 2628 .addReg(TrueReg) 2629 .addReg(FalseReg); 2630 2631 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2632 return; 2633 } 2634 2635 static const int16_t Sub0_15[] = { 2636 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 2637 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 2638 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 2639 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 2640 }; 2641 2642 static const int16_t Sub0_15_64[] = { 2643 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 2644 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 2645 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 2646 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 2647 }; 2648 2649 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 2650 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 2651 const int16_t *SubIndices = Sub0_15; 2652 int NElts = DstSize / 32; 2653 2654 // 64-bit select is only available for SALU. 2655 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 2656 if (Pred == SCC_TRUE) { 2657 if (NElts % 2) { 2658 SelOp = AMDGPU::S_CSELECT_B32; 2659 EltRC = &AMDGPU::SGPR_32RegClass; 2660 } else { 2661 SelOp = AMDGPU::S_CSELECT_B64; 2662 EltRC = &AMDGPU::SGPR_64RegClass; 2663 SubIndices = Sub0_15_64; 2664 NElts /= 2; 2665 } 2666 } 2667 2668 MachineInstrBuilder MIB = BuildMI( 2669 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 2670 2671 I = MIB->getIterator(); 2672 2673 SmallVector<Register, 8> Regs; 2674 for (int Idx = 0; Idx != NElts; ++Idx) { 2675 Register DstElt = MRI.createVirtualRegister(EltRC); 2676 Regs.push_back(DstElt); 2677 2678 unsigned SubIdx = SubIndices[Idx]; 2679 2680 MachineInstr *Select; 2681 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 2682 Select = 2683 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2684 .addReg(FalseReg, 0, SubIdx) 2685 .addReg(TrueReg, 0, SubIdx); 2686 } else { 2687 Select = 2688 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2689 .addReg(TrueReg, 0, SubIdx) 2690 .addReg(FalseReg, 0, SubIdx); 2691 } 2692 2693 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2694 fixImplicitOperands(*Select); 2695 2696 MIB.addReg(DstElt) 2697 .addImm(SubIdx); 2698 } 2699 } 2700 2701 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const { 2702 switch (MI.getOpcode()) { 2703 case AMDGPU::V_MOV_B32_e32: 2704 case AMDGPU::V_MOV_B32_e64: 2705 case AMDGPU::V_MOV_B64_PSEUDO: { 2706 // If there are additional implicit register operands, this may be used for 2707 // register indexing so the source register operand isn't simply copied. 2708 unsigned NumOps = MI.getDesc().getNumOperands() + 2709 MI.getDesc().getNumImplicitUses(); 2710 2711 return MI.getNumOperands() == NumOps; 2712 } 2713 case AMDGPU::S_MOV_B32: 2714 case AMDGPU::S_MOV_B64: 2715 case AMDGPU::COPY: 2716 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2717 case AMDGPU::V_ACCVGPR_READ_B32_e64: 2718 case AMDGPU::V_ACCVGPR_MOV_B32: 2719 return true; 2720 default: 2721 return false; 2722 } 2723 } 2724 2725 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind( 2726 unsigned Kind) const { 2727 switch(Kind) { 2728 case PseudoSourceValue::Stack: 2729 case PseudoSourceValue::FixedStack: 2730 return AMDGPUAS::PRIVATE_ADDRESS; 2731 case PseudoSourceValue::ConstantPool: 2732 case PseudoSourceValue::GOT: 2733 case PseudoSourceValue::JumpTable: 2734 case PseudoSourceValue::GlobalValueCallEntry: 2735 case PseudoSourceValue::ExternalSymbolCallEntry: 2736 case PseudoSourceValue::TargetCustom: 2737 return AMDGPUAS::CONSTANT_ADDRESS; 2738 } 2739 return AMDGPUAS::FLAT_ADDRESS; 2740 } 2741 2742 static void removeModOperands(MachineInstr &MI) { 2743 unsigned Opc = MI.getOpcode(); 2744 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2745 AMDGPU::OpName::src0_modifiers); 2746 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2747 AMDGPU::OpName::src1_modifiers); 2748 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2749 AMDGPU::OpName::src2_modifiers); 2750 2751 MI.RemoveOperand(Src2ModIdx); 2752 MI.RemoveOperand(Src1ModIdx); 2753 MI.RemoveOperand(Src0ModIdx); 2754 } 2755 2756 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 2757 Register Reg, MachineRegisterInfo *MRI) const { 2758 if (!MRI->hasOneNonDBGUse(Reg)) 2759 return false; 2760 2761 switch (DefMI.getOpcode()) { 2762 default: 2763 return false; 2764 case AMDGPU::S_MOV_B64: 2765 // TODO: We could fold 64-bit immediates, but this get compilicated 2766 // when there are sub-registers. 2767 return false; 2768 2769 case AMDGPU::V_MOV_B32_e32: 2770 case AMDGPU::S_MOV_B32: 2771 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2772 break; 2773 } 2774 2775 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 2776 assert(ImmOp); 2777 // FIXME: We could handle FrameIndex values here. 2778 if (!ImmOp->isImm()) 2779 return false; 2780 2781 unsigned Opc = UseMI.getOpcode(); 2782 if (Opc == AMDGPU::COPY) { 2783 Register DstReg = UseMI.getOperand(0).getReg(); 2784 bool Is16Bit = getOpSize(UseMI, 0) == 2; 2785 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 2786 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; 2787 APInt Imm(32, ImmOp->getImm()); 2788 2789 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) 2790 Imm = Imm.ashr(16); 2791 2792 if (RI.isAGPR(*MRI, DstReg)) { 2793 if (!isInlineConstant(Imm)) 2794 return false; 2795 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 2796 } 2797 2798 if (Is16Bit) { 2799 if (isVGPRCopy) 2800 return false; // Do not clobber vgpr_hi16 2801 2802 if (DstReg.isVirtual() && 2803 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 2804 return false; 2805 2806 UseMI.getOperand(0).setSubReg(0); 2807 if (DstReg.isPhysical()) { 2808 DstReg = RI.get32BitRegister(DstReg); 2809 UseMI.getOperand(0).setReg(DstReg); 2810 } 2811 assert(UseMI.getOperand(1).getReg().isVirtual()); 2812 } 2813 2814 UseMI.setDesc(get(NewOpc)); 2815 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 2816 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 2817 return true; 2818 } 2819 2820 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2821 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 2822 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2823 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) { 2824 // Don't fold if we are using source or output modifiers. The new VOP2 2825 // instructions don't have them. 2826 if (hasAnyModifiersSet(UseMI)) 2827 return false; 2828 2829 // If this is a free constant, there's no reason to do this. 2830 // TODO: We could fold this here instead of letting SIFoldOperands do it 2831 // later. 2832 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 2833 2834 // Any src operand can be used for the legality check. 2835 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 2836 return false; 2837 2838 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2839 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 2840 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2841 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64; 2842 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 2843 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 2844 2845 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 2846 // We should only expect these to be on src0 due to canonicalizations. 2847 if (Src0->isReg() && Src0->getReg() == Reg) { 2848 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))) 2849 return false; 2850 2851 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 2852 return false; 2853 2854 unsigned NewOpc = 2855 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) 2856 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 2857 if (pseudoToMCOpcode(NewOpc) == -1) 2858 return false; 2859 2860 // We need to swap operands 0 and 1 since madmk constant is at operand 1. 2861 2862 const int64_t Imm = ImmOp->getImm(); 2863 2864 // FIXME: This would be a lot easier if we could return a new instruction 2865 // instead of having to modify in place. 2866 2867 // Remove these first since they are at the end. 2868 UseMI.RemoveOperand( 2869 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2870 UseMI.RemoveOperand( 2871 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2872 2873 Register Src1Reg = Src1->getReg(); 2874 unsigned Src1SubReg = Src1->getSubReg(); 2875 Src0->setReg(Src1Reg); 2876 Src0->setSubReg(Src1SubReg); 2877 Src0->setIsKill(Src1->isKill()); 2878 2879 if (Opc == AMDGPU::V_MAC_F32_e64 || 2880 Opc == AMDGPU::V_MAC_F16_e64 || 2881 Opc == AMDGPU::V_FMAC_F32_e64 || 2882 Opc == AMDGPU::V_FMAC_F16_e64) 2883 UseMI.untieRegOperand( 2884 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2885 2886 Src1->ChangeToImmediate(Imm); 2887 2888 removeModOperands(UseMI); 2889 UseMI.setDesc(get(NewOpc)); 2890 2891 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2892 if (DeleteDef) 2893 DefMI.eraseFromParent(); 2894 2895 return true; 2896 } 2897 2898 // Added part is the constant: Use v_madak_{f16, f32}. 2899 if (Src2->isReg() && Src2->getReg() == Reg) { 2900 // Not allowed to use constant bus for another operand. 2901 // We can however allow an inline immediate as src0. 2902 bool Src0Inlined = false; 2903 if (Src0->isReg()) { 2904 // Try to inline constant if possible. 2905 // If the Def moves immediate and the use is single 2906 // We are saving VGPR here. 2907 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 2908 if (Def && Def->isMoveImmediate() && 2909 isInlineConstant(Def->getOperand(1)) && 2910 MRI->hasOneUse(Src0->getReg())) { 2911 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2912 Src0Inlined = true; 2913 } else if ((Src0->getReg().isPhysical() && 2914 (ST.getConstantBusLimit(Opc) <= 1 && 2915 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) || 2916 (Src0->getReg().isVirtual() && 2917 (ST.getConstantBusLimit(Opc) <= 1 && 2918 RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))) 2919 return false; 2920 // VGPR is okay as Src0 - fallthrough 2921 } 2922 2923 if (Src1->isReg() && !Src0Inlined ) { 2924 // We have one slot for inlinable constant so far - try to fill it 2925 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 2926 if (Def && Def->isMoveImmediate() && 2927 isInlineConstant(Def->getOperand(1)) && 2928 MRI->hasOneUse(Src1->getReg()) && 2929 commuteInstruction(UseMI)) { 2930 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2931 } else if ((Src1->getReg().isPhysical() && 2932 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) || 2933 (Src1->getReg().isVirtual() && 2934 RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 2935 return false; 2936 // VGPR is okay as Src1 - fallthrough 2937 } 2938 2939 unsigned NewOpc = 2940 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) 2941 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 2942 if (pseudoToMCOpcode(NewOpc) == -1) 2943 return false; 2944 2945 const int64_t Imm = ImmOp->getImm(); 2946 2947 // FIXME: This would be a lot easier if we could return a new instruction 2948 // instead of having to modify in place. 2949 2950 // Remove these first since they are at the end. 2951 UseMI.RemoveOperand( 2952 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2953 UseMI.RemoveOperand( 2954 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2955 2956 if (Opc == AMDGPU::V_MAC_F32_e64 || 2957 Opc == AMDGPU::V_MAC_F16_e64 || 2958 Opc == AMDGPU::V_FMAC_F32_e64 || 2959 Opc == AMDGPU::V_FMAC_F16_e64) 2960 UseMI.untieRegOperand( 2961 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2962 2963 // ChangingToImmediate adds Src2 back to the instruction. 2964 Src2->ChangeToImmediate(Imm); 2965 2966 // These come before src2. 2967 removeModOperands(UseMI); 2968 UseMI.setDesc(get(NewOpc)); 2969 // It might happen that UseMI was commuted 2970 // and we now have SGPR as SRC1. If so 2 inlined 2971 // constant and SGPR are illegal. 2972 legalizeOperands(UseMI); 2973 2974 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2975 if (DeleteDef) 2976 DefMI.eraseFromParent(); 2977 2978 return true; 2979 } 2980 } 2981 2982 return false; 2983 } 2984 2985 static bool 2986 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 2987 ArrayRef<const MachineOperand *> BaseOps2) { 2988 if (BaseOps1.size() != BaseOps2.size()) 2989 return false; 2990 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 2991 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 2992 return false; 2993 } 2994 return true; 2995 } 2996 2997 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 2998 int WidthB, int OffsetB) { 2999 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 3000 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 3001 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 3002 return LowOffset + LowWidth <= HighOffset; 3003 } 3004 3005 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 3006 const MachineInstr &MIb) const { 3007 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 3008 int64_t Offset0, Offset1; 3009 unsigned Dummy0, Dummy1; 3010 bool Offset0IsScalable, Offset1IsScalable; 3011 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 3012 Dummy0, &RI) || 3013 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 3014 Dummy1, &RI)) 3015 return false; 3016 3017 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 3018 return false; 3019 3020 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 3021 // FIXME: Handle ds_read2 / ds_write2. 3022 return false; 3023 } 3024 unsigned Width0 = MIa.memoperands().front()->getSize(); 3025 unsigned Width1 = MIb.memoperands().front()->getSize(); 3026 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 3027 } 3028 3029 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 3030 const MachineInstr &MIb) const { 3031 assert(MIa.mayLoadOrStore() && 3032 "MIa must load from or modify a memory location"); 3033 assert(MIb.mayLoadOrStore() && 3034 "MIb must load from or modify a memory location"); 3035 3036 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 3037 return false; 3038 3039 // XXX - Can we relax this between address spaces? 3040 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 3041 return false; 3042 3043 // TODO: Should we check the address space from the MachineMemOperand? That 3044 // would allow us to distinguish objects we know don't alias based on the 3045 // underlying address space, even if it was lowered to a different one, 3046 // e.g. private accesses lowered to use MUBUF instructions on a scratch 3047 // buffer. 3048 if (isDS(MIa)) { 3049 if (isDS(MIb)) 3050 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3051 3052 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 3053 } 3054 3055 if (isMUBUF(MIa) || isMTBUF(MIa)) { 3056 if (isMUBUF(MIb) || isMTBUF(MIb)) 3057 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3058 3059 return !isFLAT(MIb) && !isSMRD(MIb); 3060 } 3061 3062 if (isSMRD(MIa)) { 3063 if (isSMRD(MIb)) 3064 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3065 3066 return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb); 3067 } 3068 3069 if (isFLAT(MIa)) { 3070 if (isFLAT(MIb)) 3071 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3072 3073 return false; 3074 } 3075 3076 return false; 3077 } 3078 3079 static int64_t getFoldableImm(const MachineOperand* MO) { 3080 if (!MO->isReg()) 3081 return false; 3082 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 3083 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3084 auto Def = MRI.getUniqueVRegDef(MO->getReg()); 3085 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && 3086 Def->getOperand(1).isImm()) 3087 return Def->getOperand(1).getImm(); 3088 return AMDGPU::NoRegister; 3089 } 3090 3091 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 3092 MachineInstr &NewMI) { 3093 if (LV) { 3094 unsigned NumOps = MI.getNumOperands(); 3095 for (unsigned I = 1; I < NumOps; ++I) { 3096 MachineOperand &Op = MI.getOperand(I); 3097 if (Op.isReg() && Op.isKill()) 3098 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 3099 } 3100 } 3101 } 3102 3103 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB, 3104 MachineInstr &MI, 3105 LiveVariables *LV) const { 3106 unsigned Opc = MI.getOpcode(); 3107 bool IsF16 = false; 3108 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 3109 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3110 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3111 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3112 3113 switch (Opc) { 3114 default: 3115 return nullptr; 3116 case AMDGPU::V_MAC_F16_e64: 3117 case AMDGPU::V_FMAC_F16_e64: 3118 IsF16 = true; 3119 LLVM_FALLTHROUGH; 3120 case AMDGPU::V_MAC_F32_e64: 3121 case AMDGPU::V_FMAC_F32_e64: 3122 case AMDGPU::V_FMAC_F64_e64: 3123 break; 3124 case AMDGPU::V_MAC_F16_e32: 3125 case AMDGPU::V_FMAC_F16_e32: 3126 IsF16 = true; 3127 LLVM_FALLTHROUGH; 3128 case AMDGPU::V_MAC_F32_e32: 3129 case AMDGPU::V_FMAC_F32_e32: 3130 case AMDGPU::V_FMAC_F64_e32: { 3131 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3132 AMDGPU::OpName::src0); 3133 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3134 if (!Src0->isReg() && !Src0->isImm()) 3135 return nullptr; 3136 3137 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3138 return nullptr; 3139 3140 break; 3141 } 3142 } 3143 3144 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3145 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3146 const MachineOperand *Src0Mods = 3147 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3148 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3149 const MachineOperand *Src1Mods = 3150 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3151 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3152 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3153 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3154 MachineInstrBuilder MIB; 3155 3156 if (!Src0Mods && !Src1Mods && !Clamp && !Omod && !IsF64 && 3157 // If we have an SGPR input, we will violate the constant bus restriction. 3158 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3159 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) { 3160 if (auto Imm = getFoldableImm(Src2)) { 3161 unsigned NewOpc = 3162 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) 3163 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3164 if (pseudoToMCOpcode(NewOpc) != -1) { 3165 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3166 .add(*Dst) 3167 .add(*Src0) 3168 .add(*Src1) 3169 .addImm(Imm); 3170 updateLiveVariables(LV, MI, *MIB); 3171 return MIB; 3172 } 3173 } 3174 unsigned NewOpc = IsFMA 3175 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) 3176 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3177 if (auto Imm = getFoldableImm(Src1)) { 3178 if (pseudoToMCOpcode(NewOpc) != -1) { 3179 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3180 .add(*Dst) 3181 .add(*Src0) 3182 .addImm(Imm) 3183 .add(*Src2); 3184 updateLiveVariables(LV, MI, *MIB); 3185 return MIB; 3186 } 3187 } 3188 if (auto Imm = getFoldableImm(Src0)) { 3189 if (pseudoToMCOpcode(NewOpc) != -1 && 3190 isOperandLegal( 3191 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3192 Src1)) { 3193 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3194 .add(*Dst) 3195 .add(*Src1) 3196 .addImm(Imm) 3197 .add(*Src2); 3198 updateLiveVariables(LV, MI, *MIB); 3199 return MIB; 3200 } 3201 } 3202 } 3203 3204 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64 3205 : IsF64 ? AMDGPU::V_FMA_F64_e64 3206 : AMDGPU::V_FMA_F32_e64) 3207 : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64); 3208 if (pseudoToMCOpcode(NewOpc) == -1) 3209 return nullptr; 3210 3211 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3212 .add(*Dst) 3213 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3214 .add(*Src0) 3215 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3216 .add(*Src1) 3217 .addImm(0) // Src mods 3218 .add(*Src2) 3219 .addImm(Clamp ? Clamp->getImm() : 0) 3220 .addImm(Omod ? Omod->getImm() : 0); 3221 updateLiveVariables(LV, MI, *MIB); 3222 return MIB; 3223 } 3224 3225 // It's not generally safe to move VALU instructions across these since it will 3226 // start using the register as a base index rather than directly. 3227 // XXX - Why isn't hasSideEffects sufficient for these? 3228 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3229 switch (MI.getOpcode()) { 3230 case AMDGPU::S_SET_GPR_IDX_ON: 3231 case AMDGPU::S_SET_GPR_IDX_MODE: 3232 case AMDGPU::S_SET_GPR_IDX_OFF: 3233 return true; 3234 default: 3235 return false; 3236 } 3237 } 3238 3239 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 3240 const MachineBasicBlock *MBB, 3241 const MachineFunction &MF) const { 3242 // Skipping the check for SP writes in the base implementation. The reason it 3243 // was added was apparently due to compile time concerns. 3244 // 3245 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 3246 // but is probably avoidable. 3247 3248 // Copied from base implementation. 3249 // Terminators and labels can't be scheduled around. 3250 if (MI.isTerminator() || MI.isPosition()) 3251 return true; 3252 3253 // INLINEASM_BR can jump to another block 3254 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 3255 return true; 3256 3257 // Target-independent instructions do not have an implicit-use of EXEC, even 3258 // when they operate on VGPRs. Treating EXEC modifications as scheduling 3259 // boundaries prevents incorrect movements of such instructions. 3260 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 3261 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 3262 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 3263 changesVGPRIndexingMode(MI); 3264 } 3265 3266 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 3267 return Opcode == AMDGPU::DS_ORDERED_COUNT || 3268 Opcode == AMDGPU::DS_GWS_INIT || 3269 Opcode == AMDGPU::DS_GWS_SEMA_V || 3270 Opcode == AMDGPU::DS_GWS_SEMA_BR || 3271 Opcode == AMDGPU::DS_GWS_SEMA_P || 3272 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || 3273 Opcode == AMDGPU::DS_GWS_BARRIER; 3274 } 3275 3276 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 3277 // Skip the full operand and register alias search modifiesRegister 3278 // does. There's only a handful of instructions that touch this, it's only an 3279 // implicit def, and doesn't alias any other registers. 3280 if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) { 3281 for (; ImpDef && *ImpDef; ++ImpDef) { 3282 if (*ImpDef == AMDGPU::MODE) 3283 return true; 3284 } 3285 } 3286 3287 return false; 3288 } 3289 3290 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 3291 unsigned Opcode = MI.getOpcode(); 3292 3293 if (MI.mayStore() && isSMRD(MI)) 3294 return true; // scalar store or atomic 3295 3296 // This will terminate the function when other lanes may need to continue. 3297 if (MI.isReturn()) 3298 return true; 3299 3300 // These instructions cause shader I/O that may cause hardware lockups 3301 // when executed with an empty EXEC mask. 3302 // 3303 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 3304 // EXEC = 0, but checking for that case here seems not worth it 3305 // given the typical code patterns. 3306 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 3307 isEXP(Opcode) || 3308 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 3309 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 3310 return true; 3311 3312 if (MI.isCall() || MI.isInlineAsm()) 3313 return true; // conservative assumption 3314 3315 // A mode change is a scalar operation that influences vector instructions. 3316 if (modifiesModeRegister(MI)) 3317 return true; 3318 3319 // These are like SALU instructions in terms of effects, so it's questionable 3320 // whether we should return true for those. 3321 // 3322 // However, executing them with EXEC = 0 causes them to operate on undefined 3323 // data, which we avoid by returning true here. 3324 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 3325 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) 3326 return true; 3327 3328 return false; 3329 } 3330 3331 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 3332 const MachineInstr &MI) const { 3333 if (MI.isMetaInstruction()) 3334 return false; 3335 3336 // This won't read exec if this is an SGPR->SGPR copy. 3337 if (MI.isCopyLike()) { 3338 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 3339 return true; 3340 3341 // Make sure this isn't copying exec as a normal operand 3342 return MI.readsRegister(AMDGPU::EXEC, &RI); 3343 } 3344 3345 // Make a conservative assumption about the callee. 3346 if (MI.isCall()) 3347 return true; 3348 3349 // Be conservative with any unhandled generic opcodes. 3350 if (!isTargetSpecificOpcode(MI.getOpcode())) 3351 return true; 3352 3353 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 3354 } 3355 3356 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 3357 switch (Imm.getBitWidth()) { 3358 case 1: // This likely will be a condition code mask. 3359 return true; 3360 3361 case 32: 3362 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 3363 ST.hasInv2PiInlineImm()); 3364 case 64: 3365 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 3366 ST.hasInv2PiInlineImm()); 3367 case 16: 3368 return ST.has16BitInsts() && 3369 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 3370 ST.hasInv2PiInlineImm()); 3371 default: 3372 llvm_unreachable("invalid bitwidth"); 3373 } 3374 } 3375 3376 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 3377 uint8_t OperandType) const { 3378 if (!MO.isImm() || 3379 OperandType < AMDGPU::OPERAND_SRC_FIRST || 3380 OperandType > AMDGPU::OPERAND_SRC_LAST) 3381 return false; 3382 3383 // MachineOperand provides no way to tell the true operand size, since it only 3384 // records a 64-bit value. We need to know the size to determine if a 32-bit 3385 // floating point immediate bit pattern is legal for an integer immediate. It 3386 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 3387 3388 int64_t Imm = MO.getImm(); 3389 switch (OperandType) { 3390 case AMDGPU::OPERAND_REG_IMM_INT32: 3391 case AMDGPU::OPERAND_REG_IMM_FP32: 3392 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3393 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3394 case AMDGPU::OPERAND_REG_IMM_V2FP32: 3395 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 3396 case AMDGPU::OPERAND_REG_IMM_V2INT32: 3397 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: 3398 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3399 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { 3400 int32_t Trunc = static_cast<int32_t>(Imm); 3401 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 3402 } 3403 case AMDGPU::OPERAND_REG_IMM_INT64: 3404 case AMDGPU::OPERAND_REG_IMM_FP64: 3405 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3406 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3407 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 3408 return AMDGPU::isInlinableLiteral64(MO.getImm(), 3409 ST.hasInv2PiInlineImm()); 3410 case AMDGPU::OPERAND_REG_IMM_INT16: 3411 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3412 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3413 // We would expect inline immediates to not be concerned with an integer/fp 3414 // distinction. However, in the case of 16-bit integer operations, the 3415 // "floating point" values appear to not work. It seems read the low 16-bits 3416 // of 32-bit immediates, which happens to always work for the integer 3417 // values. 3418 // 3419 // See llvm bugzilla 46302. 3420 // 3421 // TODO: Theoretically we could use op-sel to use the high bits of the 3422 // 32-bit FP values. 3423 return AMDGPU::isInlinableIntLiteral(Imm); 3424 case AMDGPU::OPERAND_REG_IMM_V2INT16: 3425 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 3426 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 3427 // This suffers the same problem as the scalar 16-bit cases. 3428 return AMDGPU::isInlinableIntLiteralV216(Imm); 3429 case AMDGPU::OPERAND_REG_IMM_FP16: 3430 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3431 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 3432 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 3433 // A few special case instructions have 16-bit operands on subtargets 3434 // where 16-bit instructions are not legal. 3435 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 3436 // constants in these cases 3437 int16_t Trunc = static_cast<int16_t>(Imm); 3438 return ST.has16BitInsts() && 3439 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 3440 } 3441 3442 return false; 3443 } 3444 case AMDGPU::OPERAND_REG_IMM_V2FP16: 3445 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 3446 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { 3447 uint32_t Trunc = static_cast<uint32_t>(Imm); 3448 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); 3449 } 3450 default: 3451 llvm_unreachable("invalid bitwidth"); 3452 } 3453 } 3454 3455 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO, 3456 const MCOperandInfo &OpInfo) const { 3457 switch (MO.getType()) { 3458 case MachineOperand::MO_Register: 3459 return false; 3460 case MachineOperand::MO_Immediate: 3461 return !isInlineConstant(MO, OpInfo); 3462 case MachineOperand::MO_FrameIndex: 3463 case MachineOperand::MO_MachineBasicBlock: 3464 case MachineOperand::MO_ExternalSymbol: 3465 case MachineOperand::MO_GlobalAddress: 3466 case MachineOperand::MO_MCSymbol: 3467 return true; 3468 default: 3469 llvm_unreachable("unexpected operand type"); 3470 } 3471 } 3472 3473 static bool compareMachineOp(const MachineOperand &Op0, 3474 const MachineOperand &Op1) { 3475 if (Op0.getType() != Op1.getType()) 3476 return false; 3477 3478 switch (Op0.getType()) { 3479 case MachineOperand::MO_Register: 3480 return Op0.getReg() == Op1.getReg(); 3481 case MachineOperand::MO_Immediate: 3482 return Op0.getImm() == Op1.getImm(); 3483 default: 3484 llvm_unreachable("Didn't expect to be comparing these operand types"); 3485 } 3486 } 3487 3488 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 3489 const MachineOperand &MO) const { 3490 const MCInstrDesc &InstDesc = MI.getDesc(); 3491 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo]; 3492 3493 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 3494 3495 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 3496 return true; 3497 3498 if (OpInfo.RegClass < 0) 3499 return false; 3500 3501 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 3502 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 3503 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3504 AMDGPU::OpName::src2)) 3505 return false; 3506 return RI.opCanUseInlineConstant(OpInfo.OperandType); 3507 } 3508 3509 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 3510 return false; 3511 3512 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 3513 return true; 3514 3515 return ST.hasVOP3Literal(); 3516 } 3517 3518 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 3519 // GFX90A does not have V_MUL_LEGACY_F32_e32. 3520 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) 3521 return false; 3522 3523 int Op32 = AMDGPU::getVOPe32(Opcode); 3524 if (Op32 == -1) 3525 return false; 3526 3527 return pseudoToMCOpcode(Op32) != -1; 3528 } 3529 3530 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 3531 // The src0_modifier operand is present on all instructions 3532 // that have modifiers. 3533 3534 return AMDGPU::getNamedOperandIdx(Opcode, 3535 AMDGPU::OpName::src0_modifiers) != -1; 3536 } 3537 3538 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 3539 unsigned OpName) const { 3540 const MachineOperand *Mods = getNamedOperand(MI, OpName); 3541 return Mods && Mods->getImm(); 3542 } 3543 3544 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 3545 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || 3546 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || 3547 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || 3548 hasModifiersSet(MI, AMDGPU::OpName::clamp) || 3549 hasModifiersSet(MI, AMDGPU::OpName::omod); 3550 } 3551 3552 bool SIInstrInfo::canShrink(const MachineInstr &MI, 3553 const MachineRegisterInfo &MRI) const { 3554 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3555 // Can't shrink instruction with three operands. 3556 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add 3557 // a special case for it. It can only be shrunk if the third operand 3558 // is vcc, and src0_modifiers and src1_modifiers are not set. 3559 // We should handle this the same way we handle vopc, by addding 3560 // a register allocation hint pre-regalloc and then do the shrinking 3561 // post-regalloc. 3562 if (Src2) { 3563 switch (MI.getOpcode()) { 3564 default: return false; 3565 3566 case AMDGPU::V_ADDC_U32_e64: 3567 case AMDGPU::V_SUBB_U32_e64: 3568 case AMDGPU::V_SUBBREV_U32_e64: { 3569 const MachineOperand *Src1 3570 = getNamedOperand(MI, AMDGPU::OpName::src1); 3571 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 3572 return false; 3573 // Additional verification is needed for sdst/src2. 3574 return true; 3575 } 3576 case AMDGPU::V_MAC_F32_e64: 3577 case AMDGPU::V_MAC_F16_e64: 3578 case AMDGPU::V_FMAC_F32_e64: 3579 case AMDGPU::V_FMAC_F16_e64: 3580 case AMDGPU::V_FMAC_F64_e64: 3581 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 3582 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 3583 return false; 3584 break; 3585 3586 case AMDGPU::V_CNDMASK_B32_e64: 3587 break; 3588 } 3589 } 3590 3591 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3592 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 3593 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 3594 return false; 3595 3596 // We don't need to check src0, all input types are legal, so just make sure 3597 // src0 isn't using any modifiers. 3598 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 3599 return false; 3600 3601 // Can it be shrunk to a valid 32 bit opcode? 3602 if (!hasVALU32BitEncoding(MI.getOpcode())) 3603 return false; 3604 3605 // Check output modifiers 3606 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 3607 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 3608 } 3609 3610 // Set VCC operand with all flags from \p Orig, except for setting it as 3611 // implicit. 3612 static void copyFlagsToImplicitVCC(MachineInstr &MI, 3613 const MachineOperand &Orig) { 3614 3615 for (MachineOperand &Use : MI.implicit_operands()) { 3616 if (Use.isUse() && 3617 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 3618 Use.setIsUndef(Orig.isUndef()); 3619 Use.setIsKill(Orig.isKill()); 3620 return; 3621 } 3622 } 3623 } 3624 3625 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 3626 unsigned Op32) const { 3627 MachineBasicBlock *MBB = MI.getParent();; 3628 MachineInstrBuilder Inst32 = 3629 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 3630 .setMIFlags(MI.getFlags()); 3631 3632 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 3633 // For VOPC instructions, this is replaced by an implicit def of vcc. 3634 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); 3635 if (Op32DstIdx != -1) { 3636 // dst 3637 Inst32.add(MI.getOperand(0)); 3638 } else { 3639 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 3640 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 3641 "Unexpected case"); 3642 } 3643 3644 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 3645 3646 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3647 if (Src1) 3648 Inst32.add(*Src1); 3649 3650 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3651 3652 if (Src2) { 3653 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 3654 if (Op32Src2Idx != -1) { 3655 Inst32.add(*Src2); 3656 } else { 3657 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 3658 // replaced with an implicit read of vcc or vcc_lo. The implicit read 3659 // of vcc was already added during the initial BuildMI, but we 3660 // 1) may need to change vcc to vcc_lo to preserve the original register 3661 // 2) have to preserve the original flags. 3662 fixImplicitOperands(*Inst32); 3663 copyFlagsToImplicitVCC(*Inst32, *Src2); 3664 } 3665 } 3666 3667 return Inst32; 3668 } 3669 3670 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 3671 const MachineOperand &MO, 3672 const MCOperandInfo &OpInfo) const { 3673 // Literal constants use the constant bus. 3674 //if (isLiteralConstantLike(MO, OpInfo)) 3675 // return true; 3676 if (MO.isImm()) 3677 return !isInlineConstant(MO, OpInfo); 3678 3679 if (!MO.isReg()) 3680 return true; // Misc other operands like FrameIndex 3681 3682 if (!MO.isUse()) 3683 return false; 3684 3685 if (MO.getReg().isVirtual()) 3686 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 3687 3688 // Null is free 3689 if (MO.getReg() == AMDGPU::SGPR_NULL) 3690 return false; 3691 3692 // SGPRs use the constant bus 3693 if (MO.isImplicit()) { 3694 return MO.getReg() == AMDGPU::M0 || 3695 MO.getReg() == AMDGPU::VCC || 3696 MO.getReg() == AMDGPU::VCC_LO; 3697 } else { 3698 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 3699 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 3700 } 3701 } 3702 3703 static Register findImplicitSGPRRead(const MachineInstr &MI) { 3704 for (const MachineOperand &MO : MI.implicit_operands()) { 3705 // We only care about reads. 3706 if (MO.isDef()) 3707 continue; 3708 3709 switch (MO.getReg()) { 3710 case AMDGPU::VCC: 3711 case AMDGPU::VCC_LO: 3712 case AMDGPU::VCC_HI: 3713 case AMDGPU::M0: 3714 case AMDGPU::FLAT_SCR: 3715 return MO.getReg(); 3716 3717 default: 3718 break; 3719 } 3720 } 3721 3722 return AMDGPU::NoRegister; 3723 } 3724 3725 static bool shouldReadExec(const MachineInstr &MI) { 3726 if (SIInstrInfo::isVALU(MI)) { 3727 switch (MI.getOpcode()) { 3728 case AMDGPU::V_READLANE_B32: 3729 case AMDGPU::V_WRITELANE_B32: 3730 return false; 3731 } 3732 3733 return true; 3734 } 3735 3736 if (MI.isPreISelOpcode() || 3737 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 3738 SIInstrInfo::isSALU(MI) || 3739 SIInstrInfo::isSMRD(MI)) 3740 return false; 3741 3742 return true; 3743 } 3744 3745 static bool isSubRegOf(const SIRegisterInfo &TRI, 3746 const MachineOperand &SuperVec, 3747 const MachineOperand &SubReg) { 3748 if (SubReg.getReg().isPhysical()) 3749 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 3750 3751 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 3752 SubReg.getReg() == SuperVec.getReg(); 3753 } 3754 3755 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 3756 StringRef &ErrInfo) const { 3757 uint16_t Opcode = MI.getOpcode(); 3758 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 3759 return true; 3760 3761 const MachineFunction *MF = MI.getParent()->getParent(); 3762 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3763 3764 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 3765 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 3766 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 3767 3768 // Make sure the number of operands is correct. 3769 const MCInstrDesc &Desc = get(Opcode); 3770 if (!Desc.isVariadic() && 3771 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 3772 ErrInfo = "Instruction has wrong number of operands."; 3773 return false; 3774 } 3775 3776 if (MI.isInlineAsm()) { 3777 // Verify register classes for inlineasm constraints. 3778 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 3779 I != E; ++I) { 3780 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 3781 if (!RC) 3782 continue; 3783 3784 const MachineOperand &Op = MI.getOperand(I); 3785 if (!Op.isReg()) 3786 continue; 3787 3788 Register Reg = Op.getReg(); 3789 if (!Reg.isVirtual() && !RC->contains(Reg)) { 3790 ErrInfo = "inlineasm operand has incorrect register class."; 3791 return false; 3792 } 3793 } 3794 3795 return true; 3796 } 3797 3798 if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 3799 ErrInfo = "missing memory operand from MIMG instruction."; 3800 return false; 3801 } 3802 3803 // Make sure the register classes are correct. 3804 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 3805 const MachineOperand &MO = MI.getOperand(i); 3806 if (MO.isFPImm()) { 3807 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 3808 "all fp values to integers."; 3809 return false; 3810 } 3811 3812 int RegClass = Desc.OpInfo[i].RegClass; 3813 3814 switch (Desc.OpInfo[i].OperandType) { 3815 case MCOI::OPERAND_REGISTER: 3816 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 3817 ErrInfo = "Illegal immediate value for operand."; 3818 return false; 3819 } 3820 break; 3821 case AMDGPU::OPERAND_REG_IMM_INT32: 3822 case AMDGPU::OPERAND_REG_IMM_FP32: 3823 break; 3824 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3825 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3826 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3827 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3828 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3829 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3830 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3831 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 3832 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3833 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 3834 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { 3835 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 3836 ErrInfo = "Illegal immediate value for operand."; 3837 return false; 3838 } 3839 break; 3840 } 3841 case MCOI::OPERAND_IMMEDIATE: 3842 case AMDGPU::OPERAND_KIMM32: 3843 // Check if this operand is an immediate. 3844 // FrameIndex operands will be replaced by immediates, so they are 3845 // allowed. 3846 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 3847 ErrInfo = "Expected immediate, but got non-immediate"; 3848 return false; 3849 } 3850 LLVM_FALLTHROUGH; 3851 default: 3852 continue; 3853 } 3854 3855 if (!MO.isReg()) 3856 continue; 3857 Register Reg = MO.getReg(); 3858 if (!Reg) 3859 continue; 3860 3861 // FIXME: Ideally we would have separate instruction definitions with the 3862 // aligned register constraint. 3863 // FIXME: We do not verify inline asm operands, but custom inline asm 3864 // verification is broken anyway 3865 if (ST.needsAlignedVGPRs()) { 3866 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg); 3867 const bool IsVGPR = RI.hasVGPRs(RC); 3868 const bool IsAGPR = !IsVGPR && RI.hasAGPRs(RC); 3869 if ((IsVGPR || IsAGPR) && MO.getSubReg()) { 3870 const TargetRegisterClass *SubRC = 3871 RI.getSubRegClass(RC, MO.getSubReg()); 3872 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg()); 3873 if (RC) 3874 RC = SubRC; 3875 } 3876 3877 // Check that this is the aligned version of the class. 3878 if (!RC || !RI.isProperlyAlignedRC(*RC)) { 3879 ErrInfo = "Subtarget requires even aligned vector registers"; 3880 return false; 3881 } 3882 } 3883 3884 if (RegClass != -1) { 3885 if (Reg.isVirtual()) 3886 continue; 3887 3888 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 3889 if (!RC->contains(Reg)) { 3890 ErrInfo = "Operand has incorrect register class."; 3891 return false; 3892 } 3893 } 3894 } 3895 3896 // Verify SDWA 3897 if (isSDWA(MI)) { 3898 if (!ST.hasSDWA()) { 3899 ErrInfo = "SDWA is not supported on this target"; 3900 return false; 3901 } 3902 3903 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 3904 3905 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx }; 3906 3907 for (int OpIdx: OpIndicies) { 3908 if (OpIdx == -1) 3909 continue; 3910 const MachineOperand &MO = MI.getOperand(OpIdx); 3911 3912 if (!ST.hasSDWAScalar()) { 3913 // Only VGPRS on VI 3914 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 3915 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 3916 return false; 3917 } 3918 } else { 3919 // No immediates on GFX9 3920 if (!MO.isReg()) { 3921 ErrInfo = 3922 "Only reg allowed as operands in SDWA instructions on GFX9+"; 3923 return false; 3924 } 3925 } 3926 } 3927 3928 if (!ST.hasSDWAOmod()) { 3929 // No omod allowed on VI 3930 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3931 if (OMod != nullptr && 3932 (!OMod->isImm() || OMod->getImm() != 0)) { 3933 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 3934 return false; 3935 } 3936 } 3937 3938 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 3939 if (isVOPC(BasicOpcode)) { 3940 if (!ST.hasSDWASdst() && DstIdx != -1) { 3941 // Only vcc allowed as dst on VI for VOPC 3942 const MachineOperand &Dst = MI.getOperand(DstIdx); 3943 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 3944 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 3945 return false; 3946 } 3947 } else if (!ST.hasSDWAOutModsVOPC()) { 3948 // No clamp allowed on GFX9 for VOPC 3949 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3950 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 3951 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 3952 return false; 3953 } 3954 3955 // No omod allowed on GFX9 for VOPC 3956 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3957 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 3958 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 3959 return false; 3960 } 3961 } 3962 } 3963 3964 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 3965 if (DstUnused && DstUnused->isImm() && 3966 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 3967 const MachineOperand &Dst = MI.getOperand(DstIdx); 3968 if (!Dst.isReg() || !Dst.isTied()) { 3969 ErrInfo = "Dst register should have tied register"; 3970 return false; 3971 } 3972 3973 const MachineOperand &TiedMO = 3974 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 3975 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 3976 ErrInfo = 3977 "Dst register should be tied to implicit use of preserved register"; 3978 return false; 3979 } else if (TiedMO.getReg().isPhysical() && 3980 Dst.getReg() != TiedMO.getReg()) { 3981 ErrInfo = "Dst register should use same physical register as preserved"; 3982 return false; 3983 } 3984 } 3985 } 3986 3987 // Verify MIMG 3988 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) { 3989 // Ensure that the return type used is large enough for all the options 3990 // being used TFE/LWE require an extra result register. 3991 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 3992 if (DMask) { 3993 uint64_t DMaskImm = DMask->getImm(); 3994 uint32_t RegCount = 3995 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm); 3996 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 3997 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 3998 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 3999 4000 // Adjust for packed 16 bit values 4001 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 4002 RegCount >>= 1; 4003 4004 // Adjust if using LWE or TFE 4005 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 4006 RegCount += 1; 4007 4008 const uint32_t DstIdx = 4009 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 4010 const MachineOperand &Dst = MI.getOperand(DstIdx); 4011 if (Dst.isReg()) { 4012 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 4013 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 4014 if (RegCount > DstSize) { 4015 ErrInfo = "MIMG instruction returns too many registers for dst " 4016 "register class"; 4017 return false; 4018 } 4019 } 4020 } 4021 } 4022 4023 // Verify VOP*. Ignore multiple sgpr operands on writelane. 4024 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 4025 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) { 4026 // Only look at the true operands. Only a real operand can use the constant 4027 // bus, and we don't want to check pseudo-operands like the source modifier 4028 // flags. 4029 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx }; 4030 4031 unsigned ConstantBusCount = 0; 4032 bool UsesLiteral = false; 4033 const MachineOperand *LiteralVal = nullptr; 4034 4035 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) 4036 ++ConstantBusCount; 4037 4038 SmallVector<Register, 2> SGPRsUsed; 4039 Register SGPRUsed; 4040 4041 for (int OpIdx : OpIndices) { 4042 if (OpIdx == -1) 4043 break; 4044 const MachineOperand &MO = MI.getOperand(OpIdx); 4045 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 4046 if (MO.isReg()) { 4047 SGPRUsed = MO.getReg(); 4048 if (llvm::all_of(SGPRsUsed, [SGPRUsed](unsigned SGPR) { 4049 return SGPRUsed != SGPR; 4050 })) { 4051 ++ConstantBusCount; 4052 SGPRsUsed.push_back(SGPRUsed); 4053 } 4054 } else { 4055 if (!UsesLiteral) { 4056 ++ConstantBusCount; 4057 UsesLiteral = true; 4058 LiteralVal = &MO; 4059 } else if (!MO.isIdenticalTo(*LiteralVal)) { 4060 assert(isVOP3(MI)); 4061 ErrInfo = "VOP3 instruction uses more than one literal"; 4062 return false; 4063 } 4064 } 4065 } 4066 } 4067 4068 SGPRUsed = findImplicitSGPRRead(MI); 4069 if (SGPRUsed != AMDGPU::NoRegister) { 4070 // Implicit uses may safely overlap true overands 4071 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 4072 return !RI.regsOverlap(SGPRUsed, SGPR); 4073 })) { 4074 ++ConstantBusCount; 4075 SGPRsUsed.push_back(SGPRUsed); 4076 } 4077 } 4078 4079 // v_writelane_b32 is an exception from constant bus restriction: 4080 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 4081 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 4082 Opcode != AMDGPU::V_WRITELANE_B32) { 4083 ErrInfo = "VOP* instruction violates constant bus restriction"; 4084 return false; 4085 } 4086 4087 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) { 4088 ErrInfo = "VOP3 instruction uses literal"; 4089 return false; 4090 } 4091 } 4092 4093 // Special case for writelane - this can break the multiple constant bus rule, 4094 // but still can't use more than one SGPR register 4095 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 4096 unsigned SGPRCount = 0; 4097 Register SGPRUsed = AMDGPU::NoRegister; 4098 4099 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) { 4100 if (OpIdx == -1) 4101 break; 4102 4103 const MachineOperand &MO = MI.getOperand(OpIdx); 4104 4105 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 4106 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 4107 if (MO.getReg() != SGPRUsed) 4108 ++SGPRCount; 4109 SGPRUsed = MO.getReg(); 4110 } 4111 } 4112 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 4113 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 4114 return false; 4115 } 4116 } 4117 } 4118 4119 // Verify misc. restrictions on specific instructions. 4120 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 4121 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 4122 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4123 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4124 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 4125 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 4126 if (!compareMachineOp(Src0, Src1) && 4127 !compareMachineOp(Src0, Src2)) { 4128 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 4129 return false; 4130 } 4131 } 4132 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 4133 SISrcMods::ABS) || 4134 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 4135 SISrcMods::ABS) || 4136 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 4137 SISrcMods::ABS)) { 4138 ErrInfo = "ABS not allowed in VOP3B instructions"; 4139 return false; 4140 } 4141 } 4142 4143 if (isSOP2(MI) || isSOPC(MI)) { 4144 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4145 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4146 unsigned Immediates = 0; 4147 4148 if (!Src0.isReg() && 4149 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType)) 4150 Immediates++; 4151 if (!Src1.isReg() && 4152 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType)) 4153 Immediates++; 4154 4155 if (Immediates > 1) { 4156 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 4157 return false; 4158 } 4159 } 4160 4161 if (isSOPK(MI)) { 4162 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 4163 if (Desc.isBranch()) { 4164 if (!Op->isMBB()) { 4165 ErrInfo = "invalid branch target for SOPK instruction"; 4166 return false; 4167 } 4168 } else { 4169 uint64_t Imm = Op->getImm(); 4170 if (sopkIsZext(MI)) { 4171 if (!isUInt<16>(Imm)) { 4172 ErrInfo = "invalid immediate for SOPK instruction"; 4173 return false; 4174 } 4175 } else { 4176 if (!isInt<16>(Imm)) { 4177 ErrInfo = "invalid immediate for SOPK instruction"; 4178 return false; 4179 } 4180 } 4181 } 4182 } 4183 4184 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4185 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4186 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4187 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4188 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4189 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4190 4191 const unsigned StaticNumOps = Desc.getNumOperands() + 4192 Desc.getNumImplicitUses(); 4193 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4194 4195 // Allow additional implicit operands. This allows a fixup done by the post 4196 // RA scheduler where the main implicit operand is killed and implicit-defs 4197 // are added for sub-registers that remain live after this instruction. 4198 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4199 ErrInfo = "missing implicit register operands"; 4200 return false; 4201 } 4202 4203 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4204 if (IsDst) { 4205 if (!Dst->isUse()) { 4206 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4207 return false; 4208 } 4209 4210 unsigned UseOpIdx; 4211 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4212 UseOpIdx != StaticNumOps + 1) { 4213 ErrInfo = "movrel implicit operands should be tied"; 4214 return false; 4215 } 4216 } 4217 4218 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4219 const MachineOperand &ImpUse 4220 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4221 if (!ImpUse.isReg() || !ImpUse.isUse() || 4222 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4223 ErrInfo = "src0 should be subreg of implicit vector use"; 4224 return false; 4225 } 4226 } 4227 4228 // Make sure we aren't losing exec uses in the td files. This mostly requires 4229 // being careful when using let Uses to try to add other use registers. 4230 if (shouldReadExec(MI)) { 4231 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4232 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4233 return false; 4234 } 4235 } 4236 4237 if (isSMRD(MI)) { 4238 if (MI.mayStore()) { 4239 // The register offset form of scalar stores may only use m0 as the 4240 // soffset register. 4241 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); 4242 if (Soff && Soff->getReg() != AMDGPU::M0) { 4243 ErrInfo = "scalar stores must use m0 as offset register"; 4244 return false; 4245 } 4246 } 4247 } 4248 4249 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 4250 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 4251 if (Offset->getImm() != 0) { 4252 ErrInfo = "subtarget does not support offsets in flat instructions"; 4253 return false; 4254 } 4255 } 4256 4257 if (isMIMG(MI)) { 4258 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 4259 if (DimOp) { 4260 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 4261 AMDGPU::OpName::vaddr0); 4262 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 4263 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 4264 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 4265 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 4266 const AMDGPU::MIMGDimInfo *Dim = 4267 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 4268 4269 if (!Dim) { 4270 ErrInfo = "dim is out of range"; 4271 return false; 4272 } 4273 4274 bool IsA16 = false; 4275 if (ST.hasR128A16()) { 4276 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 4277 IsA16 = R128A16->getImm() != 0; 4278 } else if (ST.hasGFX10A16()) { 4279 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 4280 IsA16 = A16->getImm() != 0; 4281 } 4282 4283 bool IsNSA = SRsrcIdx - VAddr0Idx > 1; 4284 4285 unsigned AddrWords = 4286 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); 4287 4288 unsigned VAddrWords; 4289 if (IsNSA) { 4290 VAddrWords = SRsrcIdx - VAddr0Idx; 4291 } else { 4292 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx); 4293 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32; 4294 if (AddrWords > 8) 4295 AddrWords = 16; 4296 } 4297 4298 if (VAddrWords != AddrWords) { 4299 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 4300 << " but got " << VAddrWords << "\n"); 4301 ErrInfo = "bad vaddr size"; 4302 return false; 4303 } 4304 } 4305 } 4306 4307 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 4308 if (DppCt) { 4309 using namespace AMDGPU::DPP; 4310 4311 unsigned DC = DppCt->getImm(); 4312 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 4313 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 4314 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 4315 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 4316 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 4317 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 4318 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 4319 ErrInfo = "Invalid dpp_ctrl value"; 4320 return false; 4321 } 4322 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 4323 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4324 ErrInfo = "Invalid dpp_ctrl value: " 4325 "wavefront shifts are not supported on GFX10+"; 4326 return false; 4327 } 4328 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 4329 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4330 ErrInfo = "Invalid dpp_ctrl value: " 4331 "broadcasts are not supported on GFX10+"; 4332 return false; 4333 } 4334 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 4335 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 4336 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST && 4337 DC <= DppCtrl::ROW_NEWBCAST_LAST && 4338 !ST.hasGFX90AInsts()) { 4339 ErrInfo = "Invalid dpp_ctrl value: " 4340 "row_newbroadcast/row_share is not supported before " 4341 "GFX90A/GFX10"; 4342 return false; 4343 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) { 4344 ErrInfo = "Invalid dpp_ctrl value: " 4345 "row_share and row_xmask are not supported before GFX10"; 4346 return false; 4347 } 4348 } 4349 4350 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4351 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 4352 4353 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && 4354 ((DstIdx >= 0 && 4355 (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID || 4356 Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) || 4357 ((Src0Idx >= 0 && 4358 (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID || 4359 Desc.OpInfo[Src0Idx].RegClass == 4360 AMDGPU::VReg_64_Align2RegClassID)))) && 4361 !AMDGPU::isLegal64BitDPPControl(DC)) { 4362 ErrInfo = "Invalid dpp_ctrl value: " 4363 "64 bit dpp only support row_newbcast"; 4364 return false; 4365 } 4366 } 4367 4368 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) { 4369 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4370 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 4371 : AMDGPU::OpName::vdata; 4372 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx); 4373 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); 4374 if (Data && !Data->isReg()) 4375 Data = nullptr; 4376 4377 if (ST.hasGFX90AInsts()) { 4378 if (Dst && Data && 4379 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) { 4380 ErrInfo = "Invalid register class: " 4381 "vdata and vdst should be both VGPR or AGPR"; 4382 return false; 4383 } 4384 if (Data && Data2 && 4385 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) { 4386 ErrInfo = "Invalid register class: " 4387 "both data operands should be VGPR or AGPR"; 4388 return false; 4389 } 4390 } else { 4391 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) || 4392 (Data && RI.isAGPR(MRI, Data->getReg())) || 4393 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) { 4394 ErrInfo = "Invalid register class: " 4395 "agpr loads and stores not supported on this GPU"; 4396 return false; 4397 } 4398 } 4399 } 4400 4401 if (ST.needsAlignedVGPRs() && 4402 (MI.getOpcode() == AMDGPU::DS_GWS_INIT || 4403 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || 4404 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) { 4405 const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0); 4406 Register Reg = Op->getReg(); 4407 bool Aligned = true; 4408 if (Reg.isPhysical()) { 4409 Aligned = !(RI.getHWRegIndex(Reg) & 1); 4410 } else { 4411 const TargetRegisterClass &RC = *MRI.getRegClass(Reg); 4412 Aligned = RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) && 4413 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1); 4414 } 4415 4416 if (!Aligned) { 4417 ErrInfo = "Subtarget requires even aligned vector registers " 4418 "for DS_GWS instructions"; 4419 return false; 4420 } 4421 } 4422 4423 return true; 4424 } 4425 4426 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 4427 switch (MI.getOpcode()) { 4428 default: return AMDGPU::INSTRUCTION_LIST_END; 4429 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 4430 case AMDGPU::COPY: return AMDGPU::COPY; 4431 case AMDGPU::PHI: return AMDGPU::PHI; 4432 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 4433 case AMDGPU::WQM: return AMDGPU::WQM; 4434 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 4435 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; 4436 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; 4437 case AMDGPU::S_MOV_B32: { 4438 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4439 return MI.getOperand(1).isReg() || 4440 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 4441 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 4442 } 4443 case AMDGPU::S_ADD_I32: 4444 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 4445 case AMDGPU::S_ADDC_U32: 4446 return AMDGPU::V_ADDC_U32_e32; 4447 case AMDGPU::S_SUB_I32: 4448 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 4449 // FIXME: These are not consistently handled, and selected when the carry is 4450 // used. 4451 case AMDGPU::S_ADD_U32: 4452 return AMDGPU::V_ADD_CO_U32_e32; 4453 case AMDGPU::S_SUB_U32: 4454 return AMDGPU::V_SUB_CO_U32_e32; 4455 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 4456 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 4457 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 4458 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 4459 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 4460 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 4461 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 4462 case AMDGPU::S_XNOR_B32: 4463 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 4464 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 4465 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 4466 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 4467 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 4468 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 4469 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 4470 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 4471 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 4472 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 4473 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 4474 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 4475 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 4476 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 4477 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 4478 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 4479 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 4480 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 4481 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 4482 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; 4483 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; 4484 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; 4485 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; 4486 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; 4487 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; 4488 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; 4489 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; 4490 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; 4491 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; 4492 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; 4493 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; 4494 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; 4495 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; 4496 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 4497 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 4498 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 4499 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 4500 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 4501 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 4502 } 4503 llvm_unreachable( 4504 "Unexpected scalar opcode without corresponding vector one!"); 4505 } 4506 4507 static unsigned adjustAllocatableRegClass(const GCNSubtarget &ST, 4508 const MachineRegisterInfo &MRI, 4509 const MCInstrDesc &TID, 4510 unsigned RCID, 4511 bool IsAllocatable) { 4512 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 4513 (TID.mayLoad() || TID.mayStore() || 4514 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { 4515 switch (RCID) { 4516 case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID; 4517 case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID; 4518 case AMDGPU::AV_96RegClassID: return AMDGPU::VReg_96RegClassID; 4519 case AMDGPU::AV_128RegClassID: return AMDGPU::VReg_128RegClassID; 4520 case AMDGPU::AV_160RegClassID: return AMDGPU::VReg_160RegClassID; 4521 default: 4522 break; 4523 } 4524 } 4525 return RCID; 4526 } 4527 4528 const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, 4529 unsigned OpNum, const TargetRegisterInfo *TRI, 4530 const MachineFunction &MF) 4531 const { 4532 if (OpNum >= TID.getNumOperands()) 4533 return nullptr; 4534 auto RegClass = TID.OpInfo[OpNum].RegClass; 4535 bool IsAllocatable = false; 4536 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) { 4537 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions 4538 // with two data operands. Request register class constainted to VGPR only 4539 // of both operands present as Machine Copy Propagation can not check this 4540 // constraint and possibly other passes too. 4541 // 4542 // The check is limited to FLAT and DS because atomics in non-flat encoding 4543 // have their vdst and vdata tied to be the same register. 4544 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4545 AMDGPU::OpName::vdst); 4546 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4547 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 4548 : AMDGPU::OpName::vdata); 4549 if (DataIdx != -1) { 4550 IsAllocatable = VDstIdx != -1 || 4551 AMDGPU::getNamedOperandIdx(TID.Opcode, 4552 AMDGPU::OpName::data1) != -1; 4553 } 4554 } 4555 RegClass = adjustAllocatableRegClass(ST, MF.getRegInfo(), TID, RegClass, 4556 IsAllocatable); 4557 return RI.getRegClass(RegClass); 4558 } 4559 4560 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 4561 unsigned OpNo) const { 4562 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4563 const MCInstrDesc &Desc = get(MI.getOpcode()); 4564 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 4565 Desc.OpInfo[OpNo].RegClass == -1) { 4566 Register Reg = MI.getOperand(OpNo).getReg(); 4567 4568 if (Reg.isVirtual()) 4569 return MRI.getRegClass(Reg); 4570 return RI.getPhysRegClass(Reg); 4571 } 4572 4573 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 4574 RCID = adjustAllocatableRegClass(ST, MRI, Desc, RCID, true); 4575 return RI.getRegClass(RCID); 4576 } 4577 4578 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 4579 MachineBasicBlock::iterator I = MI; 4580 MachineBasicBlock *MBB = MI.getParent(); 4581 MachineOperand &MO = MI.getOperand(OpIdx); 4582 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 4583 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass; 4584 const TargetRegisterClass *RC = RI.getRegClass(RCID); 4585 unsigned Size = RI.getRegSizeInBits(*RC); 4586 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 4587 if (MO.isReg()) 4588 Opcode = AMDGPU::COPY; 4589 else if (RI.isSGPRClass(RC)) 4590 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 4591 4592 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 4593 const TargetRegisterClass *VRC64 = RI.getVGPR64Class(); 4594 if (RI.getCommonSubClass(VRC64, VRC)) 4595 VRC = VRC64; 4596 else 4597 VRC = &AMDGPU::VGPR_32RegClass; 4598 4599 Register Reg = MRI.createVirtualRegister(VRC); 4600 DebugLoc DL = MBB->findDebugLoc(I); 4601 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 4602 MO.ChangeToRegister(Reg, false); 4603 } 4604 4605 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI, 4606 MachineRegisterInfo &MRI, 4607 MachineOperand &SuperReg, 4608 const TargetRegisterClass *SuperRC, 4609 unsigned SubIdx, 4610 const TargetRegisterClass *SubRC) 4611 const { 4612 MachineBasicBlock *MBB = MI->getParent(); 4613 DebugLoc DL = MI->getDebugLoc(); 4614 Register SubReg = MRI.createVirtualRegister(SubRC); 4615 4616 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 4617 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4618 .addReg(SuperReg.getReg(), 0, SubIdx); 4619 return SubReg; 4620 } 4621 4622 // Just in case the super register is itself a sub-register, copy it to a new 4623 // value so we don't need to worry about merging its subreg index with the 4624 // SubIdx passed to this function. The register coalescer should be able to 4625 // eliminate this extra copy. 4626 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 4627 4628 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 4629 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 4630 4631 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4632 .addReg(NewSuperReg, 0, SubIdx); 4633 4634 return SubReg; 4635 } 4636 4637 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 4638 MachineBasicBlock::iterator MII, 4639 MachineRegisterInfo &MRI, 4640 MachineOperand &Op, 4641 const TargetRegisterClass *SuperRC, 4642 unsigned SubIdx, 4643 const TargetRegisterClass *SubRC) const { 4644 if (Op.isImm()) { 4645 if (SubIdx == AMDGPU::sub0) 4646 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 4647 if (SubIdx == AMDGPU::sub1) 4648 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 4649 4650 llvm_unreachable("Unhandled register index for immediate"); 4651 } 4652 4653 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 4654 SubIdx, SubRC); 4655 return MachineOperand::CreateReg(SubReg, false); 4656 } 4657 4658 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 4659 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 4660 assert(Inst.getNumExplicitOperands() == 3); 4661 MachineOperand Op1 = Inst.getOperand(1); 4662 Inst.RemoveOperand(1); 4663 Inst.addOperand(Op1); 4664 } 4665 4666 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 4667 const MCOperandInfo &OpInfo, 4668 const MachineOperand &MO) const { 4669 if (!MO.isReg()) 4670 return false; 4671 4672 Register Reg = MO.getReg(); 4673 4674 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 4675 if (Reg.isPhysical()) 4676 return DRC->contains(Reg); 4677 4678 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 4679 4680 if (MO.getSubReg()) { 4681 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 4682 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 4683 if (!SuperRC) 4684 return false; 4685 4686 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 4687 if (!DRC) 4688 return false; 4689 } 4690 return RC->hasSuperClassEq(DRC); 4691 } 4692 4693 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 4694 const MCOperandInfo &OpInfo, 4695 const MachineOperand &MO) const { 4696 if (MO.isReg()) 4697 return isLegalRegOperand(MRI, OpInfo, MO); 4698 4699 // Handle non-register types that are treated like immediates. 4700 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 4701 return true; 4702 } 4703 4704 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 4705 const MachineOperand *MO) const { 4706 const MachineFunction &MF = *MI.getParent()->getParent(); 4707 const MachineRegisterInfo &MRI = MF.getRegInfo(); 4708 const MCInstrDesc &InstDesc = MI.getDesc(); 4709 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx]; 4710 const TargetRegisterClass *DefinedRC = 4711 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 4712 if (!MO) 4713 MO = &MI.getOperand(OpIdx); 4714 4715 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 4716 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4717 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 4718 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--) 4719 return false; 4720 4721 SmallDenseSet<RegSubRegPair> SGPRsUsed; 4722 if (MO->isReg()) 4723 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 4724 4725 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4726 if (i == OpIdx) 4727 continue; 4728 const MachineOperand &Op = MI.getOperand(i); 4729 if (Op.isReg()) { 4730 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 4731 if (!SGPRsUsed.count(SGPR) && 4732 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) { 4733 if (--ConstantBusLimit <= 0) 4734 return false; 4735 SGPRsUsed.insert(SGPR); 4736 } 4737 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { 4738 if (--ConstantBusLimit <= 0) 4739 return false; 4740 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) && 4741 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) { 4742 if (!VOP3LiteralLimit--) 4743 return false; 4744 if (--ConstantBusLimit <= 0) 4745 return false; 4746 } 4747 } 4748 } 4749 4750 if (MO->isReg()) { 4751 assert(DefinedRC); 4752 if (!isLegalRegOperand(MRI, OpInfo, *MO)) 4753 return false; 4754 bool IsAGPR = RI.isAGPR(MRI, MO->getReg()); 4755 if (IsAGPR && !ST.hasMAIInsts()) 4756 return false; 4757 unsigned Opc = MI.getOpcode(); 4758 if (IsAGPR && 4759 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 4760 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc))) 4761 return false; 4762 // Atomics should have both vdst and vdata either vgpr or agpr. 4763 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 4764 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, 4765 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); 4766 if ((int)OpIdx == VDstIdx && DataIdx != -1 && 4767 MI.getOperand(DataIdx).isReg() && 4768 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR) 4769 return false; 4770 if ((int)OpIdx == DataIdx) { 4771 if (VDstIdx != -1 && 4772 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR) 4773 return false; 4774 // DS instructions with 2 src operands also must have tied RC. 4775 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, 4776 AMDGPU::OpName::data1); 4777 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() && 4778 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR) 4779 return false; 4780 } 4781 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 4782 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && 4783 RI.isSGPRReg(MRI, MO->getReg())) 4784 return false; 4785 return true; 4786 } 4787 4788 // Handle non-register types that are treated like immediates. 4789 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 4790 4791 if (!DefinedRC) { 4792 // This operand expects an immediate. 4793 return true; 4794 } 4795 4796 return isImmOperandLegal(MI, OpIdx, *MO); 4797 } 4798 4799 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 4800 MachineInstr &MI) const { 4801 unsigned Opc = MI.getOpcode(); 4802 const MCInstrDesc &InstrDesc = get(Opc); 4803 4804 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 4805 MachineOperand &Src0 = MI.getOperand(Src0Idx); 4806 4807 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 4808 MachineOperand &Src1 = MI.getOperand(Src1Idx); 4809 4810 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 4811 // we need to only have one constant bus use before GFX10. 4812 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; 4813 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && 4814 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) || 4815 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx]))) 4816 legalizeOpWithMove(MI, Src0Idx); 4817 4818 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 4819 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 4820 // src0/src1 with V_READFIRSTLANE. 4821 if (Opc == AMDGPU::V_WRITELANE_B32) { 4822 const DebugLoc &DL = MI.getDebugLoc(); 4823 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 4824 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4825 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4826 .add(Src0); 4827 Src0.ChangeToRegister(Reg, false); 4828 } 4829 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 4830 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4831 const DebugLoc &DL = MI.getDebugLoc(); 4832 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4833 .add(Src1); 4834 Src1.ChangeToRegister(Reg, false); 4835 } 4836 return; 4837 } 4838 4839 // No VOP2 instructions support AGPRs. 4840 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 4841 legalizeOpWithMove(MI, Src0Idx); 4842 4843 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 4844 legalizeOpWithMove(MI, Src1Idx); 4845 4846 // VOP2 src0 instructions support all operand types, so we don't need to check 4847 // their legality. If src1 is already legal, we don't need to do anything. 4848 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1)) 4849 return; 4850 4851 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 4852 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 4853 // select is uniform. 4854 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 4855 RI.isVGPR(MRI, Src1.getReg())) { 4856 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4857 const DebugLoc &DL = MI.getDebugLoc(); 4858 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4859 .add(Src1); 4860 Src1.ChangeToRegister(Reg, false); 4861 return; 4862 } 4863 4864 // We do not use commuteInstruction here because it is too aggressive and will 4865 // commute if it is possible. We only want to commute here if it improves 4866 // legality. This can be called a fairly large number of times so don't waste 4867 // compile time pointlessly swapping and checking legality again. 4868 if (HasImplicitSGPR || !MI.isCommutable()) { 4869 legalizeOpWithMove(MI, Src1Idx); 4870 return; 4871 } 4872 4873 // If src0 can be used as src1, commuting will make the operands legal. 4874 // Otherwise we have to give up and insert a move. 4875 // 4876 // TODO: Other immediate-like operand kinds could be commuted if there was a 4877 // MachineOperand::ChangeTo* for them. 4878 if ((!Src1.isImm() && !Src1.isReg()) || 4879 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) { 4880 legalizeOpWithMove(MI, Src1Idx); 4881 return; 4882 } 4883 4884 int CommutedOpc = commuteOpcode(MI); 4885 if (CommutedOpc == -1) { 4886 legalizeOpWithMove(MI, Src1Idx); 4887 return; 4888 } 4889 4890 MI.setDesc(get(CommutedOpc)); 4891 4892 Register Src0Reg = Src0.getReg(); 4893 unsigned Src0SubReg = Src0.getSubReg(); 4894 bool Src0Kill = Src0.isKill(); 4895 4896 if (Src1.isImm()) 4897 Src0.ChangeToImmediate(Src1.getImm()); 4898 else if (Src1.isReg()) { 4899 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 4900 Src0.setSubReg(Src1.getSubReg()); 4901 } else 4902 llvm_unreachable("Should only have register or immediate operands"); 4903 4904 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 4905 Src1.setSubReg(Src0SubReg); 4906 fixImplicitOperands(MI); 4907 } 4908 4909 // Legalize VOP3 operands. All operand types are supported for any operand 4910 // but only one literal constant and only starting from GFX10. 4911 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 4912 MachineInstr &MI) const { 4913 unsigned Opc = MI.getOpcode(); 4914 4915 int VOP3Idx[3] = { 4916 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 4917 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 4918 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 4919 }; 4920 4921 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 4922 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 4923 // src1 and src2 must be scalar 4924 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 4925 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 4926 const DebugLoc &DL = MI.getDebugLoc(); 4927 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 4928 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4929 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4930 .add(Src1); 4931 Src1.ChangeToRegister(Reg, false); 4932 } 4933 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 4934 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4935 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4936 .add(Src2); 4937 Src2.ChangeToRegister(Reg, false); 4938 } 4939 } 4940 4941 // Find the one SGPR operand we are allowed to use. 4942 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 4943 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4944 SmallDenseSet<unsigned> SGPRsUsed; 4945 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 4946 if (SGPRReg != AMDGPU::NoRegister) { 4947 SGPRsUsed.insert(SGPRReg); 4948 --ConstantBusLimit; 4949 } 4950 4951 for (unsigned i = 0; i < 3; ++i) { 4952 int Idx = VOP3Idx[i]; 4953 if (Idx == -1) 4954 break; 4955 MachineOperand &MO = MI.getOperand(Idx); 4956 4957 if (!MO.isReg()) { 4958 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx])) 4959 continue; 4960 4961 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 4962 --LiteralLimit; 4963 --ConstantBusLimit; 4964 continue; 4965 } 4966 4967 --LiteralLimit; 4968 --ConstantBusLimit; 4969 legalizeOpWithMove(MI, Idx); 4970 continue; 4971 } 4972 4973 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) && 4974 !isOperandLegal(MI, Idx, &MO)) { 4975 legalizeOpWithMove(MI, Idx); 4976 continue; 4977 } 4978 4979 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg()))) 4980 continue; // VGPRs are legal 4981 4982 // We can use one SGPR in each VOP3 instruction prior to GFX10 4983 // and two starting from GFX10. 4984 if (SGPRsUsed.count(MO.getReg())) 4985 continue; 4986 if (ConstantBusLimit > 0) { 4987 SGPRsUsed.insert(MO.getReg()); 4988 --ConstantBusLimit; 4989 continue; 4990 } 4991 4992 // If we make it this far, then the operand is not legal and we must 4993 // legalize it. 4994 legalizeOpWithMove(MI, Idx); 4995 } 4996 } 4997 4998 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 4999 MachineRegisterInfo &MRI) const { 5000 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 5001 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 5002 Register DstReg = MRI.createVirtualRegister(SRC); 5003 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 5004 5005 if (RI.hasAGPRs(VRC)) { 5006 VRC = RI.getEquivalentVGPRClass(VRC); 5007 Register NewSrcReg = MRI.createVirtualRegister(VRC); 5008 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5009 get(TargetOpcode::COPY), NewSrcReg) 5010 .addReg(SrcReg); 5011 SrcReg = NewSrcReg; 5012 } 5013 5014 if (SubRegs == 1) { 5015 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5016 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 5017 .addReg(SrcReg); 5018 return DstReg; 5019 } 5020 5021 SmallVector<unsigned, 8> SRegs; 5022 for (unsigned i = 0; i < SubRegs; ++i) { 5023 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5024 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5025 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 5026 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 5027 SRegs.push_back(SGPR); 5028 } 5029 5030 MachineInstrBuilder MIB = 5031 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5032 get(AMDGPU::REG_SEQUENCE), DstReg); 5033 for (unsigned i = 0; i < SubRegs; ++i) { 5034 MIB.addReg(SRegs[i]); 5035 MIB.addImm(RI.getSubRegFromChannel(i)); 5036 } 5037 return DstReg; 5038 } 5039 5040 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 5041 MachineInstr &MI) const { 5042 5043 // If the pointer is store in VGPRs, then we need to move them to 5044 // SGPRs using v_readfirstlane. This is safe because we only select 5045 // loads with uniform pointers to SMRD instruction so we know the 5046 // pointer value is uniform. 5047 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 5048 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 5049 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 5050 SBase->setReg(SGPR); 5051 } 5052 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); 5053 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 5054 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 5055 SOff->setReg(SGPR); 5056 } 5057 } 5058 5059 bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { 5060 unsigned Opc = Inst.getOpcode(); 5061 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 5062 if (OldSAddrIdx < 0) 5063 return false; 5064 5065 assert(isSegmentSpecificFLAT(Inst)); 5066 5067 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); 5068 if (NewOpc < 0) 5069 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); 5070 if (NewOpc < 0) 5071 return false; 5072 5073 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo(); 5074 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx); 5075 if (RI.isSGPRReg(MRI, SAddr.getReg())) 5076 return false; 5077 5078 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); 5079 if (NewVAddrIdx < 0) 5080 return false; 5081 5082 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 5083 5084 // Check vaddr, it shall be zero or absent. 5085 MachineInstr *VAddrDef = nullptr; 5086 if (OldVAddrIdx >= 0) { 5087 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx); 5088 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg()); 5089 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || 5090 !VAddrDef->getOperand(1).isImm() || 5091 VAddrDef->getOperand(1).getImm() != 0) 5092 return false; 5093 } 5094 5095 const MCInstrDesc &NewDesc = get(NewOpc); 5096 Inst.setDesc(NewDesc); 5097 5098 // Callers expect interator to be valid after this call, so modify the 5099 // instruction in place. 5100 if (OldVAddrIdx == NewVAddrIdx) { 5101 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx); 5102 // Clear use list from the old vaddr holding a zero register. 5103 MRI.removeRegOperandFromUseList(&NewVAddr); 5104 MRI.moveOperands(&NewVAddr, &SAddr, 1); 5105 Inst.RemoveOperand(OldSAddrIdx); 5106 // Update the use list with the pointer we have just moved from vaddr to 5107 // saddr poisition. Otherwise new vaddr will be missing from the use list. 5108 MRI.removeRegOperandFromUseList(&NewVAddr); 5109 MRI.addRegOperandToUseList(&NewVAddr); 5110 } else { 5111 assert(OldSAddrIdx == NewVAddrIdx); 5112 5113 if (OldVAddrIdx >= 0) { 5114 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, 5115 AMDGPU::OpName::vdst_in); 5116 5117 // RemoveOperand doesn't try to fixup tied operand indexes at it goes, so 5118 // it asserts. Untie the operands for now and retie them afterwards. 5119 if (NewVDstIn != -1) { 5120 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); 5121 Inst.untieRegOperand(OldVDstIn); 5122 } 5123 5124 Inst.RemoveOperand(OldVAddrIdx); 5125 5126 if (NewVDstIn != -1) { 5127 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 5128 Inst.tieOperands(NewVDst, NewVDstIn); 5129 } 5130 } 5131 } 5132 5133 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg())) 5134 VAddrDef->eraseFromParent(); 5135 5136 return true; 5137 } 5138 5139 // FIXME: Remove this when SelectionDAG is obsoleted. 5140 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 5141 MachineInstr &MI) const { 5142 if (!isSegmentSpecificFLAT(MI)) 5143 return; 5144 5145 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 5146 // thinks they are uniform, so a readfirstlane should be valid. 5147 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 5148 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 5149 return; 5150 5151 if (moveFlatAddrToVGPR(MI)) 5152 return; 5153 5154 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 5155 SAddr->setReg(ToSGPR); 5156 } 5157 5158 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 5159 MachineBasicBlock::iterator I, 5160 const TargetRegisterClass *DstRC, 5161 MachineOperand &Op, 5162 MachineRegisterInfo &MRI, 5163 const DebugLoc &DL) const { 5164 Register OpReg = Op.getReg(); 5165 unsigned OpSubReg = Op.getSubReg(); 5166 5167 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 5168 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 5169 5170 // Check if operand is already the correct register class. 5171 if (DstRC == OpRC) 5172 return; 5173 5174 Register DstReg = MRI.createVirtualRegister(DstRC); 5175 MachineInstr *Copy = 5176 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 5177 5178 Op.setReg(DstReg); 5179 Op.setSubReg(0); 5180 5181 MachineInstr *Def = MRI.getVRegDef(OpReg); 5182 if (!Def) 5183 return; 5184 5185 // Try to eliminate the copy if it is copying an immediate value. 5186 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 5187 FoldImmediate(*Copy, *Def, OpReg, &MRI); 5188 5189 bool ImpDef = Def->isImplicitDef(); 5190 while (!ImpDef && Def && Def->isCopy()) { 5191 if (Def->getOperand(1).getReg().isPhysical()) 5192 break; 5193 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 5194 ImpDef = Def && Def->isImplicitDef(); 5195 } 5196 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 5197 !ImpDef) 5198 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 5199 } 5200 5201 // Emit the actual waterfall loop, executing the wrapped instruction for each 5202 // unique value of \p Rsrc across all lanes. In the best case we execute 1 5203 // iteration, in the worst case we execute 64 (once per lane). 5204 static void 5205 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, 5206 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, 5207 const DebugLoc &DL, MachineOperand &Rsrc) { 5208 MachineFunction &MF = *OrigBB.getParent(); 5209 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5210 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5211 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5212 unsigned SaveExecOpc = 5213 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 5214 unsigned XorTermOpc = 5215 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 5216 unsigned AndOpc = 5217 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 5218 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5219 5220 MachineBasicBlock::iterator I = LoopBB.begin(); 5221 5222 SmallVector<Register, 8> ReadlanePieces; 5223 Register CondReg = AMDGPU::NoRegister; 5224 5225 Register VRsrc = Rsrc.getReg(); 5226 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef()); 5227 5228 unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI); 5229 unsigned NumSubRegs = RegSize / 32; 5230 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size"); 5231 5232 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 5233 5234 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5235 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5236 5237 // Read the next variant <- also loop target. 5238 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 5239 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx)); 5240 5241 // Read the next variant <- also loop target. 5242 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 5243 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1)); 5244 5245 ReadlanePieces.push_back(CurRegLo); 5246 ReadlanePieces.push_back(CurRegHi); 5247 5248 // Comparison is to be done as 64-bit. 5249 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 5250 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 5251 .addReg(CurRegLo) 5252 .addImm(AMDGPU::sub0) 5253 .addReg(CurRegHi) 5254 .addImm(AMDGPU::sub1); 5255 5256 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 5257 auto Cmp = 5258 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) 5259 .addReg(CurReg); 5260 if (NumSubRegs <= 2) 5261 Cmp.addReg(VRsrc); 5262 else 5263 Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2)); 5264 5265 // Combine the comparision results with AND. 5266 if (CondReg == AMDGPU::NoRegister) // First. 5267 CondReg = NewCondReg; 5268 else { // If not the first, we create an AND. 5269 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 5270 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 5271 .addReg(CondReg) 5272 .addReg(NewCondReg); 5273 CondReg = AndReg; 5274 } 5275 } // End for loop. 5276 5277 auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc)); 5278 Register SRsrc = MRI.createVirtualRegister(SRsrcRC); 5279 5280 // Build scalar Rsrc. 5281 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); 5282 unsigned Channel = 0; 5283 for (Register Piece : ReadlanePieces) { 5284 Merge.addReg(Piece) 5285 .addImm(TRI->getSubRegFromChannel(Channel++)); 5286 } 5287 5288 // Update Rsrc operand to use the SGPR Rsrc. 5289 Rsrc.setReg(SRsrc); 5290 Rsrc.setIsKill(true); 5291 5292 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5293 MRI.setSimpleHint(SaveExec, CondReg); 5294 5295 // Update EXEC to matching lanes, saving original to SaveExec. 5296 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 5297 .addReg(CondReg, RegState::Kill); 5298 5299 // The original instruction is here; we insert the terminators after it. 5300 I = LoopBB.end(); 5301 5302 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 5303 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec) 5304 .addReg(Exec) 5305 .addReg(SaveExec); 5306 5307 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::SI_WATERFALL_LOOP)).addMBB(&LoopBB); 5308 } 5309 5310 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register 5311 // with SGPRs by iterating over all unique values across all lanes. 5312 // Returns the loop basic block that now contains \p MI. 5313 static MachineBasicBlock * 5314 loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 5315 MachineOperand &Rsrc, MachineDominatorTree *MDT, 5316 MachineBasicBlock::iterator Begin = nullptr, 5317 MachineBasicBlock::iterator End = nullptr) { 5318 MachineBasicBlock &MBB = *MI.getParent(); 5319 MachineFunction &MF = *MBB.getParent(); 5320 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5321 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5322 MachineRegisterInfo &MRI = MF.getRegInfo(); 5323 if (!Begin.isValid()) 5324 Begin = &MI; 5325 if (!End.isValid()) { 5326 End = &MI; 5327 ++End; 5328 } 5329 const DebugLoc &DL = MI.getDebugLoc(); 5330 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5331 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5332 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5333 5334 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5335 5336 // Save the EXEC mask 5337 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 5338 5339 // Killed uses in the instruction we are waterfalling around will be 5340 // incorrect due to the added control-flow. 5341 MachineBasicBlock::iterator AfterMI = MI; 5342 ++AfterMI; 5343 for (auto I = Begin; I != AfterMI; I++) { 5344 for (auto &MO : I->uses()) { 5345 if (MO.isReg() && MO.isUse()) { 5346 MRI.clearKillFlags(MO.getReg()); 5347 } 5348 } 5349 } 5350 5351 // To insert the loop we need to split the block. Move everything after this 5352 // point to a new block, and insert a new empty block between the two. 5353 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 5354 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 5355 MachineFunction::iterator MBBI(MBB); 5356 ++MBBI; 5357 5358 MF.insert(MBBI, LoopBB); 5359 MF.insert(MBBI, RemainderBB); 5360 5361 LoopBB->addSuccessor(LoopBB); 5362 LoopBB->addSuccessor(RemainderBB); 5363 5364 // Move Begin to MI to the LoopBB, and the remainder of the block to 5365 // RemainderBB. 5366 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 5367 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 5368 LoopBB->splice(LoopBB->begin(), &MBB, Begin, MBB.end()); 5369 5370 MBB.addSuccessor(LoopBB); 5371 5372 // Update dominators. We know that MBB immediately dominates LoopBB, that 5373 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately 5374 // dominates all of the successors transferred to it from MBB that MBB used 5375 // to properly dominate. 5376 if (MDT) { 5377 MDT->addNewBlock(LoopBB, &MBB); 5378 MDT->addNewBlock(RemainderBB, LoopBB); 5379 for (auto &Succ : RemainderBB->successors()) { 5380 if (MDT->properlyDominates(&MBB, Succ)) { 5381 MDT->changeImmediateDominator(Succ, RemainderBB); 5382 } 5383 } 5384 } 5385 5386 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc); 5387 5388 // Restore the EXEC mask 5389 MachineBasicBlock::iterator First = RemainderBB->begin(); 5390 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 5391 return LoopBB; 5392 } 5393 5394 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 5395 static std::tuple<unsigned, unsigned> 5396 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 5397 MachineBasicBlock &MBB = *MI.getParent(); 5398 MachineFunction &MF = *MBB.getParent(); 5399 MachineRegisterInfo &MRI = MF.getRegInfo(); 5400 5401 // Extract the ptr from the resource descriptor. 5402 unsigned RsrcPtr = 5403 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 5404 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 5405 5406 // Create an empty resource descriptor 5407 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 5408 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5409 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5410 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 5411 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 5412 5413 // Zero64 = 0 5414 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 5415 .addImm(0); 5416 5417 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 5418 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 5419 .addImm(RsrcDataFormat & 0xFFFFFFFF); 5420 5421 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 5422 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 5423 .addImm(RsrcDataFormat >> 32); 5424 5425 // NewSRsrc = {Zero64, SRsrcFormat} 5426 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 5427 .addReg(Zero64) 5428 .addImm(AMDGPU::sub0_sub1) 5429 .addReg(SRsrcFormatLo) 5430 .addImm(AMDGPU::sub2) 5431 .addReg(SRsrcFormatHi) 5432 .addImm(AMDGPU::sub3); 5433 5434 return std::make_tuple(RsrcPtr, NewSRsrc); 5435 } 5436 5437 MachineBasicBlock * 5438 SIInstrInfo::legalizeOperands(MachineInstr &MI, 5439 MachineDominatorTree *MDT) const { 5440 MachineFunction &MF = *MI.getParent()->getParent(); 5441 MachineRegisterInfo &MRI = MF.getRegInfo(); 5442 MachineBasicBlock *CreatedBB = nullptr; 5443 5444 // Legalize VOP2 5445 if (isVOP2(MI) || isVOPC(MI)) { 5446 legalizeOperandsVOP2(MRI, MI); 5447 return CreatedBB; 5448 } 5449 5450 // Legalize VOP3 5451 if (isVOP3(MI)) { 5452 legalizeOperandsVOP3(MRI, MI); 5453 return CreatedBB; 5454 } 5455 5456 // Legalize SMRD 5457 if (isSMRD(MI)) { 5458 legalizeOperandsSMRD(MRI, MI); 5459 return CreatedBB; 5460 } 5461 5462 // Legalize FLAT 5463 if (isFLAT(MI)) { 5464 legalizeOperandsFLAT(MRI, MI); 5465 return CreatedBB; 5466 } 5467 5468 // Legalize REG_SEQUENCE and PHI 5469 // The register class of the operands much be the same type as the register 5470 // class of the output. 5471 if (MI.getOpcode() == AMDGPU::PHI) { 5472 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 5473 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 5474 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 5475 continue; 5476 const TargetRegisterClass *OpRC = 5477 MRI.getRegClass(MI.getOperand(i).getReg()); 5478 if (RI.hasVectorRegisters(OpRC)) { 5479 VRC = OpRC; 5480 } else { 5481 SRC = OpRC; 5482 } 5483 } 5484 5485 // If any of the operands are VGPR registers, then they all most be 5486 // otherwise we will create illegal VGPR->SGPR copies when legalizing 5487 // them. 5488 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 5489 if (!VRC) { 5490 assert(SRC); 5491 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 5492 VRC = &AMDGPU::VReg_1RegClass; 5493 } else 5494 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5495 ? RI.getEquivalentAGPRClass(SRC) 5496 : RI.getEquivalentVGPRClass(SRC); 5497 } else { 5498 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5499 ? RI.getEquivalentAGPRClass(VRC) 5500 : RI.getEquivalentVGPRClass(VRC); 5501 } 5502 RC = VRC; 5503 } else { 5504 RC = SRC; 5505 } 5506 5507 // Update all the operands so they have the same type. 5508 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5509 MachineOperand &Op = MI.getOperand(I); 5510 if (!Op.isReg() || !Op.getReg().isVirtual()) 5511 continue; 5512 5513 // MI is a PHI instruction. 5514 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 5515 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 5516 5517 // Avoid creating no-op copies with the same src and dst reg class. These 5518 // confuse some of the machine passes. 5519 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 5520 } 5521 } 5522 5523 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 5524 // VGPR dest type and SGPR sources, insert copies so all operands are 5525 // VGPRs. This seems to help operand folding / the register coalescer. 5526 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 5527 MachineBasicBlock *MBB = MI.getParent(); 5528 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 5529 if (RI.hasVGPRs(DstRC)) { 5530 // Update all the operands so they are VGPR register classes. These may 5531 // not be the same register class because REG_SEQUENCE supports mixing 5532 // subregister index types e.g. sub0_sub1 + sub2 + sub3 5533 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5534 MachineOperand &Op = MI.getOperand(I); 5535 if (!Op.isReg() || !Op.getReg().isVirtual()) 5536 continue; 5537 5538 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 5539 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 5540 if (VRC == OpRC) 5541 continue; 5542 5543 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 5544 Op.setIsKill(); 5545 } 5546 } 5547 5548 return CreatedBB; 5549 } 5550 5551 // Legalize INSERT_SUBREG 5552 // src0 must have the same register class as dst 5553 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 5554 Register Dst = MI.getOperand(0).getReg(); 5555 Register Src0 = MI.getOperand(1).getReg(); 5556 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 5557 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 5558 if (DstRC != Src0RC) { 5559 MachineBasicBlock *MBB = MI.getParent(); 5560 MachineOperand &Op = MI.getOperand(1); 5561 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 5562 } 5563 return CreatedBB; 5564 } 5565 5566 // Legalize SI_INIT_M0 5567 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 5568 MachineOperand &Src = MI.getOperand(0); 5569 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 5570 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 5571 return CreatedBB; 5572 } 5573 5574 // Legalize MIMG and MUBUF/MTBUF for shaders. 5575 // 5576 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 5577 // scratch memory access. In both cases, the legalization never involves 5578 // conversion to the addr64 form. 5579 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 5580 (isMUBUF(MI) || isMTBUF(MI)))) { 5581 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); 5582 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 5583 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT); 5584 5585 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); 5586 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 5587 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT); 5588 5589 return CreatedBB; 5590 } 5591 5592 // Legalize SI_CALL 5593 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 5594 MachineOperand *Dest = &MI.getOperand(0); 5595 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 5596 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 5597 // following copies, we also need to move copies from and to physical 5598 // registers into the loop block. 5599 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 5600 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 5601 5602 // Also move the copies to physical registers into the loop block 5603 MachineBasicBlock &MBB = *MI.getParent(); 5604 MachineBasicBlock::iterator Start(&MI); 5605 while (Start->getOpcode() != FrameSetupOpcode) 5606 --Start; 5607 MachineBasicBlock::iterator End(&MI); 5608 while (End->getOpcode() != FrameDestroyOpcode) 5609 ++End; 5610 // Also include following copies of the return value 5611 ++End; 5612 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 5613 MI.definesRegister(End->getOperand(1).getReg())) 5614 ++End; 5615 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End); 5616 } 5617 } 5618 5619 // Legalize MUBUF* instructions. 5620 int RsrcIdx = 5621 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 5622 if (RsrcIdx != -1) { 5623 // We have an MUBUF instruction 5624 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 5625 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass; 5626 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()), 5627 RI.getRegClass(RsrcRC))) { 5628 // The operands are legal. 5629 // FIXME: We may need to legalize operands besided srsrc. 5630 return CreatedBB; 5631 } 5632 5633 // Legalize a VGPR Rsrc. 5634 // 5635 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 5636 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 5637 // a zero-value SRsrc. 5638 // 5639 // If the instruction is _OFFSET (both idxen and offen disabled), and we 5640 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 5641 // above. 5642 // 5643 // Otherwise we are on non-ADDR64 hardware, and/or we have 5644 // idxen/offen/bothen and we fall back to a waterfall loop. 5645 5646 MachineBasicBlock &MBB = *MI.getParent(); 5647 5648 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 5649 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 5650 // This is already an ADDR64 instruction so we need to add the pointer 5651 // extracted from the resource descriptor to the current value of VAddr. 5652 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5653 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5654 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5655 5656 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5657 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 5658 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 5659 5660 unsigned RsrcPtr, NewSRsrc; 5661 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5662 5663 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 5664 const DebugLoc &DL = MI.getDebugLoc(); 5665 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 5666 .addDef(CondReg0) 5667 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5668 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 5669 .addImm(0); 5670 5671 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 5672 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 5673 .addDef(CondReg1, RegState::Dead) 5674 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5675 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 5676 .addReg(CondReg0, RegState::Kill) 5677 .addImm(0); 5678 5679 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5680 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 5681 .addReg(NewVAddrLo) 5682 .addImm(AMDGPU::sub0) 5683 .addReg(NewVAddrHi) 5684 .addImm(AMDGPU::sub1); 5685 5686 VAddr->setReg(NewVAddr); 5687 Rsrc->setReg(NewSRsrc); 5688 } else if (!VAddr && ST.hasAddr64()) { 5689 // This instructions is the _OFFSET variant, so we need to convert it to 5690 // ADDR64. 5691 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 5692 "FIXME: Need to emit flat atomics here"); 5693 5694 unsigned RsrcPtr, NewSRsrc; 5695 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5696 5697 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5698 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 5699 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 5700 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 5701 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 5702 5703 // Atomics rith return have have an additional tied operand and are 5704 // missing some of the special bits. 5705 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 5706 MachineInstr *Addr64; 5707 5708 if (!VDataIn) { 5709 // Regular buffer load / store. 5710 MachineInstrBuilder MIB = 5711 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5712 .add(*VData) 5713 .addReg(NewVAddr) 5714 .addReg(NewSRsrc) 5715 .add(*SOffset) 5716 .add(*Offset); 5717 5718 if (const MachineOperand *CPol = 5719 getNamedOperand(MI, AMDGPU::OpName::cpol)) { 5720 MIB.addImm(CPol->getImm()); 5721 } 5722 5723 if (const MachineOperand *TFE = 5724 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 5725 MIB.addImm(TFE->getImm()); 5726 } 5727 5728 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 5729 5730 MIB.cloneMemRefs(MI); 5731 Addr64 = MIB; 5732 } else { 5733 // Atomics with return. 5734 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5735 .add(*VData) 5736 .add(*VDataIn) 5737 .addReg(NewVAddr) 5738 .addReg(NewSRsrc) 5739 .add(*SOffset) 5740 .add(*Offset) 5741 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) 5742 .cloneMemRefs(MI); 5743 } 5744 5745 MI.removeFromParent(); 5746 5747 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5748 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 5749 NewVAddr) 5750 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5751 .addImm(AMDGPU::sub0) 5752 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5753 .addImm(AMDGPU::sub1); 5754 } else { 5755 // This is another variant; legalize Rsrc with waterfall loop from VGPRs 5756 // to SGPRs. 5757 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT); 5758 return CreatedBB; 5759 } 5760 } 5761 return CreatedBB; 5762 } 5763 5764 MachineBasicBlock *SIInstrInfo::moveToVALU(MachineInstr &TopInst, 5765 MachineDominatorTree *MDT) const { 5766 SetVectorType Worklist; 5767 Worklist.insert(&TopInst); 5768 MachineBasicBlock *CreatedBB = nullptr; 5769 MachineBasicBlock *CreatedBBTmp = nullptr; 5770 5771 while (!Worklist.empty()) { 5772 MachineInstr &Inst = *Worklist.pop_back_val(); 5773 MachineBasicBlock *MBB = Inst.getParent(); 5774 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 5775 5776 unsigned Opcode = Inst.getOpcode(); 5777 unsigned NewOpcode = getVALUOp(Inst); 5778 5779 // Handle some special cases 5780 switch (Opcode) { 5781 default: 5782 break; 5783 case AMDGPU::S_ADD_U64_PSEUDO: 5784 case AMDGPU::S_SUB_U64_PSEUDO: 5785 splitScalar64BitAddSub(Worklist, Inst, MDT); 5786 Inst.eraseFromParent(); 5787 continue; 5788 case AMDGPU::S_ADD_I32: 5789 case AMDGPU::S_SUB_I32: { 5790 // FIXME: The u32 versions currently selected use the carry. 5791 bool Changed; 5792 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 5793 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5794 CreatedBB = CreatedBBTmp; 5795 if (Changed) 5796 continue; 5797 5798 // Default handling 5799 break; 5800 } 5801 case AMDGPU::S_AND_B64: 5802 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 5803 Inst.eraseFromParent(); 5804 continue; 5805 5806 case AMDGPU::S_OR_B64: 5807 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 5808 Inst.eraseFromParent(); 5809 continue; 5810 5811 case AMDGPU::S_XOR_B64: 5812 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 5813 Inst.eraseFromParent(); 5814 continue; 5815 5816 case AMDGPU::S_NAND_B64: 5817 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 5818 Inst.eraseFromParent(); 5819 continue; 5820 5821 case AMDGPU::S_NOR_B64: 5822 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 5823 Inst.eraseFromParent(); 5824 continue; 5825 5826 case AMDGPU::S_XNOR_B64: 5827 if (ST.hasDLInsts()) 5828 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 5829 else 5830 splitScalar64BitXnor(Worklist, Inst, MDT); 5831 Inst.eraseFromParent(); 5832 continue; 5833 5834 case AMDGPU::S_ANDN2_B64: 5835 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 5836 Inst.eraseFromParent(); 5837 continue; 5838 5839 case AMDGPU::S_ORN2_B64: 5840 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 5841 Inst.eraseFromParent(); 5842 continue; 5843 5844 case AMDGPU::S_BREV_B64: 5845 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); 5846 Inst.eraseFromParent(); 5847 continue; 5848 5849 case AMDGPU::S_NOT_B64: 5850 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 5851 Inst.eraseFromParent(); 5852 continue; 5853 5854 case AMDGPU::S_BCNT1_I32_B64: 5855 splitScalar64BitBCNT(Worklist, Inst); 5856 Inst.eraseFromParent(); 5857 continue; 5858 5859 case AMDGPU::S_BFE_I64: 5860 splitScalar64BitBFE(Worklist, Inst); 5861 Inst.eraseFromParent(); 5862 continue; 5863 5864 case AMDGPU::S_LSHL_B32: 5865 if (ST.hasOnlyRevVALUShifts()) { 5866 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 5867 swapOperands(Inst); 5868 } 5869 break; 5870 case AMDGPU::S_ASHR_I32: 5871 if (ST.hasOnlyRevVALUShifts()) { 5872 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 5873 swapOperands(Inst); 5874 } 5875 break; 5876 case AMDGPU::S_LSHR_B32: 5877 if (ST.hasOnlyRevVALUShifts()) { 5878 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 5879 swapOperands(Inst); 5880 } 5881 break; 5882 case AMDGPU::S_LSHL_B64: 5883 if (ST.hasOnlyRevVALUShifts()) { 5884 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; 5885 swapOperands(Inst); 5886 } 5887 break; 5888 case AMDGPU::S_ASHR_I64: 5889 if (ST.hasOnlyRevVALUShifts()) { 5890 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 5891 swapOperands(Inst); 5892 } 5893 break; 5894 case AMDGPU::S_LSHR_B64: 5895 if (ST.hasOnlyRevVALUShifts()) { 5896 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 5897 swapOperands(Inst); 5898 } 5899 break; 5900 5901 case AMDGPU::S_ABS_I32: 5902 lowerScalarAbs(Worklist, Inst); 5903 Inst.eraseFromParent(); 5904 continue; 5905 5906 case AMDGPU::S_CBRANCH_SCC0: 5907 case AMDGPU::S_CBRANCH_SCC1: 5908 // Clear unused bits of vcc 5909 if (ST.isWave32()) 5910 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32), 5911 AMDGPU::VCC_LO) 5912 .addReg(AMDGPU::EXEC_LO) 5913 .addReg(AMDGPU::VCC_LO); 5914 else 5915 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), 5916 AMDGPU::VCC) 5917 .addReg(AMDGPU::EXEC) 5918 .addReg(AMDGPU::VCC); 5919 break; 5920 5921 case AMDGPU::S_BFE_U64: 5922 case AMDGPU::S_BFM_B64: 5923 llvm_unreachable("Moving this op to VALU not implemented"); 5924 5925 case AMDGPU::S_PACK_LL_B32_B16: 5926 case AMDGPU::S_PACK_LH_B32_B16: 5927 case AMDGPU::S_PACK_HH_B32_B16: 5928 movePackToVALU(Worklist, MRI, Inst); 5929 Inst.eraseFromParent(); 5930 continue; 5931 5932 case AMDGPU::S_XNOR_B32: 5933 lowerScalarXnor(Worklist, Inst); 5934 Inst.eraseFromParent(); 5935 continue; 5936 5937 case AMDGPU::S_NAND_B32: 5938 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 5939 Inst.eraseFromParent(); 5940 continue; 5941 5942 case AMDGPU::S_NOR_B32: 5943 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 5944 Inst.eraseFromParent(); 5945 continue; 5946 5947 case AMDGPU::S_ANDN2_B32: 5948 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 5949 Inst.eraseFromParent(); 5950 continue; 5951 5952 case AMDGPU::S_ORN2_B32: 5953 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 5954 Inst.eraseFromParent(); 5955 continue; 5956 5957 // TODO: remove as soon as everything is ready 5958 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 5959 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 5960 // can only be selected from the uniform SDNode. 5961 case AMDGPU::S_ADD_CO_PSEUDO: 5962 case AMDGPU::S_SUB_CO_PSEUDO: { 5963 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 5964 ? AMDGPU::V_ADDC_U32_e64 5965 : AMDGPU::V_SUBB_U32_e64; 5966 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5967 5968 Register CarryInReg = Inst.getOperand(4).getReg(); 5969 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 5970 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 5971 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 5972 .addReg(CarryInReg); 5973 } 5974 5975 Register CarryOutReg = Inst.getOperand(1).getReg(); 5976 5977 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 5978 MRI.getRegClass(Inst.getOperand(0).getReg()))); 5979 MachineInstr *CarryOp = 5980 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 5981 .addReg(CarryOutReg, RegState::Define) 5982 .add(Inst.getOperand(2)) 5983 .add(Inst.getOperand(3)) 5984 .addReg(CarryInReg) 5985 .addImm(0); 5986 CreatedBBTmp = legalizeOperands(*CarryOp); 5987 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5988 CreatedBB = CreatedBBTmp; 5989 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 5990 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 5991 Inst.eraseFromParent(); 5992 } 5993 continue; 5994 case AMDGPU::S_UADDO_PSEUDO: 5995 case AMDGPU::S_USUBO_PSEUDO: { 5996 const DebugLoc &DL = Inst.getDebugLoc(); 5997 MachineOperand &Dest0 = Inst.getOperand(0); 5998 MachineOperand &Dest1 = Inst.getOperand(1); 5999 MachineOperand &Src0 = Inst.getOperand(2); 6000 MachineOperand &Src1 = Inst.getOperand(3); 6001 6002 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 6003 ? AMDGPU::V_ADD_CO_U32_e64 6004 : AMDGPU::V_SUB_CO_U32_e64; 6005 const TargetRegisterClass *NewRC = 6006 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 6007 Register DestReg = MRI.createVirtualRegister(NewRC); 6008 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 6009 .addReg(Dest1.getReg(), RegState::Define) 6010 .add(Src0) 6011 .add(Src1) 6012 .addImm(0); // clamp bit 6013 6014 CreatedBBTmp = legalizeOperands(*NewInstr, MDT); 6015 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6016 CreatedBB = CreatedBBTmp; 6017 6018 MRI.replaceRegWith(Dest0.getReg(), DestReg); 6019 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 6020 Worklist); 6021 Inst.eraseFromParent(); 6022 } 6023 continue; 6024 6025 case AMDGPU::S_CSELECT_B32: 6026 case AMDGPU::S_CSELECT_B64: 6027 lowerSelect(Worklist, Inst, MDT); 6028 Inst.eraseFromParent(); 6029 continue; 6030 } 6031 6032 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 6033 // We cannot move this instruction to the VALU, so we should try to 6034 // legalize its operands instead. 6035 CreatedBBTmp = legalizeOperands(Inst, MDT); 6036 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6037 CreatedBB = CreatedBBTmp; 6038 continue; 6039 } 6040 6041 // Use the new VALU Opcode. 6042 const MCInstrDesc &NewDesc = get(NewOpcode); 6043 Inst.setDesc(NewDesc); 6044 6045 // Remove any references to SCC. Vector instructions can't read from it, and 6046 // We're just about to add the implicit use / defs of VCC, and we don't want 6047 // both. 6048 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) { 6049 MachineOperand &Op = Inst.getOperand(i); 6050 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { 6051 // Only propagate through live-def of SCC. 6052 if (Op.isDef() && !Op.isDead()) 6053 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 6054 if (Op.isUse()) 6055 addSCCDefsToVALUWorklist(Op, Worklist); 6056 Inst.RemoveOperand(i); 6057 } 6058 } 6059 6060 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 6061 // We are converting these to a BFE, so we need to add the missing 6062 // operands for the size and offset. 6063 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 6064 Inst.addOperand(MachineOperand::CreateImm(0)); 6065 Inst.addOperand(MachineOperand::CreateImm(Size)); 6066 6067 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 6068 // The VALU version adds the second operand to the result, so insert an 6069 // extra 0 operand. 6070 Inst.addOperand(MachineOperand::CreateImm(0)); 6071 } 6072 6073 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent()); 6074 fixImplicitOperands(Inst); 6075 6076 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 6077 const MachineOperand &OffsetWidthOp = Inst.getOperand(2); 6078 // If we need to move this to VGPRs, we need to unpack the second operand 6079 // back into the 2 separate ones for bit offset and width. 6080 assert(OffsetWidthOp.isImm() && 6081 "Scalar BFE is only implemented for constant width and offset"); 6082 uint32_t Imm = OffsetWidthOp.getImm(); 6083 6084 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6085 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6086 Inst.RemoveOperand(2); // Remove old immediate. 6087 Inst.addOperand(MachineOperand::CreateImm(Offset)); 6088 Inst.addOperand(MachineOperand::CreateImm(BitWidth)); 6089 } 6090 6091 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef(); 6092 unsigned NewDstReg = AMDGPU::NoRegister; 6093 if (HasDst) { 6094 Register DstReg = Inst.getOperand(0).getReg(); 6095 if (DstReg.isPhysical()) 6096 continue; 6097 6098 // Update the destination register class. 6099 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 6100 if (!NewDstRC) 6101 continue; 6102 6103 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 6104 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 6105 // Instead of creating a copy where src and dst are the same register 6106 // class, we just replace all uses of dst with src. These kinds of 6107 // copies interfere with the heuristics MachineSink uses to decide 6108 // whether or not to split a critical edge. Since the pass assumes 6109 // that copies will end up as machine instructions and not be 6110 // eliminated. 6111 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 6112 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 6113 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 6114 Inst.getOperand(0).setReg(DstReg); 6115 6116 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 6117 // these are deleted later, but at -O0 it would leave a suspicious 6118 // looking illegal copy of an undef register. 6119 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 6120 Inst.RemoveOperand(I); 6121 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 6122 continue; 6123 } 6124 6125 NewDstReg = MRI.createVirtualRegister(NewDstRC); 6126 MRI.replaceRegWith(DstReg, NewDstReg); 6127 } 6128 6129 // Legalize the operands 6130 CreatedBBTmp = legalizeOperands(Inst, MDT); 6131 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6132 CreatedBB = CreatedBBTmp; 6133 6134 if (HasDst) 6135 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 6136 } 6137 return CreatedBB; 6138 } 6139 6140 // Add/sub require special handling to deal with carry outs. 6141 std::pair<bool, MachineBasicBlock *> 6142 SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst, 6143 MachineDominatorTree *MDT) const { 6144 if (ST.hasAddNoCarry()) { 6145 // Assume there is no user of scc since we don't select this in that case. 6146 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 6147 // is used. 6148 6149 MachineBasicBlock &MBB = *Inst.getParent(); 6150 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6151 6152 Register OldDstReg = Inst.getOperand(0).getReg(); 6153 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6154 6155 unsigned Opc = Inst.getOpcode(); 6156 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 6157 6158 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 6159 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 6160 6161 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 6162 Inst.RemoveOperand(3); 6163 6164 Inst.setDesc(get(NewOpc)); 6165 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 6166 Inst.addImplicitDefUseOperands(*MBB.getParent()); 6167 MRI.replaceRegWith(OldDstReg, ResultReg); 6168 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 6169 6170 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6171 return std::make_pair(true, NewBB); 6172 } 6173 6174 return std::make_pair(false, nullptr); 6175 } 6176 6177 void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst, 6178 MachineDominatorTree *MDT) const { 6179 6180 MachineBasicBlock &MBB = *Inst.getParent(); 6181 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6182 MachineBasicBlock::iterator MII = Inst; 6183 DebugLoc DL = Inst.getDebugLoc(); 6184 6185 MachineOperand &Dest = Inst.getOperand(0); 6186 MachineOperand &Src0 = Inst.getOperand(1); 6187 MachineOperand &Src1 = Inst.getOperand(2); 6188 MachineOperand &Cond = Inst.getOperand(3); 6189 6190 Register SCCSource = Cond.getReg(); 6191 // Find SCC def, and if that is a copy (SCC = COPY reg) then use reg instead. 6192 if (!Cond.isUndef()) { 6193 for (MachineInstr &CandI : 6194 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 6195 Inst.getParent()->rend())) { 6196 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 6197 -1) { 6198 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 6199 SCCSource = CandI.getOperand(1).getReg(); 6200 } 6201 break; 6202 } 6203 } 6204 } 6205 6206 // If this is a trivial select where the condition is effectively not SCC 6207 // (SCCSource is a source of copy to SCC), then the select is semantically 6208 // equivalent to copying SCCSource. Hence, there is no need to create 6209 // V_CNDMASK, we can just use that and bail out. 6210 if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) && 6211 Src1.isImm() && (Src1.getImm() == 0)) { 6212 MRI.replaceRegWith(Dest.getReg(), SCCSource); 6213 return; 6214 } 6215 6216 const TargetRegisterClass *TC = ST.getWavefrontSize() == 64 6217 ? &AMDGPU::SReg_64_XEXECRegClass 6218 : &AMDGPU::SReg_32_XM0_XEXECRegClass; 6219 Register CopySCC = MRI.createVirtualRegister(TC); 6220 6221 if (SCCSource == AMDGPU::SCC) { 6222 // Insert a trivial select instead of creating a copy, because a copy from 6223 // SCC would semantically mean just copying a single bit, but we may need 6224 // the result to be a vector condition mask that needs preserving. 6225 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 6226 : AMDGPU::S_CSELECT_B32; 6227 auto NewSelect = 6228 BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0); 6229 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 6230 } else { 6231 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource); 6232 } 6233 6234 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6235 6236 auto UpdatedInst = 6237 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) 6238 .addImm(0) 6239 .add(Src1) // False 6240 .addImm(0) 6241 .add(Src0) // True 6242 .addReg(CopySCC); 6243 6244 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6245 legalizeOperands(*UpdatedInst, MDT); 6246 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6247 } 6248 6249 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist, 6250 MachineInstr &Inst) const { 6251 MachineBasicBlock &MBB = *Inst.getParent(); 6252 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6253 MachineBasicBlock::iterator MII = Inst; 6254 DebugLoc DL = Inst.getDebugLoc(); 6255 6256 MachineOperand &Dest = Inst.getOperand(0); 6257 MachineOperand &Src = Inst.getOperand(1); 6258 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6259 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6260 6261 unsigned SubOp = ST.hasAddNoCarry() ? 6262 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 6263 6264 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 6265 .addImm(0) 6266 .addReg(Src.getReg()); 6267 6268 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 6269 .addReg(Src.getReg()) 6270 .addReg(TmpReg); 6271 6272 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6273 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6274 } 6275 6276 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist, 6277 MachineInstr &Inst) const { 6278 MachineBasicBlock &MBB = *Inst.getParent(); 6279 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6280 MachineBasicBlock::iterator MII = Inst; 6281 const DebugLoc &DL = Inst.getDebugLoc(); 6282 6283 MachineOperand &Dest = Inst.getOperand(0); 6284 MachineOperand &Src0 = Inst.getOperand(1); 6285 MachineOperand &Src1 = Inst.getOperand(2); 6286 6287 if (ST.hasDLInsts()) { 6288 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6289 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 6290 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 6291 6292 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 6293 .add(Src0) 6294 .add(Src1); 6295 6296 MRI.replaceRegWith(Dest.getReg(), NewDest); 6297 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6298 } else { 6299 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 6300 // invert either source and then perform the XOR. If either source is a 6301 // scalar register, then we can leave the inversion on the scalar unit to 6302 // acheive a better distrubution of scalar and vector instructions. 6303 bool Src0IsSGPR = Src0.isReg() && 6304 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 6305 bool Src1IsSGPR = Src1.isReg() && 6306 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 6307 MachineInstr *Xor; 6308 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6309 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6310 6311 // Build a pair of scalar instructions and add them to the work list. 6312 // The next iteration over the work list will lower these to the vector 6313 // unit as necessary. 6314 if (Src0IsSGPR) { 6315 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 6316 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6317 .addReg(Temp) 6318 .add(Src1); 6319 } else if (Src1IsSGPR) { 6320 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 6321 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6322 .add(Src0) 6323 .addReg(Temp); 6324 } else { 6325 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 6326 .add(Src0) 6327 .add(Src1); 6328 MachineInstr *Not = 6329 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 6330 Worklist.insert(Not); 6331 } 6332 6333 MRI.replaceRegWith(Dest.getReg(), NewDest); 6334 6335 Worklist.insert(Xor); 6336 6337 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6338 } 6339 } 6340 6341 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist, 6342 MachineInstr &Inst, 6343 unsigned Opcode) const { 6344 MachineBasicBlock &MBB = *Inst.getParent(); 6345 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6346 MachineBasicBlock::iterator MII = Inst; 6347 const DebugLoc &DL = Inst.getDebugLoc(); 6348 6349 MachineOperand &Dest = Inst.getOperand(0); 6350 MachineOperand &Src0 = Inst.getOperand(1); 6351 MachineOperand &Src1 = Inst.getOperand(2); 6352 6353 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6354 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6355 6356 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 6357 .add(Src0) 6358 .add(Src1); 6359 6360 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 6361 .addReg(Interm); 6362 6363 Worklist.insert(&Op); 6364 Worklist.insert(&Not); 6365 6366 MRI.replaceRegWith(Dest.getReg(), NewDest); 6367 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6368 } 6369 6370 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist, 6371 MachineInstr &Inst, 6372 unsigned Opcode) const { 6373 MachineBasicBlock &MBB = *Inst.getParent(); 6374 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6375 MachineBasicBlock::iterator MII = Inst; 6376 const DebugLoc &DL = Inst.getDebugLoc(); 6377 6378 MachineOperand &Dest = Inst.getOperand(0); 6379 MachineOperand &Src0 = Inst.getOperand(1); 6380 MachineOperand &Src1 = Inst.getOperand(2); 6381 6382 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6383 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6384 6385 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 6386 .add(Src1); 6387 6388 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 6389 .add(Src0) 6390 .addReg(Interm); 6391 6392 Worklist.insert(&Not); 6393 Worklist.insert(&Op); 6394 6395 MRI.replaceRegWith(Dest.getReg(), NewDest); 6396 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6397 } 6398 6399 void SIInstrInfo::splitScalar64BitUnaryOp( 6400 SetVectorType &Worklist, MachineInstr &Inst, 6401 unsigned Opcode, bool Swap) const { 6402 MachineBasicBlock &MBB = *Inst.getParent(); 6403 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6404 6405 MachineOperand &Dest = Inst.getOperand(0); 6406 MachineOperand &Src0 = Inst.getOperand(1); 6407 DebugLoc DL = Inst.getDebugLoc(); 6408 6409 MachineBasicBlock::iterator MII = Inst; 6410 6411 const MCInstrDesc &InstDesc = get(Opcode); 6412 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6413 MRI.getRegClass(Src0.getReg()) : 6414 &AMDGPU::SGPR_32RegClass; 6415 6416 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6417 6418 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6419 AMDGPU::sub0, Src0SubRC); 6420 6421 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6422 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6423 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6424 6425 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6426 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 6427 6428 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6429 AMDGPU::sub1, Src0SubRC); 6430 6431 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6432 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 6433 6434 if (Swap) 6435 std::swap(DestSub0, DestSub1); 6436 6437 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6438 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6439 .addReg(DestSub0) 6440 .addImm(AMDGPU::sub0) 6441 .addReg(DestSub1) 6442 .addImm(AMDGPU::sub1); 6443 6444 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6445 6446 Worklist.insert(&LoHalf); 6447 Worklist.insert(&HiHalf); 6448 6449 // We don't need to legalizeOperands here because for a single operand, src0 6450 // will support any kind of input. 6451 6452 // Move all users of this moved value. 6453 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6454 } 6455 6456 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist, 6457 MachineInstr &Inst, 6458 MachineDominatorTree *MDT) const { 6459 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); 6460 6461 MachineBasicBlock &MBB = *Inst.getParent(); 6462 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6463 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6464 6465 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6466 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6467 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6468 6469 Register CarryReg = MRI.createVirtualRegister(CarryRC); 6470 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); 6471 6472 MachineOperand &Dest = Inst.getOperand(0); 6473 MachineOperand &Src0 = Inst.getOperand(1); 6474 MachineOperand &Src1 = Inst.getOperand(2); 6475 const DebugLoc &DL = Inst.getDebugLoc(); 6476 MachineBasicBlock::iterator MII = Inst; 6477 6478 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 6479 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 6480 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6481 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6482 6483 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6484 AMDGPU::sub0, Src0SubRC); 6485 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6486 AMDGPU::sub0, Src1SubRC); 6487 6488 6489 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6490 AMDGPU::sub1, Src0SubRC); 6491 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6492 AMDGPU::sub1, Src1SubRC); 6493 6494 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 6495 MachineInstr *LoHalf = 6496 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0) 6497 .addReg(CarryReg, RegState::Define) 6498 .add(SrcReg0Sub0) 6499 .add(SrcReg1Sub0) 6500 .addImm(0); // clamp bit 6501 6502 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 6503 MachineInstr *HiHalf = 6504 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1) 6505 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 6506 .add(SrcReg0Sub1) 6507 .add(SrcReg1Sub1) 6508 .addReg(CarryReg, RegState::Kill) 6509 .addImm(0); // clamp bit 6510 6511 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6512 .addReg(DestSub0) 6513 .addImm(AMDGPU::sub0) 6514 .addReg(DestSub1) 6515 .addImm(AMDGPU::sub1); 6516 6517 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6518 6519 // Try to legalize the operands in case we need to swap the order to keep it 6520 // valid. 6521 legalizeOperands(*LoHalf, MDT); 6522 legalizeOperands(*HiHalf, MDT); 6523 6524 // Move all users of this moved vlaue. 6525 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6526 } 6527 6528 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist, 6529 MachineInstr &Inst, unsigned Opcode, 6530 MachineDominatorTree *MDT) const { 6531 MachineBasicBlock &MBB = *Inst.getParent(); 6532 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6533 6534 MachineOperand &Dest = Inst.getOperand(0); 6535 MachineOperand &Src0 = Inst.getOperand(1); 6536 MachineOperand &Src1 = Inst.getOperand(2); 6537 DebugLoc DL = Inst.getDebugLoc(); 6538 6539 MachineBasicBlock::iterator MII = Inst; 6540 6541 const MCInstrDesc &InstDesc = get(Opcode); 6542 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6543 MRI.getRegClass(Src0.getReg()) : 6544 &AMDGPU::SGPR_32RegClass; 6545 6546 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6547 const TargetRegisterClass *Src1RC = Src1.isReg() ? 6548 MRI.getRegClass(Src1.getReg()) : 6549 &AMDGPU::SGPR_32RegClass; 6550 6551 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6552 6553 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6554 AMDGPU::sub0, Src0SubRC); 6555 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6556 AMDGPU::sub0, Src1SubRC); 6557 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6558 AMDGPU::sub1, Src0SubRC); 6559 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6560 AMDGPU::sub1, Src1SubRC); 6561 6562 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6563 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6564 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6565 6566 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6567 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 6568 .add(SrcReg0Sub0) 6569 .add(SrcReg1Sub0); 6570 6571 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6572 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 6573 .add(SrcReg0Sub1) 6574 .add(SrcReg1Sub1); 6575 6576 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6577 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6578 .addReg(DestSub0) 6579 .addImm(AMDGPU::sub0) 6580 .addReg(DestSub1) 6581 .addImm(AMDGPU::sub1); 6582 6583 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6584 6585 Worklist.insert(&LoHalf); 6586 Worklist.insert(&HiHalf); 6587 6588 // Move all users of this moved vlaue. 6589 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6590 } 6591 6592 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist, 6593 MachineInstr &Inst, 6594 MachineDominatorTree *MDT) const { 6595 MachineBasicBlock &MBB = *Inst.getParent(); 6596 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6597 6598 MachineOperand &Dest = Inst.getOperand(0); 6599 MachineOperand &Src0 = Inst.getOperand(1); 6600 MachineOperand &Src1 = Inst.getOperand(2); 6601 const DebugLoc &DL = Inst.getDebugLoc(); 6602 6603 MachineBasicBlock::iterator MII = Inst; 6604 6605 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6606 6607 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 6608 6609 MachineOperand* Op0; 6610 MachineOperand* Op1; 6611 6612 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 6613 Op0 = &Src0; 6614 Op1 = &Src1; 6615 } else { 6616 Op0 = &Src1; 6617 Op1 = &Src0; 6618 } 6619 6620 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 6621 .add(*Op0); 6622 6623 Register NewDest = MRI.createVirtualRegister(DestRC); 6624 6625 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 6626 .addReg(Interm) 6627 .add(*Op1); 6628 6629 MRI.replaceRegWith(Dest.getReg(), NewDest); 6630 6631 Worklist.insert(&Xor); 6632 } 6633 6634 void SIInstrInfo::splitScalar64BitBCNT( 6635 SetVectorType &Worklist, MachineInstr &Inst) const { 6636 MachineBasicBlock &MBB = *Inst.getParent(); 6637 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6638 6639 MachineBasicBlock::iterator MII = Inst; 6640 const DebugLoc &DL = Inst.getDebugLoc(); 6641 6642 MachineOperand &Dest = Inst.getOperand(0); 6643 MachineOperand &Src = Inst.getOperand(1); 6644 6645 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 6646 const TargetRegisterClass *SrcRC = Src.isReg() ? 6647 MRI.getRegClass(Src.getReg()) : 6648 &AMDGPU::SGPR_32RegClass; 6649 6650 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6651 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6652 6653 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); 6654 6655 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6656 AMDGPU::sub0, SrcSubRC); 6657 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6658 AMDGPU::sub1, SrcSubRC); 6659 6660 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 6661 6662 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 6663 6664 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6665 6666 // We don't need to legalize operands here. src0 for etiher instruction can be 6667 // an SGPR, and the second input is unused or determined here. 6668 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6669 } 6670 6671 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist, 6672 MachineInstr &Inst) const { 6673 MachineBasicBlock &MBB = *Inst.getParent(); 6674 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6675 MachineBasicBlock::iterator MII = Inst; 6676 const DebugLoc &DL = Inst.getDebugLoc(); 6677 6678 MachineOperand &Dest = Inst.getOperand(0); 6679 uint32_t Imm = Inst.getOperand(2).getImm(); 6680 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6681 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6682 6683 (void) Offset; 6684 6685 // Only sext_inreg cases handled. 6686 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 6687 Offset == 0 && "Not implemented"); 6688 6689 if (BitWidth < 32) { 6690 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6691 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6692 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6693 6694 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 6695 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 6696 .addImm(0) 6697 .addImm(BitWidth); 6698 6699 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 6700 .addImm(31) 6701 .addReg(MidRegLo); 6702 6703 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6704 .addReg(MidRegLo) 6705 .addImm(AMDGPU::sub0) 6706 .addReg(MidRegHi) 6707 .addImm(AMDGPU::sub1); 6708 6709 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6710 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6711 return; 6712 } 6713 6714 MachineOperand &Src = Inst.getOperand(1); 6715 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6716 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6717 6718 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 6719 .addImm(31) 6720 .addReg(Src.getReg(), 0, AMDGPU::sub0); 6721 6722 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6723 .addReg(Src.getReg(), 0, AMDGPU::sub0) 6724 .addImm(AMDGPU::sub0) 6725 .addReg(TmpReg) 6726 .addImm(AMDGPU::sub1); 6727 6728 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6729 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6730 } 6731 6732 void SIInstrInfo::addUsersToMoveToVALUWorklist( 6733 Register DstReg, 6734 MachineRegisterInfo &MRI, 6735 SetVectorType &Worklist) const { 6736 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 6737 E = MRI.use_end(); I != E;) { 6738 MachineInstr &UseMI = *I->getParent(); 6739 6740 unsigned OpNo = 0; 6741 6742 switch (UseMI.getOpcode()) { 6743 case AMDGPU::COPY: 6744 case AMDGPU::WQM: 6745 case AMDGPU::SOFT_WQM: 6746 case AMDGPU::STRICT_WWM: 6747 case AMDGPU::STRICT_WQM: 6748 case AMDGPU::REG_SEQUENCE: 6749 case AMDGPU::PHI: 6750 case AMDGPU::INSERT_SUBREG: 6751 break; 6752 default: 6753 OpNo = I.getOperandNo(); 6754 break; 6755 } 6756 6757 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 6758 Worklist.insert(&UseMI); 6759 6760 do { 6761 ++I; 6762 } while (I != E && I->getParent() == &UseMI); 6763 } else { 6764 ++I; 6765 } 6766 } 6767 } 6768 6769 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist, 6770 MachineRegisterInfo &MRI, 6771 MachineInstr &Inst) const { 6772 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6773 MachineBasicBlock *MBB = Inst.getParent(); 6774 MachineOperand &Src0 = Inst.getOperand(1); 6775 MachineOperand &Src1 = Inst.getOperand(2); 6776 const DebugLoc &DL = Inst.getDebugLoc(); 6777 6778 switch (Inst.getOpcode()) { 6779 case AMDGPU::S_PACK_LL_B32_B16: { 6780 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6781 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6782 6783 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 6784 // 0. 6785 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6786 .addImm(0xffff); 6787 6788 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 6789 .addReg(ImmReg, RegState::Kill) 6790 .add(Src0); 6791 6792 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 6793 .add(Src1) 6794 .addImm(16) 6795 .addReg(TmpReg, RegState::Kill); 6796 break; 6797 } 6798 case AMDGPU::S_PACK_LH_B32_B16: { 6799 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6800 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6801 .addImm(0xffff); 6802 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 6803 .addReg(ImmReg, RegState::Kill) 6804 .add(Src0) 6805 .add(Src1); 6806 break; 6807 } 6808 case AMDGPU::S_PACK_HH_B32_B16: { 6809 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6810 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6811 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 6812 .addImm(16) 6813 .add(Src0); 6814 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6815 .addImm(0xffff0000); 6816 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 6817 .add(Src1) 6818 .addReg(ImmReg, RegState::Kill) 6819 .addReg(TmpReg, RegState::Kill); 6820 break; 6821 } 6822 default: 6823 llvm_unreachable("unhandled s_pack_* instruction"); 6824 } 6825 6826 MachineOperand &Dest = Inst.getOperand(0); 6827 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6828 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6829 } 6830 6831 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 6832 MachineInstr &SCCDefInst, 6833 SetVectorType &Worklist) const { 6834 bool SCCUsedImplicitly = false; 6835 6836 // Ensure that def inst defines SCC, which is still live. 6837 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 6838 !Op.isDead() && Op.getParent() == &SCCDefInst); 6839 SmallVector<MachineInstr *, 4> CopyToDelete; 6840 // This assumes that all the users of SCC are in the same block 6841 // as the SCC def. 6842 for (MachineInstr &MI : // Skip the def inst itself. 6843 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 6844 SCCDefInst.getParent()->end())) { 6845 // Check if SCC is used first. 6846 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) { 6847 if (MI.isCopy()) { 6848 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6849 Register DestReg = MI.getOperand(0).getReg(); 6850 6851 for (auto &User : MRI.use_nodbg_instructions(DestReg)) { 6852 if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) || 6853 (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) { 6854 User.getOperand(4).setReg(RI.getVCC()); 6855 Worklist.insert(&User); 6856 } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) { 6857 User.getOperand(5).setReg(RI.getVCC()); 6858 // No need to add to Worklist. 6859 } 6860 } 6861 CopyToDelete.push_back(&MI); 6862 } else { 6863 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 || 6864 MI.getOpcode() == AMDGPU::S_CSELECT_B64) { 6865 // This is an implicit use of SCC and it is really expected by 6866 // the SCC users to handle. 6867 // We cannot preserve the edge to the user so add the explicit 6868 // copy: SCC = COPY VCC. 6869 // The copy will be cleaned up during the processing of the user 6870 // in lowerSelect. 6871 SCCUsedImplicitly = true; 6872 } 6873 6874 Worklist.insert(&MI); 6875 } 6876 } 6877 // Exit if we find another SCC def. 6878 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 6879 break; 6880 } 6881 for (auto &Copy : CopyToDelete) 6882 Copy->eraseFromParent(); 6883 6884 if (SCCUsedImplicitly) { 6885 BuildMI(*SCCDefInst.getParent(), std::next(SCCDefInst.getIterator()), 6886 SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC) 6887 .addReg(RI.getVCC()); 6888 } 6889 } 6890 6891 // Instructions that use SCC may be converted to VALU instructions. When that 6892 // happens, the SCC register is changed to VCC_LO. The instruction that defines 6893 // SCC must be changed to an instruction that defines VCC. This function makes 6894 // sure that the instruction that defines SCC is added to the moveToVALU 6895 // worklist. 6896 void SIInstrInfo::addSCCDefsToVALUWorklist(MachineOperand &Op, 6897 SetVectorType &Worklist) const { 6898 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isUse()); 6899 6900 MachineInstr *SCCUseInst = Op.getParent(); 6901 // Look for a preceeding instruction that either defines VCC or SCC. If VCC 6902 // then there is nothing to do because the defining instruction has been 6903 // converted to a VALU already. If SCC then that instruction needs to be 6904 // converted to a VALU. 6905 for (MachineInstr &MI : 6906 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)), 6907 SCCUseInst->getParent()->rend())) { 6908 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) 6909 break; 6910 if (MI.definesRegister(AMDGPU::SCC, &RI)) { 6911 Worklist.insert(&MI); 6912 break; 6913 } 6914 } 6915 } 6916 6917 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 6918 const MachineInstr &Inst) const { 6919 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 6920 6921 switch (Inst.getOpcode()) { 6922 // For target instructions, getOpRegClass just returns the virtual register 6923 // class associated with the operand, so we need to find an equivalent VGPR 6924 // register class in order to move the instruction to the VALU. 6925 case AMDGPU::COPY: 6926 case AMDGPU::PHI: 6927 case AMDGPU::REG_SEQUENCE: 6928 case AMDGPU::INSERT_SUBREG: 6929 case AMDGPU::WQM: 6930 case AMDGPU::SOFT_WQM: 6931 case AMDGPU::STRICT_WWM: 6932 case AMDGPU::STRICT_WQM: { 6933 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 6934 if (RI.hasAGPRs(SrcRC)) { 6935 if (RI.hasAGPRs(NewDstRC)) 6936 return nullptr; 6937 6938 switch (Inst.getOpcode()) { 6939 case AMDGPU::PHI: 6940 case AMDGPU::REG_SEQUENCE: 6941 case AMDGPU::INSERT_SUBREG: 6942 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 6943 break; 6944 default: 6945 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6946 } 6947 6948 if (!NewDstRC) 6949 return nullptr; 6950 } else { 6951 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 6952 return nullptr; 6953 6954 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6955 if (!NewDstRC) 6956 return nullptr; 6957 } 6958 6959 return NewDstRC; 6960 } 6961 default: 6962 return NewDstRC; 6963 } 6964 } 6965 6966 // Find the one SGPR operand we are allowed to use. 6967 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 6968 int OpIndices[3]) const { 6969 const MCInstrDesc &Desc = MI.getDesc(); 6970 6971 // Find the one SGPR operand we are allowed to use. 6972 // 6973 // First we need to consider the instruction's operand requirements before 6974 // legalizing. Some operands are required to be SGPRs, such as implicit uses 6975 // of VCC, but we are still bound by the constant bus requirement to only use 6976 // one. 6977 // 6978 // If the operand's class is an SGPR, we can never move it. 6979 6980 Register SGPRReg = findImplicitSGPRRead(MI); 6981 if (SGPRReg != AMDGPU::NoRegister) 6982 return SGPRReg; 6983 6984 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; 6985 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6986 6987 for (unsigned i = 0; i < 3; ++i) { 6988 int Idx = OpIndices[i]; 6989 if (Idx == -1) 6990 break; 6991 6992 const MachineOperand &MO = MI.getOperand(Idx); 6993 if (!MO.isReg()) 6994 continue; 6995 6996 // Is this operand statically required to be an SGPR based on the operand 6997 // constraints? 6998 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass); 6999 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 7000 if (IsRequiredSGPR) 7001 return MO.getReg(); 7002 7003 // If this could be a VGPR or an SGPR, Check the dynamic register class. 7004 Register Reg = MO.getReg(); 7005 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 7006 if (RI.isSGPRClass(RegRC)) 7007 UsedSGPRs[i] = Reg; 7008 } 7009 7010 // We don't have a required SGPR operand, so we have a bit more freedom in 7011 // selecting operands to move. 7012 7013 // Try to select the most used SGPR. If an SGPR is equal to one of the 7014 // others, we choose that. 7015 // 7016 // e.g. 7017 // V_FMA_F32 v0, s0, s0, s0 -> No moves 7018 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 7019 7020 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 7021 // prefer those. 7022 7023 if (UsedSGPRs[0] != AMDGPU::NoRegister) { 7024 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 7025 SGPRReg = UsedSGPRs[0]; 7026 } 7027 7028 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { 7029 if (UsedSGPRs[1] == UsedSGPRs[2]) 7030 SGPRReg = UsedSGPRs[1]; 7031 } 7032 7033 return SGPRReg; 7034 } 7035 7036 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 7037 unsigned OperandName) const { 7038 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 7039 if (Idx == -1) 7040 return nullptr; 7041 7042 return &MI.getOperand(Idx); 7043 } 7044 7045 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 7046 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 7047 return (AMDGPU::MTBUFFormat::UFMT_32_FLOAT << 44) | 7048 (1ULL << 56) | // RESOURCE_LEVEL = 1 7049 (3ULL << 60); // OOB_SELECT = 3 7050 } 7051 7052 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 7053 if (ST.isAmdHsaOS()) { 7054 // Set ATC = 1. GFX9 doesn't have this bit. 7055 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 7056 RsrcDataFormat |= (1ULL << 56); 7057 7058 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 7059 // BTW, it disables TC L2 and therefore decreases performance. 7060 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 7061 RsrcDataFormat |= (2ULL << 59); 7062 } 7063 7064 return RsrcDataFormat; 7065 } 7066 7067 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 7068 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 7069 AMDGPU::RSRC_TID_ENABLE | 7070 0xffffffff; // Size; 7071 7072 // GFX9 doesn't have ELEMENT_SIZE. 7073 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 7074 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 7075 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 7076 } 7077 7078 // IndexStride = 64 / 32. 7079 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 7080 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 7081 7082 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 7083 // Clear them unless we want a huge stride. 7084 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 7085 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 7086 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 7087 7088 return Rsrc23; 7089 } 7090 7091 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 7092 unsigned Opc = MI.getOpcode(); 7093 7094 return isSMRD(Opc); 7095 } 7096 7097 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 7098 return get(Opc).mayLoad() && 7099 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 7100 } 7101 7102 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 7103 int &FrameIndex) const { 7104 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 7105 if (!Addr || !Addr->isFI()) 7106 return AMDGPU::NoRegister; 7107 7108 assert(!MI.memoperands_empty() && 7109 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 7110 7111 FrameIndex = Addr->getIndex(); 7112 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 7113 } 7114 7115 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 7116 int &FrameIndex) const { 7117 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 7118 assert(Addr && Addr->isFI()); 7119 FrameIndex = Addr->getIndex(); 7120 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 7121 } 7122 7123 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 7124 int &FrameIndex) const { 7125 if (!MI.mayLoad()) 7126 return AMDGPU::NoRegister; 7127 7128 if (isMUBUF(MI) || isVGPRSpill(MI)) 7129 return isStackAccess(MI, FrameIndex); 7130 7131 if (isSGPRSpill(MI)) 7132 return isSGPRStackAccess(MI, FrameIndex); 7133 7134 return AMDGPU::NoRegister; 7135 } 7136 7137 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 7138 int &FrameIndex) const { 7139 if (!MI.mayStore()) 7140 return AMDGPU::NoRegister; 7141 7142 if (isMUBUF(MI) || isVGPRSpill(MI)) 7143 return isStackAccess(MI, FrameIndex); 7144 7145 if (isSGPRSpill(MI)) 7146 return isSGPRStackAccess(MI, FrameIndex); 7147 7148 return AMDGPU::NoRegister; 7149 } 7150 7151 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 7152 unsigned Size = 0; 7153 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 7154 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 7155 while (++I != E && I->isInsideBundle()) { 7156 assert(!I->isBundle() && "No nested bundle!"); 7157 Size += getInstSizeInBytes(*I); 7158 } 7159 7160 return Size; 7161 } 7162 7163 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 7164 unsigned Opc = MI.getOpcode(); 7165 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 7166 unsigned DescSize = Desc.getSize(); 7167 7168 // If we have a definitive size, we can use it. Otherwise we need to inspect 7169 // the operands to know the size. 7170 if (isFixedSize(MI)) { 7171 unsigned Size = DescSize; 7172 7173 // If we hit the buggy offset, an extra nop will be inserted in MC so 7174 // estimate the worst case. 7175 if (MI.isBranch() && ST.hasOffset3fBug()) 7176 Size += 4; 7177 7178 return Size; 7179 } 7180 7181 // 4-byte instructions may have a 32-bit literal encoded after them. Check 7182 // operands that coud ever be literals. 7183 if (isVALU(MI) || isSALU(MI)) { 7184 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 7185 if (Src0Idx == -1) 7186 return DescSize; // No operands. 7187 7188 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx])) 7189 return isVOP3(MI) ? 12 : (DescSize + 4); 7190 7191 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 7192 if (Src1Idx == -1) 7193 return DescSize; 7194 7195 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx])) 7196 return isVOP3(MI) ? 12 : (DescSize + 4); 7197 7198 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 7199 if (Src2Idx == -1) 7200 return DescSize; 7201 7202 if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx])) 7203 return isVOP3(MI) ? 12 : (DescSize + 4); 7204 7205 return DescSize; 7206 } 7207 7208 // Check whether we have extra NSA words. 7209 if (isMIMG(MI)) { 7210 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 7211 if (VAddr0Idx < 0) 7212 return 8; 7213 7214 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 7215 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 7216 } 7217 7218 switch (Opc) { 7219 case TargetOpcode::BUNDLE: 7220 return getInstBundleSize(MI); 7221 case TargetOpcode::INLINEASM: 7222 case TargetOpcode::INLINEASM_BR: { 7223 const MachineFunction *MF = MI.getParent()->getParent(); 7224 const char *AsmStr = MI.getOperand(0).getSymbolName(); 7225 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 7226 } 7227 default: 7228 if (MI.isMetaInstruction()) 7229 return 0; 7230 return DescSize; 7231 } 7232 } 7233 7234 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 7235 if (!isFLAT(MI)) 7236 return false; 7237 7238 if (MI.memoperands_empty()) 7239 return true; 7240 7241 for (const MachineMemOperand *MMO : MI.memoperands()) { 7242 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 7243 return true; 7244 } 7245 return false; 7246 } 7247 7248 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 7249 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 7250 } 7251 7252 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 7253 MachineBasicBlock *IfEnd) const { 7254 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 7255 assert(TI != IfEntry->end()); 7256 7257 MachineInstr *Branch = &(*TI); 7258 MachineFunction *MF = IfEntry->getParent(); 7259 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 7260 7261 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7262 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7263 MachineInstr *SIIF = 7264 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 7265 .add(Branch->getOperand(0)) 7266 .add(Branch->getOperand(1)); 7267 MachineInstr *SIEND = 7268 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 7269 .addReg(DstReg); 7270 7271 IfEntry->erase(TI); 7272 IfEntry->insert(IfEntry->end(), SIIF); 7273 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 7274 } 7275 } 7276 7277 void SIInstrInfo::convertNonUniformLoopRegion( 7278 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 7279 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 7280 // We expect 2 terminators, one conditional and one unconditional. 7281 assert(TI != LoopEnd->end()); 7282 7283 MachineInstr *Branch = &(*TI); 7284 MachineFunction *MF = LoopEnd->getParent(); 7285 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 7286 7287 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7288 7289 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7290 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 7291 MachineInstrBuilder HeaderPHIBuilder = 7292 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 7293 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(), 7294 E = LoopEntry->pred_end(); 7295 PI != E; ++PI) { 7296 if (*PI == LoopEnd) { 7297 HeaderPHIBuilder.addReg(BackEdgeReg); 7298 } else { 7299 MachineBasicBlock *PMBB = *PI; 7300 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 7301 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 7302 ZeroReg, 0); 7303 HeaderPHIBuilder.addReg(ZeroReg); 7304 } 7305 HeaderPHIBuilder.addMBB(*PI); 7306 } 7307 MachineInstr *HeaderPhi = HeaderPHIBuilder; 7308 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 7309 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 7310 .addReg(DstReg) 7311 .add(Branch->getOperand(0)); 7312 MachineInstr *SILOOP = 7313 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 7314 .addReg(BackEdgeReg) 7315 .addMBB(LoopEntry); 7316 7317 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 7318 LoopEnd->erase(TI); 7319 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 7320 LoopEnd->insert(LoopEnd->end(), SILOOP); 7321 } 7322 } 7323 7324 ArrayRef<std::pair<int, const char *>> 7325 SIInstrInfo::getSerializableTargetIndices() const { 7326 static const std::pair<int, const char *> TargetIndices[] = { 7327 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 7328 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 7329 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 7330 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 7331 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 7332 return makeArrayRef(TargetIndices); 7333 } 7334 7335 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 7336 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 7337 ScheduleHazardRecognizer * 7338 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 7339 const ScheduleDAG *DAG) const { 7340 return new GCNHazardRecognizer(DAG->MF); 7341 } 7342 7343 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 7344 /// pass. 7345 ScheduleHazardRecognizer * 7346 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 7347 return new GCNHazardRecognizer(MF); 7348 } 7349 7350 std::pair<unsigned, unsigned> 7351 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 7352 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK); 7353 } 7354 7355 ArrayRef<std::pair<unsigned, const char *>> 7356 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 7357 static const std::pair<unsigned, const char *> TargetFlags[] = { 7358 { MO_GOTPCREL, "amdgpu-gotprel" }, 7359 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 7360 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 7361 { MO_REL32_LO, "amdgpu-rel32-lo" }, 7362 { MO_REL32_HI, "amdgpu-rel32-hi" }, 7363 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 7364 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 7365 }; 7366 7367 return makeArrayRef(TargetFlags); 7368 } 7369 7370 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const { 7371 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && 7372 MI.modifiesRegister(AMDGPU::EXEC, &RI); 7373 } 7374 7375 MachineInstrBuilder 7376 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7377 MachineBasicBlock::iterator I, 7378 const DebugLoc &DL, 7379 Register DestReg) const { 7380 if (ST.hasAddNoCarry()) 7381 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 7382 7383 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7384 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 7385 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 7386 7387 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7388 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7389 } 7390 7391 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7392 MachineBasicBlock::iterator I, 7393 const DebugLoc &DL, 7394 Register DestReg, 7395 RegScavenger &RS) const { 7396 if (ST.hasAddNoCarry()) 7397 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 7398 7399 // If available, prefer to use vcc. 7400 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 7401 ? Register(RI.getVCC()) 7402 : RS.scavengeRegister(RI.getBoolRC(), I, 0, false); 7403 7404 // TODO: Users need to deal with this. 7405 if (!UnusedCarry.isValid()) 7406 return MachineInstrBuilder(); 7407 7408 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7409 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7410 } 7411 7412 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 7413 switch (Opcode) { 7414 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 7415 case AMDGPU::SI_KILL_I1_TERMINATOR: 7416 return true; 7417 default: 7418 return false; 7419 } 7420 } 7421 7422 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 7423 switch (Opcode) { 7424 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 7425 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 7426 case AMDGPU::SI_KILL_I1_PSEUDO: 7427 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 7428 default: 7429 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 7430 } 7431 } 7432 7433 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 7434 if (!ST.isWave32()) 7435 return; 7436 7437 for (auto &Op : MI.implicit_operands()) { 7438 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 7439 Op.setReg(AMDGPU::VCC_LO); 7440 } 7441 } 7442 7443 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 7444 if (!isSMRD(MI)) 7445 return false; 7446 7447 // Check that it is using a buffer resource. 7448 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 7449 if (Idx == -1) // e.g. s_memtime 7450 return false; 7451 7452 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass; 7453 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 7454 } 7455 7456 // Depending on the used address space and instructions, some immediate offsets 7457 // are allowed and some are not. 7458 // In general, flat instruction offsets can only be non-negative, global and 7459 // scratch instruction offsets can also be negative. 7460 // 7461 // There are several bugs related to these offsets: 7462 // On gfx10.1, flat instructions that go into the global address space cannot 7463 // use an offset. 7464 // 7465 // For scratch instructions, the address can be either an SGPR or a VGPR. 7466 // The following offsets can be used, depending on the architecture (x means 7467 // cannot be used): 7468 // +----------------------------+------+------+ 7469 // | Address-Mode | SGPR | VGPR | 7470 // +----------------------------+------+------+ 7471 // | gfx9 | | | 7472 // | negative, 4-aligned offset | x | ok | 7473 // | negative, unaligned offset | x | ok | 7474 // +----------------------------+------+------+ 7475 // | gfx10 | | | 7476 // | negative, 4-aligned offset | ok | ok | 7477 // | negative, unaligned offset | ok | x | 7478 // +----------------------------+------+------+ 7479 // | gfx10.3 | | | 7480 // | negative, 4-aligned offset | ok | ok | 7481 // | negative, unaligned offset | ok | ok | 7482 // +----------------------------+------+------+ 7483 // 7484 // This function ignores the addressing mode, so if an offset cannot be used in 7485 // one addressing mode, it is considered illegal. 7486 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 7487 uint64_t FlatVariant) const { 7488 // TODO: Should 0 be special cased? 7489 if (!ST.hasFlatInstOffsets()) 7490 return false; 7491 7492 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT && 7493 (AddrSpace == AMDGPUAS::FLAT_ADDRESS || 7494 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS)) 7495 return false; 7496 7497 bool Signed = FlatVariant != SIInstrFlags::FLAT; 7498 if (ST.hasNegativeScratchOffsetBug() && 7499 FlatVariant == SIInstrFlags::FlatScratch) 7500 Signed = false; 7501 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7502 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 && 7503 (Offset % 4) != 0) { 7504 return false; 7505 } 7506 7507 unsigned N = AMDGPU::getNumFlatOffsetBits(ST, Signed); 7508 return Signed ? isIntN(N, Offset) : isUIntN(N, Offset); 7509 } 7510 7511 // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. 7512 std::pair<int64_t, int64_t> 7513 SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, 7514 uint64_t FlatVariant) const { 7515 int64_t RemainderOffset = COffsetVal; 7516 int64_t ImmField = 0; 7517 bool Signed = FlatVariant != SIInstrFlags::FLAT; 7518 if (ST.hasNegativeScratchOffsetBug() && 7519 FlatVariant == SIInstrFlags::FlatScratch) 7520 Signed = false; 7521 7522 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST, Signed); 7523 if (Signed) { 7524 // Use signed division by a power of two to truncate towards 0. 7525 int64_t D = 1LL << (NumBits - 1); 7526 RemainderOffset = (COffsetVal / D) * D; 7527 ImmField = COffsetVal - RemainderOffset; 7528 7529 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7530 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 && 7531 (ImmField % 4) != 0) { 7532 // Make ImmField a multiple of 4 7533 RemainderOffset += ImmField % 4; 7534 ImmField -= ImmField % 4; 7535 } 7536 } else if (COffsetVal >= 0) { 7537 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 7538 RemainderOffset = COffsetVal - ImmField; 7539 } 7540 7541 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant)); 7542 assert(RemainderOffset + ImmField == COffsetVal); 7543 return {ImmField, RemainderOffset}; 7544 } 7545 7546 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td 7547 enum SIEncodingFamily { 7548 SI = 0, 7549 VI = 1, 7550 SDWA = 2, 7551 SDWA9 = 3, 7552 GFX80 = 4, 7553 GFX9 = 5, 7554 GFX10 = 6, 7555 SDWA10 = 7, 7556 GFX90A = 8 7557 }; 7558 7559 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) { 7560 switch (ST.getGeneration()) { 7561 default: 7562 break; 7563 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 7564 case AMDGPUSubtarget::SEA_ISLANDS: 7565 return SIEncodingFamily::SI; 7566 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 7567 case AMDGPUSubtarget::GFX9: 7568 return SIEncodingFamily::VI; 7569 case AMDGPUSubtarget::GFX10: 7570 return SIEncodingFamily::GFX10; 7571 } 7572 llvm_unreachable("Unknown subtarget generation!"); 7573 } 7574 7575 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 7576 switch(MCOp) { 7577 // These opcodes use indirect register addressing so 7578 // they need special handling by codegen (currently missing). 7579 // Therefore it is too risky to allow these opcodes 7580 // to be selected by dpp combiner or sdwa peepholer. 7581 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 7582 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 7583 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 7584 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 7585 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 7586 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 7587 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 7588 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 7589 return true; 7590 default: 7591 return false; 7592 } 7593 } 7594 7595 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 7596 SIEncodingFamily Gen = subtargetEncodingFamily(ST); 7597 7598 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 7599 ST.getGeneration() == AMDGPUSubtarget::GFX9) 7600 Gen = SIEncodingFamily::GFX9; 7601 7602 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 7603 // subtarget has UnpackedD16VMem feature. 7604 // TODO: remove this when we discard GFX80 encoding. 7605 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 7606 Gen = SIEncodingFamily::GFX80; 7607 7608 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 7609 switch (ST.getGeneration()) { 7610 default: 7611 Gen = SIEncodingFamily::SDWA; 7612 break; 7613 case AMDGPUSubtarget::GFX9: 7614 Gen = SIEncodingFamily::SDWA9; 7615 break; 7616 case AMDGPUSubtarget::GFX10: 7617 Gen = SIEncodingFamily::SDWA10; 7618 break; 7619 } 7620 } 7621 7622 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 7623 7624 // -1 means that Opcode is already a native instruction. 7625 if (MCOp == -1) 7626 return Opcode; 7627 7628 if (ST.hasGFX90AInsts()) { 7629 uint16_t NMCOp = (uint16_t)-1; 7630 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); 7631 if (NMCOp == (uint16_t)-1) 7632 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); 7633 if (NMCOp != (uint16_t)-1) 7634 MCOp = NMCOp; 7635 } 7636 7637 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 7638 // no encoding in the given subtarget generation. 7639 if (MCOp == (uint16_t)-1) 7640 return -1; 7641 7642 if (isAsmOnlyOpcode(MCOp)) 7643 return -1; 7644 7645 return MCOp; 7646 } 7647 7648 static 7649 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 7650 assert(RegOpnd.isReg()); 7651 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 7652 getRegSubRegPair(RegOpnd); 7653 } 7654 7655 TargetInstrInfo::RegSubRegPair 7656 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 7657 assert(MI.isRegSequence()); 7658 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 7659 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 7660 auto &RegOp = MI.getOperand(1 + 2 * I); 7661 return getRegOrUndef(RegOp); 7662 } 7663 return TargetInstrInfo::RegSubRegPair(); 7664 } 7665 7666 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 7667 // Following a subreg of reg:subreg isn't supported 7668 static bool followSubRegDef(MachineInstr &MI, 7669 TargetInstrInfo::RegSubRegPair &RSR) { 7670 if (!RSR.SubReg) 7671 return false; 7672 switch (MI.getOpcode()) { 7673 default: break; 7674 case AMDGPU::REG_SEQUENCE: 7675 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 7676 return true; 7677 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 7678 case AMDGPU::INSERT_SUBREG: 7679 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 7680 // inserted the subreg we're looking for 7681 RSR = getRegOrUndef(MI.getOperand(2)); 7682 else { // the subreg in the rest of the reg 7683 auto R1 = getRegOrUndef(MI.getOperand(1)); 7684 if (R1.SubReg) // subreg of subreg isn't supported 7685 return false; 7686 RSR.Reg = R1.Reg; 7687 } 7688 return true; 7689 } 7690 return false; 7691 } 7692 7693 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 7694 MachineRegisterInfo &MRI) { 7695 assert(MRI.isSSA()); 7696 if (!P.Reg.isVirtual()) 7697 return nullptr; 7698 7699 auto RSR = P; 7700 auto *DefInst = MRI.getVRegDef(RSR.Reg); 7701 while (auto *MI = DefInst) { 7702 DefInst = nullptr; 7703 switch (MI->getOpcode()) { 7704 case AMDGPU::COPY: 7705 case AMDGPU::V_MOV_B32_e32: { 7706 auto &Op1 = MI->getOperand(1); 7707 if (Op1.isReg() && Op1.getReg().isVirtual()) { 7708 if (Op1.isUndef()) 7709 return nullptr; 7710 RSR = getRegSubRegPair(Op1); 7711 DefInst = MRI.getVRegDef(RSR.Reg); 7712 } 7713 break; 7714 } 7715 default: 7716 if (followSubRegDef(*MI, RSR)) { 7717 if (!RSR.Reg) 7718 return nullptr; 7719 DefInst = MRI.getVRegDef(RSR.Reg); 7720 } 7721 } 7722 if (!DefInst) 7723 return MI; 7724 } 7725 return nullptr; 7726 } 7727 7728 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 7729 Register VReg, 7730 const MachineInstr &DefMI, 7731 const MachineInstr &UseMI) { 7732 assert(MRI.isSSA() && "Must be run on SSA"); 7733 7734 auto *TRI = MRI.getTargetRegisterInfo(); 7735 auto *DefBB = DefMI.getParent(); 7736 7737 // Don't bother searching between blocks, although it is possible this block 7738 // doesn't modify exec. 7739 if (UseMI.getParent() != DefBB) 7740 return true; 7741 7742 const int MaxInstScan = 20; 7743 int NumInst = 0; 7744 7745 // Stop scan at the use. 7746 auto E = UseMI.getIterator(); 7747 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 7748 if (I->isDebugInstr()) 7749 continue; 7750 7751 if (++NumInst > MaxInstScan) 7752 return true; 7753 7754 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 7755 return true; 7756 } 7757 7758 return false; 7759 } 7760 7761 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 7762 Register VReg, 7763 const MachineInstr &DefMI) { 7764 assert(MRI.isSSA() && "Must be run on SSA"); 7765 7766 auto *TRI = MRI.getTargetRegisterInfo(); 7767 auto *DefBB = DefMI.getParent(); 7768 7769 const int MaxUseScan = 10; 7770 int NumUse = 0; 7771 7772 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 7773 auto &UseInst = *Use.getParent(); 7774 // Don't bother searching between blocks, although it is possible this block 7775 // doesn't modify exec. 7776 if (UseInst.getParent() != DefBB) 7777 return true; 7778 7779 if (++NumUse > MaxUseScan) 7780 return true; 7781 } 7782 7783 if (NumUse == 0) 7784 return false; 7785 7786 const int MaxInstScan = 20; 7787 int NumInst = 0; 7788 7789 // Stop scan when we have seen all the uses. 7790 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 7791 assert(I != DefBB->end()); 7792 7793 if (I->isDebugInstr()) 7794 continue; 7795 7796 if (++NumInst > MaxInstScan) 7797 return true; 7798 7799 for (const MachineOperand &Op : I->operands()) { 7800 // We don't check reg masks here as they're used only on calls: 7801 // 1. EXEC is only considered const within one BB 7802 // 2. Call should be a terminator instruction if present in a BB 7803 7804 if (!Op.isReg()) 7805 continue; 7806 7807 Register Reg = Op.getReg(); 7808 if (Op.isUse()) { 7809 if (Reg == VReg && --NumUse == 0) 7810 return false; 7811 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 7812 return true; 7813 } 7814 } 7815 } 7816 7817 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 7818 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 7819 const DebugLoc &DL, Register Src, Register Dst) const { 7820 auto Cur = MBB.begin(); 7821 if (Cur != MBB.end()) 7822 do { 7823 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 7824 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 7825 ++Cur; 7826 } while (Cur != MBB.end() && Cur != LastPHIIt); 7827 7828 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 7829 Dst); 7830 } 7831 7832 MachineInstr *SIInstrInfo::createPHISourceCopy( 7833 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 7834 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 7835 if (InsPt != MBB.end() && 7836 (InsPt->getOpcode() == AMDGPU::SI_IF || 7837 InsPt->getOpcode() == AMDGPU::SI_ELSE || 7838 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 7839 InsPt->definesRegister(Src)) { 7840 InsPt++; 7841 return BuildMI(MBB, InsPt, DL, 7842 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 7843 : AMDGPU::S_MOV_B64_term), 7844 Dst) 7845 .addReg(Src, 0, SrcSubReg) 7846 .addReg(AMDGPU::EXEC, RegState::Implicit); 7847 } 7848 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 7849 Dst); 7850 } 7851 7852 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 7853 7854 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 7855 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 7856 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 7857 VirtRegMap *VRM) const { 7858 // This is a bit of a hack (copied from AArch64). Consider this instruction: 7859 // 7860 // %0:sreg_32 = COPY $m0 7861 // 7862 // We explicitly chose SReg_32 for the virtual register so such a copy might 7863 // be eliminated by RegisterCoalescer. However, that may not be possible, and 7864 // %0 may even spill. We can't spill $m0 normally (it would require copying to 7865 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 7866 // TargetInstrInfo::foldMemoryOperand() is going to try. 7867 // A similar issue also exists with spilling and reloading $exec registers. 7868 // 7869 // To prevent that, constrain the %0 register class here. 7870 if (MI.isFullCopy()) { 7871 Register DstReg = MI.getOperand(0).getReg(); 7872 Register SrcReg = MI.getOperand(1).getReg(); 7873 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 7874 (DstReg.isVirtual() != SrcReg.isVirtual())) { 7875 MachineRegisterInfo &MRI = MF.getRegInfo(); 7876 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 7877 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 7878 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 7879 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 7880 return nullptr; 7881 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 7882 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 7883 return nullptr; 7884 } 7885 } 7886 } 7887 7888 return nullptr; 7889 } 7890 7891 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 7892 const MachineInstr &MI, 7893 unsigned *PredCost) const { 7894 if (MI.isBundle()) { 7895 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 7896 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 7897 unsigned Lat = 0, Count = 0; 7898 for (++I; I != E && I->isBundledWithPred(); ++I) { 7899 ++Count; 7900 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 7901 } 7902 return Lat + Count - 1; 7903 } 7904 7905 return SchedModel.computeInstrLatency(&MI); 7906 } 7907 7908 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 7909 switch (MF.getFunction().getCallingConv()) { 7910 case CallingConv::AMDGPU_PS: 7911 return 1; 7912 case CallingConv::AMDGPU_VS: 7913 return 2; 7914 case CallingConv::AMDGPU_GS: 7915 return 3; 7916 case CallingConv::AMDGPU_HS: 7917 case CallingConv::AMDGPU_LS: 7918 case CallingConv::AMDGPU_ES: 7919 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 7920 case CallingConv::AMDGPU_CS: 7921 case CallingConv::AMDGPU_KERNEL: 7922 case CallingConv::C: 7923 case CallingConv::Fast: 7924 default: 7925 // Assume other calling conventions are various compute callable functions 7926 return 0; 7927 } 7928 } 7929