1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// SI Implementation of TargetInstrInfo. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SIInstrInfo.h" 15 #include "AMDGPU.h" 16 #include "AMDGPUInstrInfo.h" 17 #include "GCNHazardRecognizer.h" 18 #include "GCNSubtarget.h" 19 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 20 #include "SIMachineFunctionInfo.h" 21 #include "llvm/Analysis/ValueTracking.h" 22 #include "llvm/CodeGen/LiveVariables.h" 23 #include "llvm/CodeGen/MachineDominators.h" 24 #include "llvm/CodeGen/RegisterScavenging.h" 25 #include "llvm/CodeGen/ScheduleDAG.h" 26 #include "llvm/IR/DiagnosticInfo.h" 27 #include "llvm/IR/IntrinsicsAMDGPU.h" 28 #include "llvm/Support/CommandLine.h" 29 #include "llvm/Target/TargetMachine.h" 30 31 using namespace llvm; 32 33 #define DEBUG_TYPE "si-instr-info" 34 35 #define GET_INSTRINFO_CTOR_DTOR 36 #include "AMDGPUGenInstrInfo.inc" 37 38 namespace llvm { 39 40 class AAResults; 41 42 namespace AMDGPU { 43 #define GET_D16ImageDimIntrinsics_IMPL 44 #define GET_ImageDimIntrinsicTable_IMPL 45 #define GET_RsrcIntrinsics_IMPL 46 #include "AMDGPUGenSearchableTables.inc" 47 } 48 } 49 50 51 // Must be at least 4 to be able to branch over minimum unconditional branch 52 // code. This is only for making it possible to write reasonably small tests for 53 // long branches. 54 static cl::opt<unsigned> 55 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), 56 cl::desc("Restrict range of branch instructions (DEBUG)")); 57 58 static cl::opt<bool> Fix16BitCopies( 59 "amdgpu-fix-16-bit-physreg-copies", 60 cl::desc("Fix copies between 32 and 16 bit registers by extending to 32 bit"), 61 cl::init(true), 62 cl::ReallyHidden); 63 64 SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) 65 : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), 66 RI(ST), ST(ST) { 67 SchedModel.init(&ST); 68 } 69 70 //===----------------------------------------------------------------------===// 71 // TargetInstrInfo callbacks 72 //===----------------------------------------------------------------------===// 73 74 static unsigned getNumOperandsNoGlue(SDNode *Node) { 75 unsigned N = Node->getNumOperands(); 76 while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) 77 --N; 78 return N; 79 } 80 81 /// Returns true if both nodes have the same value for the given 82 /// operand \p Op, or if both nodes do not have this operand. 83 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) { 84 unsigned Opc0 = N0->getMachineOpcode(); 85 unsigned Opc1 = N1->getMachineOpcode(); 86 87 int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName); 88 int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName); 89 90 if (Op0Idx == -1 && Op1Idx == -1) 91 return true; 92 93 94 if ((Op0Idx == -1 && Op1Idx != -1) || 95 (Op1Idx == -1 && Op0Idx != -1)) 96 return false; 97 98 // getNamedOperandIdx returns the index for the MachineInstr's operands, 99 // which includes the result as the first operand. We are indexing into the 100 // MachineSDNode's operands, so we need to skip the result operand to get 101 // the real index. 102 --Op0Idx; 103 --Op1Idx; 104 105 return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx); 106 } 107 108 bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI, 109 AAResults *AA) const { 110 // TODO: The generic check fails for VALU instructions that should be 111 // rematerializable due to implicit reads of exec. We really want all of the 112 // generic logic for this except for this. 113 switch (MI.getOpcode()) { 114 case AMDGPU::V_MOV_B32_e32: 115 case AMDGPU::V_MOV_B32_e64: 116 case AMDGPU::V_MOV_B64_PSEUDO: 117 case AMDGPU::V_ACCVGPR_READ_B32_e64: 118 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 119 // No non-standard implicit operands. 120 assert(MI.getDesc().getNumOperands() == 2); 121 assert(MI.getDesc().getNumImplicitDefs() == 0); 122 assert(MI.getDesc().getNumImplicitUses() == 1); 123 return MI.getNumOperands() == 3; 124 default: 125 return false; 126 } 127 } 128 129 bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1, 130 int64_t &Offset0, 131 int64_t &Offset1) const { 132 if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode()) 133 return false; 134 135 unsigned Opc0 = Load0->getMachineOpcode(); 136 unsigned Opc1 = Load1->getMachineOpcode(); 137 138 // Make sure both are actually loads. 139 if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad()) 140 return false; 141 142 if (isDS(Opc0) && isDS(Opc1)) { 143 144 // FIXME: Handle this case: 145 if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1)) 146 return false; 147 148 // Check base reg. 149 if (Load0->getOperand(0) != Load1->getOperand(0)) 150 return false; 151 152 // Skip read2 / write2 variants for simplicity. 153 // TODO: We should report true if the used offsets are adjacent (excluded 154 // st64 versions). 155 int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 156 int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 157 if (Offset0Idx == -1 || Offset1Idx == -1) 158 return false; 159 160 // XXX - be careful of datalesss loads 161 // getNamedOperandIdx returns the index for MachineInstrs. Since they 162 // include the output in the operand list, but SDNodes don't, we need to 163 // subtract the index by one. 164 Offset0Idx -= get(Opc0).NumDefs; 165 Offset1Idx -= get(Opc1).NumDefs; 166 Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue(); 167 Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue(); 168 return true; 169 } 170 171 if (isSMRD(Opc0) && isSMRD(Opc1)) { 172 // Skip time and cache invalidation instructions. 173 if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 || 174 AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1) 175 return false; 176 177 assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1)); 178 179 // Check base reg. 180 if (Load0->getOperand(0) != Load1->getOperand(0)) 181 return false; 182 183 const ConstantSDNode *Load0Offset = 184 dyn_cast<ConstantSDNode>(Load0->getOperand(1)); 185 const ConstantSDNode *Load1Offset = 186 dyn_cast<ConstantSDNode>(Load1->getOperand(1)); 187 188 if (!Load0Offset || !Load1Offset) 189 return false; 190 191 Offset0 = Load0Offset->getZExtValue(); 192 Offset1 = Load1Offset->getZExtValue(); 193 return true; 194 } 195 196 // MUBUF and MTBUF can access the same addresses. 197 if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) { 198 199 // MUBUF and MTBUF have vaddr at different indices. 200 if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) || 201 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) || 202 !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)) 203 return false; 204 205 int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset); 206 int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset); 207 208 if (OffIdx0 == -1 || OffIdx1 == -1) 209 return false; 210 211 // getNamedOperandIdx returns the index for MachineInstrs. Since they 212 // include the output in the operand list, but SDNodes don't, we need to 213 // subtract the index by one. 214 OffIdx0 -= get(Opc0).NumDefs; 215 OffIdx1 -= get(Opc1).NumDefs; 216 217 SDValue Off0 = Load0->getOperand(OffIdx0); 218 SDValue Off1 = Load1->getOperand(OffIdx1); 219 220 // The offset might be a FrameIndexSDNode. 221 if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1)) 222 return false; 223 224 Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue(); 225 Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue(); 226 return true; 227 } 228 229 return false; 230 } 231 232 static bool isStride64(unsigned Opc) { 233 switch (Opc) { 234 case AMDGPU::DS_READ2ST64_B32: 235 case AMDGPU::DS_READ2ST64_B64: 236 case AMDGPU::DS_WRITE2ST64_B32: 237 case AMDGPU::DS_WRITE2ST64_B64: 238 return true; 239 default: 240 return false; 241 } 242 } 243 244 bool SIInstrInfo::getMemOperandsWithOffsetWidth( 245 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps, 246 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width, 247 const TargetRegisterInfo *TRI) const { 248 if (!LdSt.mayLoadOrStore()) 249 return false; 250 251 unsigned Opc = LdSt.getOpcode(); 252 OffsetIsScalable = false; 253 const MachineOperand *BaseOp, *OffsetOp; 254 int DataOpIdx; 255 256 if (isDS(LdSt)) { 257 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); 258 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 259 if (OffsetOp) { 260 // Normal, single offset LDS instruction. 261 if (!BaseOp) { 262 // DS_CONSUME/DS_APPEND use M0 for the base address. 263 // TODO: find the implicit use operand for M0 and use that as BaseOp? 264 return false; 265 } 266 BaseOps.push_back(BaseOp); 267 Offset = OffsetOp->getImm(); 268 // Get appropriate operand, and compute width accordingly. 269 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 270 if (DataOpIdx == -1) 271 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 272 Width = getOpSize(LdSt, DataOpIdx); 273 } else { 274 // The 2 offset instructions use offset0 and offset1 instead. We can treat 275 // these as a load with a single offset if the 2 offsets are consecutive. 276 // We will use this for some partially aligned loads. 277 const MachineOperand *Offset0Op = 278 getNamedOperand(LdSt, AMDGPU::OpName::offset0); 279 const MachineOperand *Offset1Op = 280 getNamedOperand(LdSt, AMDGPU::OpName::offset1); 281 282 unsigned Offset0 = Offset0Op->getImm(); 283 unsigned Offset1 = Offset1Op->getImm(); 284 if (Offset0 + 1 != Offset1) 285 return false; 286 287 // Each of these offsets is in element sized units, so we need to convert 288 // to bytes of the individual reads. 289 290 unsigned EltSize; 291 if (LdSt.mayLoad()) 292 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16; 293 else { 294 assert(LdSt.mayStore()); 295 int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 296 EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8; 297 } 298 299 if (isStride64(Opc)) 300 EltSize *= 64; 301 302 BaseOps.push_back(BaseOp); 303 Offset = EltSize * Offset0; 304 // Get appropriate operand(s), and compute width accordingly. 305 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 306 if (DataOpIdx == -1) { 307 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0); 308 Width = getOpSize(LdSt, DataOpIdx); 309 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data1); 310 Width += getOpSize(LdSt, DataOpIdx); 311 } else { 312 Width = getOpSize(LdSt, DataOpIdx); 313 } 314 } 315 return true; 316 } 317 318 if (isMUBUF(LdSt) || isMTBUF(LdSt)) { 319 const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc); 320 if (!RSrc) // e.g. BUFFER_WBINVL1_VOL 321 return false; 322 BaseOps.push_back(RSrc); 323 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 324 if (BaseOp && !BaseOp->isFI()) 325 BaseOps.push_back(BaseOp); 326 const MachineOperand *OffsetImm = 327 getNamedOperand(LdSt, AMDGPU::OpName::offset); 328 Offset = OffsetImm->getImm(); 329 const MachineOperand *SOffset = 330 getNamedOperand(LdSt, AMDGPU::OpName::soffset); 331 if (SOffset) { 332 if (SOffset->isReg()) 333 BaseOps.push_back(SOffset); 334 else 335 Offset += SOffset->getImm(); 336 } 337 // Get appropriate operand, and compute width accordingly. 338 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 339 if (DataOpIdx == -1) 340 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 341 Width = getOpSize(LdSt, DataOpIdx); 342 return true; 343 } 344 345 if (isMIMG(LdSt)) { 346 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 347 BaseOps.push_back(&LdSt.getOperand(SRsrcIdx)); 348 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 349 if (VAddr0Idx >= 0) { 350 // GFX10 possible NSA encoding. 351 for (int I = VAddr0Idx; I < SRsrcIdx; ++I) 352 BaseOps.push_back(&LdSt.getOperand(I)); 353 } else { 354 BaseOps.push_back(getNamedOperand(LdSt, AMDGPU::OpName::vaddr)); 355 } 356 Offset = 0; 357 // Get appropriate operand, and compute width accordingly. 358 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 359 Width = getOpSize(LdSt, DataOpIdx); 360 return true; 361 } 362 363 if (isSMRD(LdSt)) { 364 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::sbase); 365 if (!BaseOp) // e.g. S_MEMTIME 366 return false; 367 BaseOps.push_back(BaseOp); 368 OffsetOp = getNamedOperand(LdSt, AMDGPU::OpName::offset); 369 Offset = OffsetOp ? OffsetOp->getImm() : 0; 370 // Get appropriate operand, and compute width accordingly. 371 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst); 372 Width = getOpSize(LdSt, DataOpIdx); 373 return true; 374 } 375 376 if (isFLAT(LdSt)) { 377 // Instructions have either vaddr or saddr or both or none. 378 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::vaddr); 379 if (BaseOp) 380 BaseOps.push_back(BaseOp); 381 BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr); 382 if (BaseOp) 383 BaseOps.push_back(BaseOp); 384 Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); 385 // Get appropriate operand, and compute width accordingly. 386 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 387 if (DataOpIdx == -1) 388 DataOpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata); 389 Width = getOpSize(LdSt, DataOpIdx); 390 return true; 391 } 392 393 return false; 394 } 395 396 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, 397 ArrayRef<const MachineOperand *> BaseOps1, 398 const MachineInstr &MI2, 399 ArrayRef<const MachineOperand *> BaseOps2) { 400 // Only examine the first "base" operand of each instruction, on the 401 // assumption that it represents the real base address of the memory access. 402 // Other operands are typically offsets or indices from this base address. 403 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front())) 404 return true; 405 406 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand()) 407 return false; 408 409 auto MO1 = *MI1.memoperands_begin(); 410 auto MO2 = *MI2.memoperands_begin(); 411 if (MO1->getAddrSpace() != MO2->getAddrSpace()) 412 return false; 413 414 auto Base1 = MO1->getValue(); 415 auto Base2 = MO2->getValue(); 416 if (!Base1 || !Base2) 417 return false; 418 Base1 = getUnderlyingObject(Base1); 419 Base2 = getUnderlyingObject(Base2); 420 421 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) 422 return false; 423 424 return Base1 == Base2; 425 } 426 427 bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1, 428 ArrayRef<const MachineOperand *> BaseOps2, 429 unsigned NumLoads, 430 unsigned NumBytes) const { 431 // If the mem ops (to be clustered) do not have the same base ptr, then they 432 // should not be clustered 433 if (!BaseOps1.empty() && !BaseOps2.empty()) { 434 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent(); 435 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent(); 436 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2)) 437 return false; 438 } else if (!BaseOps1.empty() || !BaseOps2.empty()) { 439 // If only one base op is empty, they do not have the same base ptr 440 return false; 441 } 442 443 // In order to avoid regester pressure, on an average, the number of DWORDS 444 // loaded together by all clustered mem ops should not exceed 8. This is an 445 // empirical value based on certain observations and performance related 446 // experiments. 447 // The good thing about this heuristic is - it avoids clustering of too many 448 // sub-word loads, and also avoids clustering of wide loads. Below is the 449 // brief summary of how the heuristic behaves for various `LoadSize`. 450 // (1) 1 <= LoadSize <= 4: cluster at max 8 mem ops 451 // (2) 5 <= LoadSize <= 8: cluster at max 4 mem ops 452 // (3) 9 <= LoadSize <= 12: cluster at max 2 mem ops 453 // (4) 13 <= LoadSize <= 16: cluster at max 2 mem ops 454 // (5) LoadSize >= 17: do not cluster 455 const unsigned LoadSize = NumBytes / NumLoads; 456 const unsigned NumDWORDs = ((LoadSize + 3) / 4) * NumLoads; 457 return NumDWORDs <= 8; 458 } 459 460 // FIXME: This behaves strangely. If, for example, you have 32 load + stores, 461 // the first 16 loads will be interleaved with the stores, and the next 16 will 462 // be clustered as expected. It should really split into 2 16 store batches. 463 // 464 // Loads are clustered until this returns false, rather than trying to schedule 465 // groups of stores. This also means we have to deal with saying different 466 // address space loads should be clustered, and ones which might cause bank 467 // conflicts. 468 // 469 // This might be deprecated so it might not be worth that much effort to fix. 470 bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, 471 int64_t Offset0, int64_t Offset1, 472 unsigned NumLoads) const { 473 assert(Offset1 > Offset0 && 474 "Second offset should be larger than first offset!"); 475 // If we have less than 16 loads in a row, and the offsets are within 64 476 // bytes, then schedule together. 477 478 // A cacheline is 64 bytes (for global memory). 479 return (NumLoads <= 16 && (Offset1 - Offset0) < 64); 480 } 481 482 static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, 483 MachineBasicBlock::iterator MI, 484 const DebugLoc &DL, MCRegister DestReg, 485 MCRegister SrcReg, bool KillSrc, 486 const char *Msg = "illegal SGPR to VGPR copy") { 487 MachineFunction *MF = MBB.getParent(); 488 DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), Msg, DL, DS_Error); 489 LLVMContext &C = MF->getFunction().getContext(); 490 C.diagnose(IllegalCopy); 491 492 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg) 493 .addReg(SrcReg, getKillRegState(KillSrc)); 494 } 495 496 /// Handle copying from SGPR to AGPR, or from AGPR to AGPR. It is not possible 497 /// to directly copy, so an intermediate VGPR needs to be used. 498 static void indirectCopyToAGPR(const SIInstrInfo &TII, 499 MachineBasicBlock &MBB, 500 MachineBasicBlock::iterator MI, 501 const DebugLoc &DL, MCRegister DestReg, 502 MCRegister SrcReg, bool KillSrc, 503 RegScavenger &RS, 504 Register ImpDefSuperReg = Register(), 505 Register ImpUseSuperReg = Register()) { 506 const SIRegisterInfo &RI = TII.getRegisterInfo(); 507 508 assert(AMDGPU::SReg_32RegClass.contains(SrcReg) || 509 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 510 511 // First try to find defining accvgpr_write to avoid temporary registers. 512 for (auto Def = MI, E = MBB.begin(); Def != E; ) { 513 --Def; 514 if (!Def->definesRegister(SrcReg, &RI)) 515 continue; 516 if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32_e64) 517 break; 518 519 MachineOperand &DefOp = Def->getOperand(1); 520 assert(DefOp.isReg() || DefOp.isImm()); 521 522 if (DefOp.isReg()) { 523 // Check that register source operand if not clobbered before MI. 524 // Immediate operands are always safe to propagate. 525 bool SafeToPropagate = true; 526 for (auto I = Def; I != MI && SafeToPropagate; ++I) 527 if (I->modifiesRegister(DefOp.getReg(), &RI)) 528 SafeToPropagate = false; 529 530 if (!SafeToPropagate) 531 break; 532 533 DefOp.setIsKill(false); 534 } 535 536 MachineInstrBuilder Builder = 537 BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 538 .add(DefOp); 539 if (ImpDefSuperReg) 540 Builder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 541 542 if (ImpUseSuperReg) { 543 Builder.addReg(ImpUseSuperReg, 544 getKillRegState(KillSrc) | RegState::Implicit); 545 } 546 547 return; 548 } 549 550 RS.enterBasicBlock(MBB); 551 RS.forward(MI); 552 553 // Ideally we want to have three registers for a long reg_sequence copy 554 // to hide 2 waitstates between v_mov_b32 and accvgpr_write. 555 unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass, 556 *MBB.getParent()); 557 558 // Registers in the sequence are allocated contiguously so we can just 559 // use register number to pick one of three round-robin temps. 560 unsigned RegNo = DestReg % 3; 561 Register Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 562 if (!Tmp) 563 report_fatal_error("Cannot scavenge VGPR to copy to AGPR"); 564 RS.setRegUsed(Tmp); 565 566 if (!TII.getSubtarget().hasGFX90AInsts()) { 567 // Only loop through if there are any free registers left, otherwise 568 // scavenger may report a fatal error without emergency spill slot 569 // or spill with the slot. 570 while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) { 571 Register Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0); 572 if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs) 573 break; 574 Tmp = Tmp2; 575 RS.setRegUsed(Tmp); 576 } 577 } 578 579 // Insert copy to temporary VGPR. 580 unsigned TmpCopyOp = AMDGPU::V_MOV_B32_e32; 581 if (AMDGPU::AGPR_32RegClass.contains(SrcReg)) { 582 TmpCopyOp = AMDGPU::V_ACCVGPR_READ_B32_e64; 583 } else { 584 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 585 } 586 587 MachineInstrBuilder UseBuilder = BuildMI(MBB, MI, DL, TII.get(TmpCopyOp), Tmp) 588 .addReg(SrcReg, getKillRegState(KillSrc)); 589 if (ImpUseSuperReg) { 590 UseBuilder.addReg(ImpUseSuperReg, 591 getKillRegState(KillSrc) | RegState::Implicit); 592 } 593 594 MachineInstrBuilder DefBuilder 595 = BuildMI(MBB, MI, DL, TII.get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 596 .addReg(Tmp, RegState::Kill); 597 598 if (ImpDefSuperReg) 599 DefBuilder.addReg(ImpDefSuperReg, RegState::Define | RegState::Implicit); 600 } 601 602 static void expandSGPRCopy(const SIInstrInfo &TII, MachineBasicBlock &MBB, 603 MachineBasicBlock::iterator MI, const DebugLoc &DL, 604 MCRegister DestReg, MCRegister SrcReg, bool KillSrc, 605 const TargetRegisterClass *RC, bool Forward) { 606 const SIRegisterInfo &RI = TII.getRegisterInfo(); 607 ArrayRef<int16_t> BaseIndices = RI.getRegSplitParts(RC, 4); 608 MachineBasicBlock::iterator I = MI; 609 MachineInstr *FirstMI = nullptr, *LastMI = nullptr; 610 611 for (unsigned Idx = 0; Idx < BaseIndices.size(); ++Idx) { 612 int16_t SubIdx = BaseIndices[Idx]; 613 Register Reg = RI.getSubReg(DestReg, SubIdx); 614 unsigned Opcode = AMDGPU::S_MOV_B32; 615 616 // Is SGPR aligned? If so try to combine with next. 617 Register Src = RI.getSubReg(SrcReg, SubIdx); 618 bool AlignedDest = ((Reg - AMDGPU::SGPR0) % 2) == 0; 619 bool AlignedSrc = ((Src - AMDGPU::SGPR0) % 2) == 0; 620 if (AlignedDest && AlignedSrc && (Idx + 1 < BaseIndices.size())) { 621 // Can use SGPR64 copy 622 unsigned Channel = RI.getChannelFromSubReg(SubIdx); 623 SubIdx = RI.getSubRegFromChannel(Channel, 2); 624 Opcode = AMDGPU::S_MOV_B64; 625 Idx++; 626 } 627 628 LastMI = BuildMI(MBB, I, DL, TII.get(Opcode), RI.getSubReg(DestReg, SubIdx)) 629 .addReg(RI.getSubReg(SrcReg, SubIdx)) 630 .addReg(SrcReg, RegState::Implicit); 631 632 if (!FirstMI) 633 FirstMI = LastMI; 634 635 if (!Forward) 636 I--; 637 } 638 639 assert(FirstMI && LastMI); 640 if (!Forward) 641 std::swap(FirstMI, LastMI); 642 643 FirstMI->addOperand( 644 MachineOperand::CreateReg(DestReg, true /*IsDef*/, true /*IsImp*/)); 645 646 if (KillSrc) 647 LastMI->addRegisterKilled(SrcReg, &RI); 648 } 649 650 void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, 651 MachineBasicBlock::iterator MI, 652 const DebugLoc &DL, MCRegister DestReg, 653 MCRegister SrcReg, bool KillSrc) const { 654 const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg); 655 656 // FIXME: This is hack to resolve copies between 16 bit and 32 bit 657 // registers until all patterns are fixed. 658 if (Fix16BitCopies && 659 ((RI.getRegSizeInBits(*RC) == 16) ^ 660 (RI.getRegSizeInBits(*RI.getPhysRegClass(SrcReg)) == 16))) { 661 MCRegister &RegToFix = (RI.getRegSizeInBits(*RC) == 16) ? DestReg : SrcReg; 662 MCRegister Super = RI.get32BitRegister(RegToFix); 663 assert(RI.getSubReg(Super, AMDGPU::lo16) == RegToFix); 664 RegToFix = Super; 665 666 if (DestReg == SrcReg) { 667 // Insert empty bundle since ExpandPostRA expects an instruction here. 668 BuildMI(MBB, MI, DL, get(AMDGPU::BUNDLE)); 669 return; 670 } 671 672 RC = RI.getPhysRegClass(DestReg); 673 } 674 675 if (RC == &AMDGPU::VGPR_32RegClass) { 676 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) || 677 AMDGPU::SReg_32RegClass.contains(SrcReg) || 678 AMDGPU::AGPR_32RegClass.contains(SrcReg)); 679 unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ? 680 AMDGPU::V_ACCVGPR_READ_B32_e64 : AMDGPU::V_MOV_B32_e32; 681 BuildMI(MBB, MI, DL, get(Opc), DestReg) 682 .addReg(SrcReg, getKillRegState(KillSrc)); 683 return; 684 } 685 686 if (RC == &AMDGPU::SReg_32_XM0RegClass || 687 RC == &AMDGPU::SReg_32RegClass) { 688 if (SrcReg == AMDGPU::SCC) { 689 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) 690 .addImm(1) 691 .addImm(0); 692 return; 693 } 694 695 if (DestReg == AMDGPU::VCC_LO) { 696 if (AMDGPU::SReg_32RegClass.contains(SrcReg)) { 697 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO) 698 .addReg(SrcReg, getKillRegState(KillSrc)); 699 } else { 700 // FIXME: Hack until VReg_1 removed. 701 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 702 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 703 .addImm(0) 704 .addReg(SrcReg, getKillRegState(KillSrc)); 705 } 706 707 return; 708 } 709 710 if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) { 711 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 712 return; 713 } 714 715 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 716 .addReg(SrcReg, getKillRegState(KillSrc)); 717 return; 718 } 719 720 if (RC == &AMDGPU::SReg_64RegClass) { 721 if (SrcReg == AMDGPU::SCC) { 722 BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B64), DestReg) 723 .addImm(1) 724 .addImm(0); 725 return; 726 } 727 728 if (DestReg == AMDGPU::VCC) { 729 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 730 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC) 731 .addReg(SrcReg, getKillRegState(KillSrc)); 732 } else { 733 // FIXME: Hack until VReg_1 removed. 734 assert(AMDGPU::VGPR_32RegClass.contains(SrcReg)); 735 BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32)) 736 .addImm(0) 737 .addReg(SrcReg, getKillRegState(KillSrc)); 738 } 739 740 return; 741 } 742 743 if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) { 744 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 745 return; 746 } 747 748 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 749 .addReg(SrcReg, getKillRegState(KillSrc)); 750 return; 751 } 752 753 if (DestReg == AMDGPU::SCC) { 754 // Copying 64-bit or 32-bit sources to SCC barely makes sense, 755 // but SelectionDAG emits such copies for i1 sources. 756 if (AMDGPU::SReg_64RegClass.contains(SrcReg)) { 757 // This copy can only be produced by patterns 758 // with explicit SCC, which are known to be enabled 759 // only for subtargets with S_CMP_LG_U64 present. 760 assert(ST.hasScalarCompareEq64()); 761 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U64)) 762 .addReg(SrcReg, getKillRegState(KillSrc)) 763 .addImm(0); 764 } else { 765 assert(AMDGPU::SReg_32RegClass.contains(SrcReg)); 766 BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32)) 767 .addReg(SrcReg, getKillRegState(KillSrc)) 768 .addImm(0); 769 } 770 771 return; 772 } 773 774 if (RC == &AMDGPU::AGPR_32RegClass) { 775 if (AMDGPU::VGPR_32RegClass.contains(SrcReg)) { 776 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DestReg) 777 .addReg(SrcReg, getKillRegState(KillSrc)); 778 return; 779 } 780 781 if (AMDGPU::AGPR_32RegClass.contains(SrcReg) && ST.hasGFX90AInsts()) { 782 BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_MOV_B32), DestReg) 783 .addReg(SrcReg, getKillRegState(KillSrc)); 784 return; 785 } 786 787 // FIXME: Pass should maintain scavenger to avoid scan through the block on 788 // every AGPR spill. 789 RegScavenger RS; 790 indirectCopyToAGPR(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RS); 791 return; 792 } 793 794 const unsigned Size = RI.getRegSizeInBits(*RC); 795 if (Size == 16) { 796 assert(AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 797 AMDGPU::VGPR_HI16RegClass.contains(SrcReg) || 798 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 799 AMDGPU::AGPR_LO16RegClass.contains(SrcReg)); 800 801 bool IsSGPRDst = AMDGPU::SReg_LO16RegClass.contains(DestReg); 802 bool IsSGPRSrc = AMDGPU::SReg_LO16RegClass.contains(SrcReg); 803 bool IsAGPRDst = AMDGPU::AGPR_LO16RegClass.contains(DestReg); 804 bool IsAGPRSrc = AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 805 bool DstLow = AMDGPU::VGPR_LO16RegClass.contains(DestReg) || 806 AMDGPU::SReg_LO16RegClass.contains(DestReg) || 807 AMDGPU::AGPR_LO16RegClass.contains(DestReg); 808 bool SrcLow = AMDGPU::VGPR_LO16RegClass.contains(SrcReg) || 809 AMDGPU::SReg_LO16RegClass.contains(SrcReg) || 810 AMDGPU::AGPR_LO16RegClass.contains(SrcReg); 811 MCRegister NewDestReg = RI.get32BitRegister(DestReg); 812 MCRegister NewSrcReg = RI.get32BitRegister(SrcReg); 813 814 if (IsSGPRDst) { 815 if (!IsSGPRSrc) { 816 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 817 return; 818 } 819 820 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), NewDestReg) 821 .addReg(NewSrcReg, getKillRegState(KillSrc)); 822 return; 823 } 824 825 if (IsAGPRDst || IsAGPRSrc) { 826 if (!DstLow || !SrcLow) { 827 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 828 "Cannot use hi16 subreg with an AGPR!"); 829 } 830 831 copyPhysReg(MBB, MI, DL, NewDestReg, NewSrcReg, KillSrc); 832 return; 833 } 834 835 if (IsSGPRSrc && !ST.hasSDWAScalar()) { 836 if (!DstLow || !SrcLow) { 837 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc, 838 "Cannot use hi16 subreg on VI!"); 839 } 840 841 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), NewDestReg) 842 .addReg(NewSrcReg, getKillRegState(KillSrc)); 843 return; 844 } 845 846 auto MIB = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_sdwa), NewDestReg) 847 .addImm(0) // src0_modifiers 848 .addReg(NewSrcReg) 849 .addImm(0) // clamp 850 .addImm(DstLow ? AMDGPU::SDWA::SdwaSel::WORD_0 851 : AMDGPU::SDWA::SdwaSel::WORD_1) 852 .addImm(AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) 853 .addImm(SrcLow ? AMDGPU::SDWA::SdwaSel::WORD_0 854 : AMDGPU::SDWA::SdwaSel::WORD_1) 855 .addReg(NewDestReg, RegState::Implicit | RegState::Undef); 856 // First implicit operand is $exec. 857 MIB->tieOperands(0, MIB->getNumOperands() - 1); 858 return; 859 } 860 861 const TargetRegisterClass *SrcRC = RI.getPhysRegClass(SrcReg); 862 if (RC == RI.getVGPR64Class() && (SrcRC == RC || RI.isSGPRClass(SrcRC))) { 863 if (ST.hasPackedFP32Ops()) { 864 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DestReg) 865 .addImm(SISrcMods::OP_SEL_1) 866 .addReg(SrcReg) 867 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 868 .addReg(SrcReg) 869 .addImm(0) // op_sel_lo 870 .addImm(0) // op_sel_hi 871 .addImm(0) // neg_lo 872 .addImm(0) // neg_hi 873 .addImm(0) // clamp 874 .addReg(SrcReg, getKillRegState(KillSrc) | RegState::Implicit); 875 return; 876 } 877 } 878 879 const bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg); 880 if (RI.isSGPRClass(RC)) { 881 if (!RI.isSGPRClass(SrcRC)) { 882 reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc); 883 return; 884 } 885 expandSGPRCopy(*this, MBB, MI, DL, DestReg, SrcReg, KillSrc, RC, Forward); 886 return; 887 } 888 889 unsigned EltSize = 4; 890 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 891 if (RI.hasAGPRs(RC)) { 892 Opcode = (RI.hasVGPRs(SrcRC)) ? 893 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 894 } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(SrcRC)) { 895 Opcode = AMDGPU::V_ACCVGPR_READ_B32_e64; 896 } else if ((Size % 64 == 0) && RI.hasVGPRs(RC) && 897 (RI.isProperlyAlignedRC(*RC) && 898 (SrcRC == RC || RI.isSGPRClass(SrcRC)))) { 899 // TODO: In 96-bit case, could do a 64-bit mov and then a 32-bit mov. 900 if (ST.hasPackedFP32Ops()) { 901 Opcode = AMDGPU::V_PK_MOV_B32; 902 EltSize = 8; 903 } 904 } 905 906 // For the cases where we need an intermediate instruction/temporary register 907 // (destination is an AGPR), we need a scavenger. 908 // 909 // FIXME: The pass should maintain this for us so we don't have to re-scan the 910 // whole block for every handled copy. 911 std::unique_ptr<RegScavenger> RS; 912 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) 913 RS.reset(new RegScavenger()); 914 915 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize); 916 917 // If there is an overlap, we can't kill the super-register on the last 918 // instruction, since it will also kill the components made live by this def. 919 const bool CanKillSuperReg = KillSrc && !RI.regsOverlap(SrcReg, DestReg); 920 921 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 922 unsigned SubIdx; 923 if (Forward) 924 SubIdx = SubIndices[Idx]; 925 else 926 SubIdx = SubIndices[SubIndices.size() - Idx - 1]; 927 928 bool UseKill = CanKillSuperReg && Idx == SubIndices.size() - 1; 929 930 if (Opcode == AMDGPU::INSTRUCTION_LIST_END) { 931 Register ImpDefSuper = Idx == 0 ? Register(DestReg) : Register(); 932 Register ImpUseSuper = SrcReg; 933 indirectCopyToAGPR(*this, MBB, MI, DL, RI.getSubReg(DestReg, SubIdx), 934 RI.getSubReg(SrcReg, SubIdx), UseKill, *RS, 935 ImpDefSuper, ImpUseSuper); 936 } else if (Opcode == AMDGPU::V_PK_MOV_B32) { 937 Register DstSubReg = RI.getSubReg(DestReg, SubIdx); 938 Register SrcSubReg = RI.getSubReg(SrcReg, SubIdx); 939 MachineInstrBuilder MIB = 940 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), DstSubReg) 941 .addImm(SISrcMods::OP_SEL_1) 942 .addReg(SrcSubReg) 943 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) 944 .addReg(SrcSubReg) 945 .addImm(0) // op_sel_lo 946 .addImm(0) // op_sel_hi 947 .addImm(0) // neg_lo 948 .addImm(0) // neg_hi 949 .addImm(0) // clamp 950 .addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 951 if (Idx == 0) 952 MIB.addReg(DestReg, RegState::Define | RegState::Implicit); 953 } else { 954 MachineInstrBuilder Builder = 955 BuildMI(MBB, MI, DL, get(Opcode), RI.getSubReg(DestReg, SubIdx)) 956 .addReg(RI.getSubReg(SrcReg, SubIdx)); 957 if (Idx == 0) 958 Builder.addReg(DestReg, RegState::Define | RegState::Implicit); 959 960 Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit); 961 } 962 } 963 } 964 965 int SIInstrInfo::commuteOpcode(unsigned Opcode) const { 966 int NewOpc; 967 968 // Try to map original to commuted opcode 969 NewOpc = AMDGPU::getCommuteRev(Opcode); 970 if (NewOpc != -1) 971 // Check if the commuted (REV) opcode exists on the target. 972 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 973 974 // Try to map commuted to original opcode 975 NewOpc = AMDGPU::getCommuteOrig(Opcode); 976 if (NewOpc != -1) 977 // Check if the original (non-REV) opcode exists on the target. 978 return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1; 979 980 return Opcode; 981 } 982 983 void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB, 984 MachineBasicBlock::iterator MI, 985 const DebugLoc &DL, unsigned DestReg, 986 int64_t Value) const { 987 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 988 const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg); 989 if (RegClass == &AMDGPU::SReg_32RegClass || 990 RegClass == &AMDGPU::SGPR_32RegClass || 991 RegClass == &AMDGPU::SReg_32_XM0RegClass || 992 RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) { 993 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg) 994 .addImm(Value); 995 return; 996 } 997 998 if (RegClass == &AMDGPU::SReg_64RegClass || 999 RegClass == &AMDGPU::SGPR_64RegClass || 1000 RegClass == &AMDGPU::SReg_64_XEXECRegClass) { 1001 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg) 1002 .addImm(Value); 1003 return; 1004 } 1005 1006 if (RegClass == &AMDGPU::VGPR_32RegClass) { 1007 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg) 1008 .addImm(Value); 1009 return; 1010 } 1011 if (RegClass->hasSuperClassEq(&AMDGPU::VReg_64RegClass)) { 1012 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg) 1013 .addImm(Value); 1014 return; 1015 } 1016 1017 unsigned EltSize = 4; 1018 unsigned Opcode = AMDGPU::V_MOV_B32_e32; 1019 if (RI.isSGPRClass(RegClass)) { 1020 if (RI.getRegSizeInBits(*RegClass) > 32) { 1021 Opcode = AMDGPU::S_MOV_B64; 1022 EltSize = 8; 1023 } else { 1024 Opcode = AMDGPU::S_MOV_B32; 1025 EltSize = 4; 1026 } 1027 } 1028 1029 ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize); 1030 for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) { 1031 int64_t IdxValue = Idx == 0 ? Value : 0; 1032 1033 MachineInstrBuilder Builder = BuildMI(MBB, MI, DL, 1034 get(Opcode), RI.getSubReg(DestReg, SubIndices[Idx])); 1035 Builder.addImm(IdxValue); 1036 } 1037 } 1038 1039 const TargetRegisterClass * 1040 SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const { 1041 return &AMDGPU::VGPR_32RegClass; 1042 } 1043 1044 void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, 1045 MachineBasicBlock::iterator I, 1046 const DebugLoc &DL, Register DstReg, 1047 ArrayRef<MachineOperand> Cond, 1048 Register TrueReg, 1049 Register FalseReg) const { 1050 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 1051 const TargetRegisterClass *BoolXExecRC = 1052 RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 1053 assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass && 1054 "Not a VGPR32 reg"); 1055 1056 if (Cond.size() == 1) { 1057 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1058 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1059 .add(Cond[0]); 1060 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1061 .addImm(0) 1062 .addReg(FalseReg) 1063 .addImm(0) 1064 .addReg(TrueReg) 1065 .addReg(SReg); 1066 } else if (Cond.size() == 2) { 1067 assert(Cond[0].isImm() && "Cond[0] is not an immediate"); 1068 switch (Cond[0].getImm()) { 1069 case SIInstrInfo::SCC_TRUE: { 1070 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1071 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1072 : AMDGPU::S_CSELECT_B64), SReg) 1073 .addImm(1) 1074 .addImm(0); 1075 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1076 .addImm(0) 1077 .addReg(FalseReg) 1078 .addImm(0) 1079 .addReg(TrueReg) 1080 .addReg(SReg); 1081 break; 1082 } 1083 case SIInstrInfo::SCC_FALSE: { 1084 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1085 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1086 : AMDGPU::S_CSELECT_B64), SReg) 1087 .addImm(0) 1088 .addImm(1); 1089 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1090 .addImm(0) 1091 .addReg(FalseReg) 1092 .addImm(0) 1093 .addReg(TrueReg) 1094 .addReg(SReg); 1095 break; 1096 } 1097 case SIInstrInfo::VCCNZ: { 1098 MachineOperand RegOp = Cond[1]; 1099 RegOp.setImplicit(false); 1100 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1101 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1102 .add(RegOp); 1103 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1104 .addImm(0) 1105 .addReg(FalseReg) 1106 .addImm(0) 1107 .addReg(TrueReg) 1108 .addReg(SReg); 1109 break; 1110 } 1111 case SIInstrInfo::VCCZ: { 1112 MachineOperand RegOp = Cond[1]; 1113 RegOp.setImplicit(false); 1114 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1115 BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg) 1116 .add(RegOp); 1117 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1118 .addImm(0) 1119 .addReg(TrueReg) 1120 .addImm(0) 1121 .addReg(FalseReg) 1122 .addReg(SReg); 1123 break; 1124 } 1125 case SIInstrInfo::EXECNZ: { 1126 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1127 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1128 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1129 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1130 .addImm(0); 1131 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1132 : AMDGPU::S_CSELECT_B64), SReg) 1133 .addImm(1) 1134 .addImm(0); 1135 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1136 .addImm(0) 1137 .addReg(FalseReg) 1138 .addImm(0) 1139 .addReg(TrueReg) 1140 .addReg(SReg); 1141 break; 1142 } 1143 case SIInstrInfo::EXECZ: { 1144 Register SReg = MRI.createVirtualRegister(BoolXExecRC); 1145 Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC()); 1146 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1147 : AMDGPU::S_OR_SAVEEXEC_B64), SReg2) 1148 .addImm(0); 1149 BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 1150 : AMDGPU::S_CSELECT_B64), SReg) 1151 .addImm(0) 1152 .addImm(1); 1153 BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) 1154 .addImm(0) 1155 .addReg(FalseReg) 1156 .addImm(0) 1157 .addReg(TrueReg) 1158 .addReg(SReg); 1159 llvm_unreachable("Unhandled branch predicate EXECZ"); 1160 break; 1161 } 1162 default: 1163 llvm_unreachable("invalid branch predicate"); 1164 } 1165 } else { 1166 llvm_unreachable("Can only handle Cond size 1 or 2"); 1167 } 1168 } 1169 1170 Register SIInstrInfo::insertEQ(MachineBasicBlock *MBB, 1171 MachineBasicBlock::iterator I, 1172 const DebugLoc &DL, 1173 Register SrcReg, int Value) const { 1174 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1175 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1176 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg) 1177 .addImm(Value) 1178 .addReg(SrcReg); 1179 1180 return Reg; 1181 } 1182 1183 Register SIInstrInfo::insertNE(MachineBasicBlock *MBB, 1184 MachineBasicBlock::iterator I, 1185 const DebugLoc &DL, 1186 Register SrcReg, int Value) const { 1187 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 1188 Register Reg = MRI.createVirtualRegister(RI.getBoolRC()); 1189 BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg) 1190 .addImm(Value) 1191 .addReg(SrcReg); 1192 1193 return Reg; 1194 } 1195 1196 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const { 1197 1198 if (RI.hasAGPRs(DstRC)) 1199 return AMDGPU::COPY; 1200 if (RI.getRegSizeInBits(*DstRC) == 32) { 1201 return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32; 1202 } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) { 1203 return AMDGPU::S_MOV_B64; 1204 } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) { 1205 return AMDGPU::V_MOV_B64_PSEUDO; 1206 } 1207 return AMDGPU::COPY; 1208 } 1209 1210 const MCInstrDesc & 1211 SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize, 1212 bool IsIndirectSrc) const { 1213 if (IsIndirectSrc) { 1214 if (VecSize <= 32) // 4 bytes 1215 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1); 1216 if (VecSize <= 64) // 8 bytes 1217 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2); 1218 if (VecSize <= 96) // 12 bytes 1219 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3); 1220 if (VecSize <= 128) // 16 bytes 1221 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4); 1222 if (VecSize <= 160) // 20 bytes 1223 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5); 1224 if (VecSize <= 256) // 32 bytes 1225 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8); 1226 if (VecSize <= 512) // 64 bytes 1227 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16); 1228 if (VecSize <= 1024) // 128 bytes 1229 return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32); 1230 1231 llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos"); 1232 } 1233 1234 if (VecSize <= 32) // 4 bytes 1235 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1); 1236 if (VecSize <= 64) // 8 bytes 1237 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2); 1238 if (VecSize <= 96) // 12 bytes 1239 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3); 1240 if (VecSize <= 128) // 16 bytes 1241 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4); 1242 if (VecSize <= 160) // 20 bytes 1243 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5); 1244 if (VecSize <= 256) // 32 bytes 1245 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8); 1246 if (VecSize <= 512) // 64 bytes 1247 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16); 1248 if (VecSize <= 1024) // 128 bytes 1249 return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32); 1250 1251 llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos"); 1252 } 1253 1254 static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) { 1255 if (VecSize <= 32) // 4 bytes 1256 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1257 if (VecSize <= 64) // 8 bytes 1258 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1259 if (VecSize <= 96) // 12 bytes 1260 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1261 if (VecSize <= 128) // 16 bytes 1262 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1263 if (VecSize <= 160) // 20 bytes 1264 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1265 if (VecSize <= 256) // 32 bytes 1266 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1267 if (VecSize <= 512) // 64 bytes 1268 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1269 if (VecSize <= 1024) // 128 bytes 1270 return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1271 1272 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1273 } 1274 1275 static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) { 1276 if (VecSize <= 32) // 4 bytes 1277 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1; 1278 if (VecSize <= 64) // 8 bytes 1279 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2; 1280 if (VecSize <= 96) // 12 bytes 1281 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3; 1282 if (VecSize <= 128) // 16 bytes 1283 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4; 1284 if (VecSize <= 160) // 20 bytes 1285 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5; 1286 if (VecSize <= 256) // 32 bytes 1287 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8; 1288 if (VecSize <= 512) // 64 bytes 1289 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16; 1290 if (VecSize <= 1024) // 128 bytes 1291 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32; 1292 1293 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1294 } 1295 1296 static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) { 1297 if (VecSize <= 64) // 8 bytes 1298 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1; 1299 if (VecSize <= 128) // 16 bytes 1300 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2; 1301 if (VecSize <= 256) // 32 bytes 1302 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4; 1303 if (VecSize <= 512) // 64 bytes 1304 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8; 1305 if (VecSize <= 1024) // 128 bytes 1306 return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16; 1307 1308 llvm_unreachable("unsupported size for IndirectRegWrite pseudos"); 1309 } 1310 1311 const MCInstrDesc & 1312 SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize, 1313 bool IsSGPR) const { 1314 if (IsSGPR) { 1315 switch (EltSize) { 1316 case 32: 1317 return get(getIndirectSGPRWriteMovRelPseudo32(VecSize)); 1318 case 64: 1319 return get(getIndirectSGPRWriteMovRelPseudo64(VecSize)); 1320 default: 1321 llvm_unreachable("invalid reg indexing elt size"); 1322 } 1323 } 1324 1325 assert(EltSize == 32 && "invalid reg indexing elt size"); 1326 return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize)); 1327 } 1328 1329 static unsigned getSGPRSpillSaveOpcode(unsigned Size) { 1330 switch (Size) { 1331 case 4: 1332 return AMDGPU::SI_SPILL_S32_SAVE; 1333 case 8: 1334 return AMDGPU::SI_SPILL_S64_SAVE; 1335 case 12: 1336 return AMDGPU::SI_SPILL_S96_SAVE; 1337 case 16: 1338 return AMDGPU::SI_SPILL_S128_SAVE; 1339 case 20: 1340 return AMDGPU::SI_SPILL_S160_SAVE; 1341 case 24: 1342 return AMDGPU::SI_SPILL_S192_SAVE; 1343 case 28: 1344 return AMDGPU::SI_SPILL_S224_SAVE; 1345 case 32: 1346 return AMDGPU::SI_SPILL_S256_SAVE; 1347 case 64: 1348 return AMDGPU::SI_SPILL_S512_SAVE; 1349 case 128: 1350 return AMDGPU::SI_SPILL_S1024_SAVE; 1351 default: 1352 llvm_unreachable("unknown register size"); 1353 } 1354 } 1355 1356 static unsigned getVGPRSpillSaveOpcode(unsigned Size) { 1357 switch (Size) { 1358 case 4: 1359 return AMDGPU::SI_SPILL_V32_SAVE; 1360 case 8: 1361 return AMDGPU::SI_SPILL_V64_SAVE; 1362 case 12: 1363 return AMDGPU::SI_SPILL_V96_SAVE; 1364 case 16: 1365 return AMDGPU::SI_SPILL_V128_SAVE; 1366 case 20: 1367 return AMDGPU::SI_SPILL_V160_SAVE; 1368 case 24: 1369 return AMDGPU::SI_SPILL_V192_SAVE; 1370 case 28: 1371 return AMDGPU::SI_SPILL_V224_SAVE; 1372 case 32: 1373 return AMDGPU::SI_SPILL_V256_SAVE; 1374 case 64: 1375 return AMDGPU::SI_SPILL_V512_SAVE; 1376 case 128: 1377 return AMDGPU::SI_SPILL_V1024_SAVE; 1378 default: 1379 llvm_unreachable("unknown register size"); 1380 } 1381 } 1382 1383 static unsigned getAGPRSpillSaveOpcode(unsigned Size) { 1384 switch (Size) { 1385 case 4: 1386 return AMDGPU::SI_SPILL_A32_SAVE; 1387 case 8: 1388 return AMDGPU::SI_SPILL_A64_SAVE; 1389 case 12: 1390 return AMDGPU::SI_SPILL_A96_SAVE; 1391 case 16: 1392 return AMDGPU::SI_SPILL_A128_SAVE; 1393 case 20: 1394 return AMDGPU::SI_SPILL_A160_SAVE; 1395 case 24: 1396 return AMDGPU::SI_SPILL_A192_SAVE; 1397 case 28: 1398 return AMDGPU::SI_SPILL_A224_SAVE; 1399 case 32: 1400 return AMDGPU::SI_SPILL_A256_SAVE; 1401 case 64: 1402 return AMDGPU::SI_SPILL_A512_SAVE; 1403 case 128: 1404 return AMDGPU::SI_SPILL_A1024_SAVE; 1405 default: 1406 llvm_unreachable("unknown register size"); 1407 } 1408 } 1409 1410 void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, 1411 MachineBasicBlock::iterator MI, 1412 Register SrcReg, bool isKill, 1413 int FrameIndex, 1414 const TargetRegisterClass *RC, 1415 const TargetRegisterInfo *TRI) const { 1416 MachineFunction *MF = MBB.getParent(); 1417 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1418 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1419 const DebugLoc &DL = MBB.findDebugLoc(MI); 1420 1421 MachinePointerInfo PtrInfo 1422 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1423 MachineMemOperand *MMO = MF->getMachineMemOperand( 1424 PtrInfo, MachineMemOperand::MOStore, FrameInfo.getObjectSize(FrameIndex), 1425 FrameInfo.getObjectAlign(FrameIndex)); 1426 unsigned SpillSize = TRI->getSpillSize(*RC); 1427 1428 if (RI.isSGPRClass(RC)) { 1429 MFI->setHasSpilledSGPRs(); 1430 assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); 1431 assert(SrcReg != AMDGPU::EXEC_LO && SrcReg != AMDGPU::EXEC_HI && 1432 SrcReg != AMDGPU::EXEC && "exec should not be spilled"); 1433 1434 // We are only allowed to create one new instruction when spilling 1435 // registers, so we need to use pseudo instruction for spilling SGPRs. 1436 const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize)); 1437 1438 // The SGPR spill/restore instructions only work on number sgprs, so we need 1439 // to make sure we are using the correct register class. 1440 if (SrcReg.isVirtual() && SpillSize == 4) { 1441 MachineRegisterInfo &MRI = MF->getRegInfo(); 1442 MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1443 } 1444 1445 BuildMI(MBB, MI, DL, OpDesc) 1446 .addReg(SrcReg, getKillRegState(isKill)) // data 1447 .addFrameIndex(FrameIndex) // addr 1448 .addMemOperand(MMO) 1449 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1450 1451 if (RI.spillSGPRToVGPR()) 1452 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1453 return; 1454 } 1455 1456 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize) 1457 : getVGPRSpillSaveOpcode(SpillSize); 1458 MFI->setHasSpilledVGPRs(); 1459 1460 BuildMI(MBB, MI, DL, get(Opcode)) 1461 .addReg(SrcReg, getKillRegState(isKill)) // data 1462 .addFrameIndex(FrameIndex) // addr 1463 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1464 .addImm(0) // offset 1465 .addMemOperand(MMO); 1466 } 1467 1468 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) { 1469 switch (Size) { 1470 case 4: 1471 return AMDGPU::SI_SPILL_S32_RESTORE; 1472 case 8: 1473 return AMDGPU::SI_SPILL_S64_RESTORE; 1474 case 12: 1475 return AMDGPU::SI_SPILL_S96_RESTORE; 1476 case 16: 1477 return AMDGPU::SI_SPILL_S128_RESTORE; 1478 case 20: 1479 return AMDGPU::SI_SPILL_S160_RESTORE; 1480 case 24: 1481 return AMDGPU::SI_SPILL_S192_RESTORE; 1482 case 28: 1483 return AMDGPU::SI_SPILL_S224_RESTORE; 1484 case 32: 1485 return AMDGPU::SI_SPILL_S256_RESTORE; 1486 case 64: 1487 return AMDGPU::SI_SPILL_S512_RESTORE; 1488 case 128: 1489 return AMDGPU::SI_SPILL_S1024_RESTORE; 1490 default: 1491 llvm_unreachable("unknown register size"); 1492 } 1493 } 1494 1495 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) { 1496 switch (Size) { 1497 case 4: 1498 return AMDGPU::SI_SPILL_V32_RESTORE; 1499 case 8: 1500 return AMDGPU::SI_SPILL_V64_RESTORE; 1501 case 12: 1502 return AMDGPU::SI_SPILL_V96_RESTORE; 1503 case 16: 1504 return AMDGPU::SI_SPILL_V128_RESTORE; 1505 case 20: 1506 return AMDGPU::SI_SPILL_V160_RESTORE; 1507 case 24: 1508 return AMDGPU::SI_SPILL_V192_RESTORE; 1509 case 28: 1510 return AMDGPU::SI_SPILL_V224_RESTORE; 1511 case 32: 1512 return AMDGPU::SI_SPILL_V256_RESTORE; 1513 case 64: 1514 return AMDGPU::SI_SPILL_V512_RESTORE; 1515 case 128: 1516 return AMDGPU::SI_SPILL_V1024_RESTORE; 1517 default: 1518 llvm_unreachable("unknown register size"); 1519 } 1520 } 1521 1522 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) { 1523 switch (Size) { 1524 case 4: 1525 return AMDGPU::SI_SPILL_A32_RESTORE; 1526 case 8: 1527 return AMDGPU::SI_SPILL_A64_RESTORE; 1528 case 12: 1529 return AMDGPU::SI_SPILL_A96_RESTORE; 1530 case 16: 1531 return AMDGPU::SI_SPILL_A128_RESTORE; 1532 case 20: 1533 return AMDGPU::SI_SPILL_A160_RESTORE; 1534 case 24: 1535 return AMDGPU::SI_SPILL_A192_RESTORE; 1536 case 28: 1537 return AMDGPU::SI_SPILL_A224_RESTORE; 1538 case 32: 1539 return AMDGPU::SI_SPILL_A256_RESTORE; 1540 case 64: 1541 return AMDGPU::SI_SPILL_A512_RESTORE; 1542 case 128: 1543 return AMDGPU::SI_SPILL_A1024_RESTORE; 1544 default: 1545 llvm_unreachable("unknown register size"); 1546 } 1547 } 1548 1549 void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, 1550 MachineBasicBlock::iterator MI, 1551 Register DestReg, int FrameIndex, 1552 const TargetRegisterClass *RC, 1553 const TargetRegisterInfo *TRI) const { 1554 MachineFunction *MF = MBB.getParent(); 1555 SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>(); 1556 MachineFrameInfo &FrameInfo = MF->getFrameInfo(); 1557 const DebugLoc &DL = MBB.findDebugLoc(MI); 1558 unsigned SpillSize = TRI->getSpillSize(*RC); 1559 1560 MachinePointerInfo PtrInfo 1561 = MachinePointerInfo::getFixedStack(*MF, FrameIndex); 1562 1563 MachineMemOperand *MMO = MF->getMachineMemOperand( 1564 PtrInfo, MachineMemOperand::MOLoad, FrameInfo.getObjectSize(FrameIndex), 1565 FrameInfo.getObjectAlign(FrameIndex)); 1566 1567 if (RI.isSGPRClass(RC)) { 1568 MFI->setHasSpilledSGPRs(); 1569 assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); 1570 assert(DestReg != AMDGPU::EXEC_LO && DestReg != AMDGPU::EXEC_HI && 1571 DestReg != AMDGPU::EXEC && "exec should not be spilled"); 1572 1573 // FIXME: Maybe this should not include a memoperand because it will be 1574 // lowered to non-memory instructions. 1575 const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize)); 1576 if (DestReg.isVirtual() && SpillSize == 4) { 1577 MachineRegisterInfo &MRI = MF->getRegInfo(); 1578 MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 1579 } 1580 1581 if (RI.spillSGPRToVGPR()) 1582 FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill); 1583 BuildMI(MBB, MI, DL, OpDesc, DestReg) 1584 .addFrameIndex(FrameIndex) // addr 1585 .addMemOperand(MMO) 1586 .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit); 1587 1588 return; 1589 } 1590 1591 unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize) 1592 : getVGPRSpillRestoreOpcode(SpillSize); 1593 BuildMI(MBB, MI, DL, get(Opcode), DestReg) 1594 .addFrameIndex(FrameIndex) // vaddr 1595 .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset 1596 .addImm(0) // offset 1597 .addMemOperand(MMO); 1598 } 1599 1600 void SIInstrInfo::insertNoop(MachineBasicBlock &MBB, 1601 MachineBasicBlock::iterator MI) const { 1602 insertNoops(MBB, MI, 1); 1603 } 1604 1605 void SIInstrInfo::insertNoops(MachineBasicBlock &MBB, 1606 MachineBasicBlock::iterator MI, 1607 unsigned Quantity) const { 1608 DebugLoc DL = MBB.findDebugLoc(MI); 1609 while (Quantity > 0) { 1610 unsigned Arg = std::min(Quantity, 8u); 1611 Quantity -= Arg; 1612 BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP)).addImm(Arg - 1); 1613 } 1614 } 1615 1616 void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const { 1617 auto MF = MBB.getParent(); 1618 SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>(); 1619 1620 assert(Info->isEntryFunction()); 1621 1622 if (MBB.succ_empty()) { 1623 bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end(); 1624 if (HasNoTerminator) { 1625 if (Info->returnsVoid()) { 1626 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0); 1627 } else { 1628 BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG)); 1629 } 1630 } 1631 } 1632 } 1633 1634 unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) { 1635 switch (MI.getOpcode()) { 1636 default: return 1; // FIXME: Do wait states equal cycles? 1637 1638 case AMDGPU::S_NOP: 1639 return MI.getOperand(0).getImm() + 1; 1640 } 1641 } 1642 1643 bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { 1644 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 1645 MachineBasicBlock &MBB = *MI.getParent(); 1646 DebugLoc DL = MBB.findDebugLoc(MI); 1647 switch (MI.getOpcode()) { 1648 default: return TargetInstrInfo::expandPostRAPseudo(MI); 1649 case AMDGPU::S_MOV_B64_term: 1650 // This is only a terminator to get the correct spill code placement during 1651 // register allocation. 1652 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1653 break; 1654 1655 case AMDGPU::S_MOV_B32_term: 1656 // This is only a terminator to get the correct spill code placement during 1657 // register allocation. 1658 MI.setDesc(get(AMDGPU::S_MOV_B32)); 1659 break; 1660 1661 case AMDGPU::S_XOR_B64_term: 1662 // This is only a terminator to get the correct spill code placement during 1663 // register allocation. 1664 MI.setDesc(get(AMDGPU::S_XOR_B64)); 1665 break; 1666 1667 case AMDGPU::S_XOR_B32_term: 1668 // This is only a terminator to get the correct spill code placement during 1669 // register allocation. 1670 MI.setDesc(get(AMDGPU::S_XOR_B32)); 1671 break; 1672 case AMDGPU::S_OR_B64_term: 1673 // This is only a terminator to get the correct spill code placement during 1674 // register allocation. 1675 MI.setDesc(get(AMDGPU::S_OR_B64)); 1676 break; 1677 case AMDGPU::S_OR_B32_term: 1678 // This is only a terminator to get the correct spill code placement during 1679 // register allocation. 1680 MI.setDesc(get(AMDGPU::S_OR_B32)); 1681 break; 1682 1683 case AMDGPU::S_ANDN2_B64_term: 1684 // This is only a terminator to get the correct spill code placement during 1685 // register allocation. 1686 MI.setDesc(get(AMDGPU::S_ANDN2_B64)); 1687 break; 1688 1689 case AMDGPU::S_ANDN2_B32_term: 1690 // This is only a terminator to get the correct spill code placement during 1691 // register allocation. 1692 MI.setDesc(get(AMDGPU::S_ANDN2_B32)); 1693 break; 1694 1695 case AMDGPU::S_AND_B64_term: 1696 // This is only a terminator to get the correct spill code placement during 1697 // register allocation. 1698 MI.setDesc(get(AMDGPU::S_AND_B64)); 1699 break; 1700 1701 case AMDGPU::S_AND_B32_term: 1702 // This is only a terminator to get the correct spill code placement during 1703 // register allocation. 1704 MI.setDesc(get(AMDGPU::S_AND_B32)); 1705 break; 1706 1707 case AMDGPU::V_MOV_B64_PSEUDO: { 1708 Register Dst = MI.getOperand(0).getReg(); 1709 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1710 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1711 1712 const MachineOperand &SrcOp = MI.getOperand(1); 1713 // FIXME: Will this work for 64-bit floating point immediates? 1714 assert(!SrcOp.isFPImm()); 1715 if (SrcOp.isImm()) { 1716 APInt Imm(64, SrcOp.getImm()); 1717 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 1718 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 1719 if (ST.hasPackedFP32Ops() && Lo == Hi && isInlineConstant(Lo)) { 1720 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1721 .addImm(SISrcMods::OP_SEL_1) 1722 .addImm(Lo.getSExtValue()) 1723 .addImm(SISrcMods::OP_SEL_1) 1724 .addImm(Lo.getSExtValue()) 1725 .addImm(0) // op_sel_lo 1726 .addImm(0) // op_sel_hi 1727 .addImm(0) // neg_lo 1728 .addImm(0) // neg_hi 1729 .addImm(0); // clamp 1730 } else { 1731 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1732 .addImm(Lo.getSExtValue()) 1733 .addReg(Dst, RegState::Implicit | RegState::Define); 1734 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1735 .addImm(Hi.getSExtValue()) 1736 .addReg(Dst, RegState::Implicit | RegState::Define); 1737 } 1738 } else { 1739 assert(SrcOp.isReg()); 1740 if (ST.hasPackedFP32Ops() && 1741 !RI.isAGPR(MBB.getParent()->getRegInfo(), SrcOp.getReg())) { 1742 BuildMI(MBB, MI, DL, get(AMDGPU::V_PK_MOV_B32), Dst) 1743 .addImm(SISrcMods::OP_SEL_1) // src0_mod 1744 .addReg(SrcOp.getReg()) 1745 .addImm(SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1) // src1_mod 1746 .addReg(SrcOp.getReg()) 1747 .addImm(0) // op_sel_lo 1748 .addImm(0) // op_sel_hi 1749 .addImm(0) // neg_lo 1750 .addImm(0) // neg_hi 1751 .addImm(0); // clamp 1752 } else { 1753 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo) 1754 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0)) 1755 .addReg(Dst, RegState::Implicit | RegState::Define); 1756 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi) 1757 .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1)) 1758 .addReg(Dst, RegState::Implicit | RegState::Define); 1759 } 1760 } 1761 MI.eraseFromParent(); 1762 break; 1763 } 1764 case AMDGPU::V_MOV_B64_DPP_PSEUDO: { 1765 expandMovDPP64(MI); 1766 break; 1767 } 1768 case AMDGPU::S_MOV_B64_IMM_PSEUDO: { 1769 const MachineOperand &SrcOp = MI.getOperand(1); 1770 assert(!SrcOp.isFPImm()); 1771 APInt Imm(64, SrcOp.getImm()); 1772 if (Imm.isIntN(32) || isInlineConstant(Imm)) { 1773 MI.setDesc(get(AMDGPU::S_MOV_B64)); 1774 break; 1775 } 1776 1777 Register Dst = MI.getOperand(0).getReg(); 1778 Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0); 1779 Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1); 1780 1781 APInt Lo(32, Imm.getLoBits(32).getZExtValue()); 1782 APInt Hi(32, Imm.getHiBits(32).getZExtValue()); 1783 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstLo) 1784 .addImm(Lo.getSExtValue()) 1785 .addReg(Dst, RegState::Implicit | RegState::Define); 1786 BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DstHi) 1787 .addImm(Hi.getSExtValue()) 1788 .addReg(Dst, RegState::Implicit | RegState::Define); 1789 MI.eraseFromParent(); 1790 break; 1791 } 1792 case AMDGPU::V_SET_INACTIVE_B32: { 1793 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1794 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1795 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 1796 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 1797 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg()) 1798 .add(MI.getOperand(2)); 1799 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1800 .addReg(Exec); 1801 MI.eraseFromParent(); 1802 break; 1803 } 1804 case AMDGPU::V_SET_INACTIVE_B64: { 1805 unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64; 1806 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1807 auto FirstNot = BuildMI(MBB, MI, DL, get(NotOpc), Exec).addReg(Exec); 1808 FirstNot->addRegisterDead(AMDGPU::SCC, TRI); // SCC is overwritten 1809 MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), 1810 MI.getOperand(0).getReg()) 1811 .add(MI.getOperand(2)); 1812 expandPostRAPseudo(*Copy); 1813 BuildMI(MBB, MI, DL, get(NotOpc), Exec) 1814 .addReg(Exec); 1815 MI.eraseFromParent(); 1816 break; 1817 } 1818 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1819 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1820 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1821 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1822 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1823 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1824 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1825 case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1826 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1: 1827 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2: 1828 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3: 1829 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4: 1830 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5: 1831 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8: 1832 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16: 1833 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32: 1834 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1: 1835 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2: 1836 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4: 1837 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8: 1838 case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: { 1839 const TargetRegisterClass *EltRC = getOpRegClass(MI, 2); 1840 1841 unsigned Opc; 1842 if (RI.hasVGPRs(EltRC)) { 1843 Opc = AMDGPU::V_MOVRELD_B32_e32; 1844 } else { 1845 Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64 1846 : AMDGPU::S_MOVRELD_B32; 1847 } 1848 1849 const MCInstrDesc &OpDesc = get(Opc); 1850 Register VecReg = MI.getOperand(0).getReg(); 1851 bool IsUndef = MI.getOperand(1).isUndef(); 1852 unsigned SubReg = MI.getOperand(3).getImm(); 1853 assert(VecReg == MI.getOperand(1).getReg()); 1854 1855 MachineInstrBuilder MIB = 1856 BuildMI(MBB, MI, DL, OpDesc) 1857 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1858 .add(MI.getOperand(2)) 1859 .addReg(VecReg, RegState::ImplicitDefine) 1860 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1861 1862 const int ImpDefIdx = 1863 OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1864 const int ImpUseIdx = ImpDefIdx + 1; 1865 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1866 MI.eraseFromParent(); 1867 break; 1868 } 1869 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1: 1870 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2: 1871 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3: 1872 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4: 1873 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5: 1874 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8: 1875 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16: 1876 case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: { 1877 assert(ST.useVGPRIndexMode()); 1878 Register VecReg = MI.getOperand(0).getReg(); 1879 bool IsUndef = MI.getOperand(1).isUndef(); 1880 Register Idx = MI.getOperand(3).getReg(); 1881 Register SubReg = MI.getOperand(4).getImm(); 1882 1883 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1884 .addReg(Idx) 1885 .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE); 1886 SetOn->getOperand(3).setIsUndef(); 1887 1888 const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect); 1889 MachineInstrBuilder MIB = 1890 BuildMI(MBB, MI, DL, OpDesc) 1891 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1892 .add(MI.getOperand(2)) 1893 .addReg(VecReg, RegState::ImplicitDefine) 1894 .addReg(VecReg, 1895 RegState::Implicit | (IsUndef ? RegState::Undef : 0)); 1896 1897 const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses(); 1898 const int ImpUseIdx = ImpDefIdx + 1; 1899 MIB->tieOperands(ImpDefIdx, ImpUseIdx); 1900 1901 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1902 1903 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1904 1905 MI.eraseFromParent(); 1906 break; 1907 } 1908 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1: 1909 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2: 1910 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3: 1911 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4: 1912 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5: 1913 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8: 1914 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16: 1915 case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: { 1916 assert(ST.useVGPRIndexMode()); 1917 Register Dst = MI.getOperand(0).getReg(); 1918 Register VecReg = MI.getOperand(1).getReg(); 1919 bool IsUndef = MI.getOperand(1).isUndef(); 1920 Register Idx = MI.getOperand(2).getReg(); 1921 Register SubReg = MI.getOperand(3).getImm(); 1922 1923 MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON)) 1924 .addReg(Idx) 1925 .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); 1926 SetOn->getOperand(3).setIsUndef(); 1927 1928 BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32)) 1929 .addDef(Dst) 1930 .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef) 1931 .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0)) 1932 .addReg(AMDGPU::M0, RegState::Implicit); 1933 1934 MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF)); 1935 1936 finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator())); 1937 1938 MI.eraseFromParent(); 1939 break; 1940 } 1941 case AMDGPU::SI_PC_ADD_REL_OFFSET: { 1942 MachineFunction &MF = *MBB.getParent(); 1943 Register Reg = MI.getOperand(0).getReg(); 1944 Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0); 1945 Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1); 1946 1947 // Create a bundle so these instructions won't be re-ordered by the 1948 // post-RA scheduler. 1949 MIBundleBuilder Bundler(MBB, MI); 1950 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg)); 1951 1952 // Add 32-bit offset from this instruction to the start of the 1953 // constant data. 1954 Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo) 1955 .addReg(RegLo) 1956 .add(MI.getOperand(1))); 1957 1958 MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi) 1959 .addReg(RegHi); 1960 MIB.add(MI.getOperand(2)); 1961 1962 Bundler.append(MIB); 1963 finalizeBundle(MBB, Bundler.begin()); 1964 1965 MI.eraseFromParent(); 1966 break; 1967 } 1968 case AMDGPU::ENTER_STRICT_WWM: { 1969 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1970 // Whole Wave Mode is entered. 1971 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32 1972 : AMDGPU::S_OR_SAVEEXEC_B64)); 1973 break; 1974 } 1975 case AMDGPU::ENTER_STRICT_WQM: { 1976 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1977 // STRICT_WQM is entered. 1978 const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 1979 const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64; 1980 const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 1981 BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec); 1982 BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec); 1983 1984 MI.eraseFromParent(); 1985 break; 1986 } 1987 case AMDGPU::EXIT_STRICT_WWM: 1988 case AMDGPU::EXIT_STRICT_WQM: { 1989 // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when 1990 // WWM/STICT_WQM is exited. 1991 MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64)); 1992 break; 1993 } 1994 } 1995 return true; 1996 } 1997 1998 std::pair<MachineInstr*, MachineInstr*> 1999 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const { 2000 assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO); 2001 2002 MachineBasicBlock &MBB = *MI.getParent(); 2003 DebugLoc DL = MBB.findDebugLoc(MI); 2004 MachineFunction *MF = MBB.getParent(); 2005 MachineRegisterInfo &MRI = MF->getRegInfo(); 2006 Register Dst = MI.getOperand(0).getReg(); 2007 unsigned Part = 0; 2008 MachineInstr *Split[2]; 2009 2010 for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) { 2011 auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp)); 2012 if (Dst.isPhysical()) { 2013 MovDPP.addDef(RI.getSubReg(Dst, Sub)); 2014 } else { 2015 assert(MRI.isSSA()); 2016 auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 2017 MovDPP.addDef(Tmp); 2018 } 2019 2020 for (unsigned I = 1; I <= 2; ++I) { // old and src operands. 2021 const MachineOperand &SrcOp = MI.getOperand(I); 2022 assert(!SrcOp.isFPImm()); 2023 if (SrcOp.isImm()) { 2024 APInt Imm(64, SrcOp.getImm()); 2025 Imm.ashrInPlace(Part * 32); 2026 MovDPP.addImm(Imm.getLoBits(32).getZExtValue()); 2027 } else { 2028 assert(SrcOp.isReg()); 2029 Register Src = SrcOp.getReg(); 2030 if (Src.isPhysical()) 2031 MovDPP.addReg(RI.getSubReg(Src, Sub)); 2032 else 2033 MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub); 2034 } 2035 } 2036 2037 for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I) 2038 MovDPP.addImm(MI.getOperand(I).getImm()); 2039 2040 Split[Part] = MovDPP; 2041 ++Part; 2042 } 2043 2044 if (Dst.isVirtual()) 2045 BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst) 2046 .addReg(Split[0]->getOperand(0).getReg()) 2047 .addImm(AMDGPU::sub0) 2048 .addReg(Split[1]->getOperand(0).getReg()) 2049 .addImm(AMDGPU::sub1); 2050 2051 MI.eraseFromParent(); 2052 return std::make_pair(Split[0], Split[1]); 2053 } 2054 2055 bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI, 2056 MachineOperand &Src0, 2057 unsigned Src0OpName, 2058 MachineOperand &Src1, 2059 unsigned Src1OpName) const { 2060 MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName); 2061 if (!Src0Mods) 2062 return false; 2063 2064 MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName); 2065 assert(Src1Mods && 2066 "All commutable instructions have both src0 and src1 modifiers"); 2067 2068 int Src0ModsVal = Src0Mods->getImm(); 2069 int Src1ModsVal = Src1Mods->getImm(); 2070 2071 Src1Mods->setImm(Src0ModsVal); 2072 Src0Mods->setImm(Src1ModsVal); 2073 return true; 2074 } 2075 2076 static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI, 2077 MachineOperand &RegOp, 2078 MachineOperand &NonRegOp) { 2079 Register Reg = RegOp.getReg(); 2080 unsigned SubReg = RegOp.getSubReg(); 2081 bool IsKill = RegOp.isKill(); 2082 bool IsDead = RegOp.isDead(); 2083 bool IsUndef = RegOp.isUndef(); 2084 bool IsDebug = RegOp.isDebug(); 2085 2086 if (NonRegOp.isImm()) 2087 RegOp.ChangeToImmediate(NonRegOp.getImm()); 2088 else if (NonRegOp.isFI()) 2089 RegOp.ChangeToFrameIndex(NonRegOp.getIndex()); 2090 else if (NonRegOp.isGlobal()) { 2091 RegOp.ChangeToGA(NonRegOp.getGlobal(), NonRegOp.getOffset(), 2092 NonRegOp.getTargetFlags()); 2093 } else 2094 return nullptr; 2095 2096 // Make sure we don't reinterpret a subreg index in the target flags. 2097 RegOp.setTargetFlags(NonRegOp.getTargetFlags()); 2098 2099 NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug); 2100 NonRegOp.setSubReg(SubReg); 2101 2102 return &MI; 2103 } 2104 2105 MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI, 2106 unsigned Src0Idx, 2107 unsigned Src1Idx) const { 2108 assert(!NewMI && "this should never be used"); 2109 2110 unsigned Opc = MI.getOpcode(); 2111 int CommutedOpcode = commuteOpcode(Opc); 2112 if (CommutedOpcode == -1) 2113 return nullptr; 2114 2115 assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) == 2116 static_cast<int>(Src0Idx) && 2117 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) == 2118 static_cast<int>(Src1Idx) && 2119 "inconsistency with findCommutedOpIndices"); 2120 2121 MachineOperand &Src0 = MI.getOperand(Src0Idx); 2122 MachineOperand &Src1 = MI.getOperand(Src1Idx); 2123 2124 MachineInstr *CommutedMI = nullptr; 2125 if (Src0.isReg() && Src1.isReg()) { 2126 if (isOperandLegal(MI, Src1Idx, &Src0)) { 2127 // Be sure to copy the source modifiers to the right place. 2128 CommutedMI 2129 = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx); 2130 } 2131 2132 } else if (Src0.isReg() && !Src1.isReg()) { 2133 // src0 should always be able to support any operand type, so no need to 2134 // check operand legality. 2135 CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1); 2136 } else if (!Src0.isReg() && Src1.isReg()) { 2137 if (isOperandLegal(MI, Src1Idx, &Src0)) 2138 CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0); 2139 } else { 2140 // FIXME: Found two non registers to commute. This does happen. 2141 return nullptr; 2142 } 2143 2144 if (CommutedMI) { 2145 swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers, 2146 Src1, AMDGPU::OpName::src1_modifiers); 2147 2148 CommutedMI->setDesc(get(CommutedOpcode)); 2149 } 2150 2151 return CommutedMI; 2152 } 2153 2154 // This needs to be implemented because the source modifiers may be inserted 2155 // between the true commutable operands, and the base 2156 // TargetInstrInfo::commuteInstruction uses it. 2157 bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI, 2158 unsigned &SrcOpIdx0, 2159 unsigned &SrcOpIdx1) const { 2160 return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1); 2161 } 2162 2163 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0, 2164 unsigned &SrcOpIdx1) const { 2165 if (!Desc.isCommutable()) 2166 return false; 2167 2168 unsigned Opc = Desc.getOpcode(); 2169 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 2170 if (Src0Idx == -1) 2171 return false; 2172 2173 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 2174 if (Src1Idx == -1) 2175 return false; 2176 2177 return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx); 2178 } 2179 2180 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp, 2181 int64_t BrOffset) const { 2182 // BranchRelaxation should never have to check s_setpc_b64 because its dest 2183 // block is unanalyzable. 2184 assert(BranchOp != AMDGPU::S_SETPC_B64); 2185 2186 // Convert to dwords. 2187 BrOffset /= 4; 2188 2189 // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is 2190 // from the next instruction. 2191 BrOffset -= 1; 2192 2193 return isIntN(BranchOffsetBits, BrOffset); 2194 } 2195 2196 MachineBasicBlock *SIInstrInfo::getBranchDestBlock( 2197 const MachineInstr &MI) const { 2198 if (MI.getOpcode() == AMDGPU::S_SETPC_B64) { 2199 // This would be a difficult analysis to perform, but can always be legal so 2200 // there's no need to analyze it. 2201 return nullptr; 2202 } 2203 2204 return MI.getOperand(0).getMBB(); 2205 } 2206 2207 unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB, 2208 MachineBasicBlock &DestBB, 2209 const DebugLoc &DL, 2210 int64_t BrOffset, 2211 RegScavenger *RS) const { 2212 assert(RS && "RegScavenger required for long branching"); 2213 assert(MBB.empty() && 2214 "new block should be inserted for expanding unconditional branch"); 2215 assert(MBB.pred_size() == 1); 2216 2217 MachineFunction *MF = MBB.getParent(); 2218 MachineRegisterInfo &MRI = MF->getRegInfo(); 2219 2220 // FIXME: Virtual register workaround for RegScavenger not working with empty 2221 // blocks. 2222 Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 2223 2224 auto I = MBB.end(); 2225 2226 // We need to compute the offset relative to the instruction immediately after 2227 // s_getpc_b64. Insert pc arithmetic code before last terminator. 2228 MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg); 2229 2230 // TODO: Handle > 32-bit block address. 2231 if (BrOffset >= 0) { 2232 BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32)) 2233 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2234 .addReg(PCReg, 0, AMDGPU::sub0) 2235 .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD); 2236 BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32)) 2237 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2238 .addReg(PCReg, 0, AMDGPU::sub1) 2239 .addImm(0); 2240 } else { 2241 // Backwards branch. 2242 BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32)) 2243 .addReg(PCReg, RegState::Define, AMDGPU::sub0) 2244 .addReg(PCReg, 0, AMDGPU::sub0) 2245 .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD); 2246 BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32)) 2247 .addReg(PCReg, RegState::Define, AMDGPU::sub1) 2248 .addReg(PCReg, 0, AMDGPU::sub1) 2249 .addImm(0); 2250 } 2251 2252 // Insert the indirect branch after the other terminator. 2253 BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64)) 2254 .addReg(PCReg); 2255 2256 // FIXME: If spilling is necessary, this will fail because this scavenger has 2257 // no emergency stack slots. It is non-trivial to spill in this situation, 2258 // because the restore code needs to be specially placed after the 2259 // jump. BranchRelaxation then needs to be made aware of the newly inserted 2260 // block. 2261 // 2262 // If a spill is needed for the pc register pair, we need to insert a spill 2263 // restore block right before the destination block, and insert a short branch 2264 // into the old destination block's fallthrough predecessor. 2265 // e.g.: 2266 // 2267 // s_cbranch_scc0 skip_long_branch: 2268 // 2269 // long_branch_bb: 2270 // spill s[8:9] 2271 // s_getpc_b64 s[8:9] 2272 // s_add_u32 s8, s8, restore_bb 2273 // s_addc_u32 s9, s9, 0 2274 // s_setpc_b64 s[8:9] 2275 // 2276 // skip_long_branch: 2277 // foo; 2278 // 2279 // ..... 2280 // 2281 // dest_bb_fallthrough_predecessor: 2282 // bar; 2283 // s_branch dest_bb 2284 // 2285 // restore_bb: 2286 // restore s[8:9] 2287 // fallthrough dest_bb 2288 /// 2289 // dest_bb: 2290 // buzz; 2291 2292 RS->enterBasicBlockEnd(MBB); 2293 Register Scav = RS->scavengeRegisterBackwards( 2294 AMDGPU::SReg_64RegClass, 2295 MachineBasicBlock::iterator(GetPC), false, 0); 2296 MRI.replaceRegWith(PCReg, Scav); 2297 MRI.clearVirtRegs(); 2298 RS->setRegUsed(Scav); 2299 2300 return 4 + 8 + 4 + 4; 2301 } 2302 2303 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) { 2304 switch (Cond) { 2305 case SIInstrInfo::SCC_TRUE: 2306 return AMDGPU::S_CBRANCH_SCC1; 2307 case SIInstrInfo::SCC_FALSE: 2308 return AMDGPU::S_CBRANCH_SCC0; 2309 case SIInstrInfo::VCCNZ: 2310 return AMDGPU::S_CBRANCH_VCCNZ; 2311 case SIInstrInfo::VCCZ: 2312 return AMDGPU::S_CBRANCH_VCCZ; 2313 case SIInstrInfo::EXECNZ: 2314 return AMDGPU::S_CBRANCH_EXECNZ; 2315 case SIInstrInfo::EXECZ: 2316 return AMDGPU::S_CBRANCH_EXECZ; 2317 default: 2318 llvm_unreachable("invalid branch predicate"); 2319 } 2320 } 2321 2322 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) { 2323 switch (Opcode) { 2324 case AMDGPU::S_CBRANCH_SCC0: 2325 return SCC_FALSE; 2326 case AMDGPU::S_CBRANCH_SCC1: 2327 return SCC_TRUE; 2328 case AMDGPU::S_CBRANCH_VCCNZ: 2329 return VCCNZ; 2330 case AMDGPU::S_CBRANCH_VCCZ: 2331 return VCCZ; 2332 case AMDGPU::S_CBRANCH_EXECNZ: 2333 return EXECNZ; 2334 case AMDGPU::S_CBRANCH_EXECZ: 2335 return EXECZ; 2336 default: 2337 return INVALID_BR; 2338 } 2339 } 2340 2341 bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB, 2342 MachineBasicBlock::iterator I, 2343 MachineBasicBlock *&TBB, 2344 MachineBasicBlock *&FBB, 2345 SmallVectorImpl<MachineOperand> &Cond, 2346 bool AllowModify) const { 2347 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2348 // Unconditional Branch 2349 TBB = I->getOperand(0).getMBB(); 2350 return false; 2351 } 2352 2353 MachineBasicBlock *CondBB = nullptr; 2354 2355 if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 2356 CondBB = I->getOperand(1).getMBB(); 2357 Cond.push_back(I->getOperand(0)); 2358 } else { 2359 BranchPredicate Pred = getBranchPredicate(I->getOpcode()); 2360 if (Pred == INVALID_BR) 2361 return true; 2362 2363 CondBB = I->getOperand(0).getMBB(); 2364 Cond.push_back(MachineOperand::CreateImm(Pred)); 2365 Cond.push_back(I->getOperand(1)); // Save the branch register. 2366 } 2367 ++I; 2368 2369 if (I == MBB.end()) { 2370 // Conditional branch followed by fall-through. 2371 TBB = CondBB; 2372 return false; 2373 } 2374 2375 if (I->getOpcode() == AMDGPU::S_BRANCH) { 2376 TBB = CondBB; 2377 FBB = I->getOperand(0).getMBB(); 2378 return false; 2379 } 2380 2381 return true; 2382 } 2383 2384 bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, 2385 MachineBasicBlock *&FBB, 2386 SmallVectorImpl<MachineOperand> &Cond, 2387 bool AllowModify) const { 2388 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2389 auto E = MBB.end(); 2390 if (I == E) 2391 return false; 2392 2393 // Skip over the instructions that are artificially terminators for special 2394 // exec management. 2395 while (I != E && !I->isBranch() && !I->isReturn()) { 2396 switch (I->getOpcode()) { 2397 case AMDGPU::S_MOV_B64_term: 2398 case AMDGPU::S_XOR_B64_term: 2399 case AMDGPU::S_OR_B64_term: 2400 case AMDGPU::S_ANDN2_B64_term: 2401 case AMDGPU::S_AND_B64_term: 2402 case AMDGPU::S_MOV_B32_term: 2403 case AMDGPU::S_XOR_B32_term: 2404 case AMDGPU::S_OR_B32_term: 2405 case AMDGPU::S_ANDN2_B32_term: 2406 case AMDGPU::S_AND_B32_term: 2407 break; 2408 case AMDGPU::SI_IF: 2409 case AMDGPU::SI_ELSE: 2410 case AMDGPU::SI_KILL_I1_TERMINATOR: 2411 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 2412 // FIXME: It's messy that these need to be considered here at all. 2413 return true; 2414 default: 2415 llvm_unreachable("unexpected non-branch terminator inst"); 2416 } 2417 2418 ++I; 2419 } 2420 2421 if (I == E) 2422 return false; 2423 2424 return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify); 2425 } 2426 2427 unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB, 2428 int *BytesRemoved) const { 2429 MachineBasicBlock::iterator I = MBB.getFirstTerminator(); 2430 2431 unsigned Count = 0; 2432 unsigned RemovedSize = 0; 2433 while (I != MBB.end()) { 2434 MachineBasicBlock::iterator Next = std::next(I); 2435 RemovedSize += getInstSizeInBytes(*I); 2436 I->eraseFromParent(); 2437 ++Count; 2438 I = Next; 2439 } 2440 2441 if (BytesRemoved) 2442 *BytesRemoved = RemovedSize; 2443 2444 return Count; 2445 } 2446 2447 // Copy the flags onto the implicit condition register operand. 2448 static void preserveCondRegFlags(MachineOperand &CondReg, 2449 const MachineOperand &OrigCond) { 2450 CondReg.setIsUndef(OrigCond.isUndef()); 2451 CondReg.setIsKill(OrigCond.isKill()); 2452 } 2453 2454 unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB, 2455 MachineBasicBlock *TBB, 2456 MachineBasicBlock *FBB, 2457 ArrayRef<MachineOperand> Cond, 2458 const DebugLoc &DL, 2459 int *BytesAdded) const { 2460 if (!FBB && Cond.empty()) { 2461 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2462 .addMBB(TBB); 2463 if (BytesAdded) 2464 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2465 return 1; 2466 } 2467 2468 if(Cond.size() == 1 && Cond[0].isReg()) { 2469 BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO)) 2470 .add(Cond[0]) 2471 .addMBB(TBB); 2472 return 1; 2473 } 2474 2475 assert(TBB && Cond[0].isImm()); 2476 2477 unsigned Opcode 2478 = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm())); 2479 2480 if (!FBB) { 2481 Cond[1].isUndef(); 2482 MachineInstr *CondBr = 2483 BuildMI(&MBB, DL, get(Opcode)) 2484 .addMBB(TBB); 2485 2486 // Copy the flags onto the implicit condition register operand. 2487 preserveCondRegFlags(CondBr->getOperand(1), Cond[1]); 2488 fixImplicitOperands(*CondBr); 2489 2490 if (BytesAdded) 2491 *BytesAdded = ST.hasOffset3fBug() ? 8 : 4; 2492 return 1; 2493 } 2494 2495 assert(TBB && FBB); 2496 2497 MachineInstr *CondBr = 2498 BuildMI(&MBB, DL, get(Opcode)) 2499 .addMBB(TBB); 2500 fixImplicitOperands(*CondBr); 2501 BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH)) 2502 .addMBB(FBB); 2503 2504 MachineOperand &CondReg = CondBr->getOperand(1); 2505 CondReg.setIsUndef(Cond[1].isUndef()); 2506 CondReg.setIsKill(Cond[1].isKill()); 2507 2508 if (BytesAdded) 2509 *BytesAdded = ST.hasOffset3fBug() ? 16 : 8; 2510 2511 return 2; 2512 } 2513 2514 bool SIInstrInfo::reverseBranchCondition( 2515 SmallVectorImpl<MachineOperand> &Cond) const { 2516 if (Cond.size() != 2) { 2517 return true; 2518 } 2519 2520 if (Cond[0].isImm()) { 2521 Cond[0].setImm(-Cond[0].getImm()); 2522 return false; 2523 } 2524 2525 return true; 2526 } 2527 2528 bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB, 2529 ArrayRef<MachineOperand> Cond, 2530 Register DstReg, Register TrueReg, 2531 Register FalseReg, int &CondCycles, 2532 int &TrueCycles, int &FalseCycles) const { 2533 switch (Cond[0].getImm()) { 2534 case VCCNZ: 2535 case VCCZ: { 2536 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2537 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2538 if (MRI.getRegClass(FalseReg) != RC) 2539 return false; 2540 2541 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2542 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2543 2544 // Limit to equal cost for branch vs. N v_cndmask_b32s. 2545 return RI.hasVGPRs(RC) && NumInsts <= 6; 2546 } 2547 case SCC_TRUE: 2548 case SCC_FALSE: { 2549 // FIXME: We could insert for VGPRs if we could replace the original compare 2550 // with a vector one. 2551 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2552 const TargetRegisterClass *RC = MRI.getRegClass(TrueReg); 2553 if (MRI.getRegClass(FalseReg) != RC) 2554 return false; 2555 2556 int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32; 2557 2558 // Multiples of 8 can do s_cselect_b64 2559 if (NumInsts % 2 == 0) 2560 NumInsts /= 2; 2561 2562 CondCycles = TrueCycles = FalseCycles = NumInsts; // ??? 2563 return RI.isSGPRClass(RC); 2564 } 2565 default: 2566 return false; 2567 } 2568 } 2569 2570 void SIInstrInfo::insertSelect(MachineBasicBlock &MBB, 2571 MachineBasicBlock::iterator I, const DebugLoc &DL, 2572 Register DstReg, ArrayRef<MachineOperand> Cond, 2573 Register TrueReg, Register FalseReg) const { 2574 BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm()); 2575 if (Pred == VCCZ || Pred == SCC_FALSE) { 2576 Pred = static_cast<BranchPredicate>(-Pred); 2577 std::swap(TrueReg, FalseReg); 2578 } 2579 2580 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 2581 const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg); 2582 unsigned DstSize = RI.getRegSizeInBits(*DstRC); 2583 2584 if (DstSize == 32) { 2585 MachineInstr *Select; 2586 if (Pred == SCC_TRUE) { 2587 Select = BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B32), DstReg) 2588 .addReg(TrueReg) 2589 .addReg(FalseReg); 2590 } else { 2591 // Instruction's operands are backwards from what is expected. 2592 Select = BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e32), DstReg) 2593 .addReg(FalseReg) 2594 .addReg(TrueReg); 2595 } 2596 2597 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2598 return; 2599 } 2600 2601 if (DstSize == 64 && Pred == SCC_TRUE) { 2602 MachineInstr *Select = 2603 BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg) 2604 .addReg(TrueReg) 2605 .addReg(FalseReg); 2606 2607 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2608 return; 2609 } 2610 2611 static const int16_t Sub0_15[] = { 2612 AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3, 2613 AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7, 2614 AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11, 2615 AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15, 2616 }; 2617 2618 static const int16_t Sub0_15_64[] = { 2619 AMDGPU::sub0_sub1, AMDGPU::sub2_sub3, 2620 AMDGPU::sub4_sub5, AMDGPU::sub6_sub7, 2621 AMDGPU::sub8_sub9, AMDGPU::sub10_sub11, 2622 AMDGPU::sub12_sub13, AMDGPU::sub14_sub15, 2623 }; 2624 2625 unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32; 2626 const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass; 2627 const int16_t *SubIndices = Sub0_15; 2628 int NElts = DstSize / 32; 2629 2630 // 64-bit select is only available for SALU. 2631 // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit. 2632 if (Pred == SCC_TRUE) { 2633 if (NElts % 2) { 2634 SelOp = AMDGPU::S_CSELECT_B32; 2635 EltRC = &AMDGPU::SGPR_32RegClass; 2636 } else { 2637 SelOp = AMDGPU::S_CSELECT_B64; 2638 EltRC = &AMDGPU::SGPR_64RegClass; 2639 SubIndices = Sub0_15_64; 2640 NElts /= 2; 2641 } 2642 } 2643 2644 MachineInstrBuilder MIB = BuildMI( 2645 MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg); 2646 2647 I = MIB->getIterator(); 2648 2649 SmallVector<Register, 8> Regs; 2650 for (int Idx = 0; Idx != NElts; ++Idx) { 2651 Register DstElt = MRI.createVirtualRegister(EltRC); 2652 Regs.push_back(DstElt); 2653 2654 unsigned SubIdx = SubIndices[Idx]; 2655 2656 MachineInstr *Select; 2657 if (SelOp == AMDGPU::V_CNDMASK_B32_e32) { 2658 Select = 2659 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2660 .addReg(FalseReg, 0, SubIdx) 2661 .addReg(TrueReg, 0, SubIdx); 2662 } else { 2663 Select = 2664 BuildMI(MBB, I, DL, get(SelOp), DstElt) 2665 .addReg(TrueReg, 0, SubIdx) 2666 .addReg(FalseReg, 0, SubIdx); 2667 } 2668 2669 preserveCondRegFlags(Select->getOperand(3), Cond[1]); 2670 fixImplicitOperands(*Select); 2671 2672 MIB.addReg(DstElt) 2673 .addImm(SubIdx); 2674 } 2675 } 2676 2677 bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const { 2678 switch (MI.getOpcode()) { 2679 case AMDGPU::V_MOV_B32_e32: 2680 case AMDGPU::V_MOV_B32_e64: 2681 case AMDGPU::V_MOV_B64_PSEUDO: { 2682 // If there are additional implicit register operands, this may be used for 2683 // register indexing so the source register operand isn't simply copied. 2684 unsigned NumOps = MI.getDesc().getNumOperands() + 2685 MI.getDesc().getNumImplicitUses(); 2686 2687 return MI.getNumOperands() == NumOps; 2688 } 2689 case AMDGPU::S_MOV_B32: 2690 case AMDGPU::S_MOV_B64: 2691 case AMDGPU::COPY: 2692 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2693 case AMDGPU::V_ACCVGPR_READ_B32_e64: 2694 case AMDGPU::V_ACCVGPR_MOV_B32: 2695 return true; 2696 default: 2697 return false; 2698 } 2699 } 2700 2701 unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind( 2702 unsigned Kind) const { 2703 switch(Kind) { 2704 case PseudoSourceValue::Stack: 2705 case PseudoSourceValue::FixedStack: 2706 return AMDGPUAS::PRIVATE_ADDRESS; 2707 case PseudoSourceValue::ConstantPool: 2708 case PseudoSourceValue::GOT: 2709 case PseudoSourceValue::JumpTable: 2710 case PseudoSourceValue::GlobalValueCallEntry: 2711 case PseudoSourceValue::ExternalSymbolCallEntry: 2712 case PseudoSourceValue::TargetCustom: 2713 return AMDGPUAS::CONSTANT_ADDRESS; 2714 } 2715 return AMDGPUAS::FLAT_ADDRESS; 2716 } 2717 2718 static void removeModOperands(MachineInstr &MI) { 2719 unsigned Opc = MI.getOpcode(); 2720 int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2721 AMDGPU::OpName::src0_modifiers); 2722 int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2723 AMDGPU::OpName::src1_modifiers); 2724 int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc, 2725 AMDGPU::OpName::src2_modifiers); 2726 2727 MI.RemoveOperand(Src2ModIdx); 2728 MI.RemoveOperand(Src1ModIdx); 2729 MI.RemoveOperand(Src0ModIdx); 2730 } 2731 2732 bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, 2733 Register Reg, MachineRegisterInfo *MRI) const { 2734 if (!MRI->hasOneNonDBGUse(Reg)) 2735 return false; 2736 2737 switch (DefMI.getOpcode()) { 2738 default: 2739 return false; 2740 case AMDGPU::S_MOV_B64: 2741 // TODO: We could fold 64-bit immediates, but this get compilicated 2742 // when there are sub-registers. 2743 return false; 2744 2745 case AMDGPU::V_MOV_B32_e32: 2746 case AMDGPU::S_MOV_B32: 2747 case AMDGPU::V_ACCVGPR_WRITE_B32_e64: 2748 break; 2749 } 2750 2751 const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0); 2752 assert(ImmOp); 2753 // FIXME: We could handle FrameIndex values here. 2754 if (!ImmOp->isImm()) 2755 return false; 2756 2757 unsigned Opc = UseMI.getOpcode(); 2758 if (Opc == AMDGPU::COPY) { 2759 Register DstReg = UseMI.getOperand(0).getReg(); 2760 bool Is16Bit = getOpSize(UseMI, 0) == 2; 2761 bool isVGPRCopy = RI.isVGPR(*MRI, DstReg); 2762 unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32; 2763 APInt Imm(32, ImmOp->getImm()); 2764 2765 if (UseMI.getOperand(1).getSubReg() == AMDGPU::hi16) 2766 Imm = Imm.ashr(16); 2767 2768 if (RI.isAGPR(*MRI, DstReg)) { 2769 if (!isInlineConstant(Imm)) 2770 return false; 2771 NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64; 2772 } 2773 2774 if (Is16Bit) { 2775 if (isVGPRCopy) 2776 return false; // Do not clobber vgpr_hi16 2777 2778 if (DstReg.isVirtual() && 2779 UseMI.getOperand(0).getSubReg() != AMDGPU::lo16) 2780 return false; 2781 2782 UseMI.getOperand(0).setSubReg(0); 2783 if (DstReg.isPhysical()) { 2784 DstReg = RI.get32BitRegister(DstReg); 2785 UseMI.getOperand(0).setReg(DstReg); 2786 } 2787 assert(UseMI.getOperand(1).getReg().isVirtual()); 2788 } 2789 2790 UseMI.setDesc(get(NewOpc)); 2791 UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue()); 2792 UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent()); 2793 return true; 2794 } 2795 2796 if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2797 Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || 2798 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2799 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64) { 2800 // Don't fold if we are using source or output modifiers. The new VOP2 2801 // instructions don't have them. 2802 if (hasAnyModifiersSet(UseMI)) 2803 return false; 2804 2805 // If this is a free constant, there's no reason to do this. 2806 // TODO: We could fold this here instead of letting SIFoldOperands do it 2807 // later. 2808 MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0); 2809 2810 // Any src operand can be used for the legality check. 2811 if (isInlineConstant(UseMI, *Src0, *ImmOp)) 2812 return false; 2813 2814 bool IsF32 = Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || 2815 Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64; 2816 bool IsFMA = Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || 2817 Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64; 2818 MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1); 2819 MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2); 2820 2821 // Multiplied part is the constant: Use v_madmk_{f16, f32}. 2822 // We should only expect these to be on src0 due to canonicalizations. 2823 if (Src0->isReg() && Src0->getReg() == Reg) { 2824 if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))) 2825 return false; 2826 2827 if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg()))) 2828 return false; 2829 2830 unsigned NewOpc = 2831 IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16) 2832 : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16); 2833 if (pseudoToMCOpcode(NewOpc) == -1) 2834 return false; 2835 2836 // We need to swap operands 0 and 1 since madmk constant is at operand 1. 2837 2838 const int64_t Imm = ImmOp->getImm(); 2839 2840 // FIXME: This would be a lot easier if we could return a new instruction 2841 // instead of having to modify in place. 2842 2843 // Remove these first since they are at the end. 2844 UseMI.RemoveOperand( 2845 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2846 UseMI.RemoveOperand( 2847 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2848 2849 Register Src1Reg = Src1->getReg(); 2850 unsigned Src1SubReg = Src1->getSubReg(); 2851 Src0->setReg(Src1Reg); 2852 Src0->setSubReg(Src1SubReg); 2853 Src0->setIsKill(Src1->isKill()); 2854 2855 if (Opc == AMDGPU::V_MAC_F32_e64 || 2856 Opc == AMDGPU::V_MAC_F16_e64 || 2857 Opc == AMDGPU::V_FMAC_F32_e64 || 2858 Opc == AMDGPU::V_FMAC_F16_e64) 2859 UseMI.untieRegOperand( 2860 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2861 2862 Src1->ChangeToImmediate(Imm); 2863 2864 removeModOperands(UseMI); 2865 UseMI.setDesc(get(NewOpc)); 2866 2867 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2868 if (DeleteDef) 2869 DefMI.eraseFromParent(); 2870 2871 return true; 2872 } 2873 2874 // Added part is the constant: Use v_madak_{f16, f32}. 2875 if (Src2->isReg() && Src2->getReg() == Reg) { 2876 // Not allowed to use constant bus for another operand. 2877 // We can however allow an inline immediate as src0. 2878 bool Src0Inlined = false; 2879 if (Src0->isReg()) { 2880 // Try to inline constant if possible. 2881 // If the Def moves immediate and the use is single 2882 // We are saving VGPR here. 2883 MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg()); 2884 if (Def && Def->isMoveImmediate() && 2885 isInlineConstant(Def->getOperand(1)) && 2886 MRI->hasOneUse(Src0->getReg())) { 2887 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2888 Src0Inlined = true; 2889 } else if ((Src0->getReg().isPhysical() && 2890 (ST.getConstantBusLimit(Opc) <= 1 && 2891 RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) || 2892 (Src0->getReg().isVirtual() && 2893 (ST.getConstantBusLimit(Opc) <= 1 && 2894 RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))) 2895 return false; 2896 // VGPR is okay as Src0 - fallthrough 2897 } 2898 2899 if (Src1->isReg() && !Src0Inlined ) { 2900 // We have one slot for inlinable constant so far - try to fill it 2901 MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg()); 2902 if (Def && Def->isMoveImmediate() && 2903 isInlineConstant(Def->getOperand(1)) && 2904 MRI->hasOneUse(Src1->getReg()) && 2905 commuteInstruction(UseMI)) { 2906 Src0->ChangeToImmediate(Def->getOperand(1).getImm()); 2907 } else if ((Src1->getReg().isPhysical() && 2908 RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) || 2909 (Src1->getReg().isVirtual() && 2910 RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))) 2911 return false; 2912 // VGPR is okay as Src1 - fallthrough 2913 } 2914 2915 unsigned NewOpc = 2916 IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16) 2917 : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16); 2918 if (pseudoToMCOpcode(NewOpc) == -1) 2919 return false; 2920 2921 const int64_t Imm = ImmOp->getImm(); 2922 2923 // FIXME: This would be a lot easier if we could return a new instruction 2924 // instead of having to modify in place. 2925 2926 // Remove these first since they are at the end. 2927 UseMI.RemoveOperand( 2928 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod)); 2929 UseMI.RemoveOperand( 2930 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp)); 2931 2932 if (Opc == AMDGPU::V_MAC_F32_e64 || 2933 Opc == AMDGPU::V_MAC_F16_e64 || 2934 Opc == AMDGPU::V_FMAC_F32_e64 || 2935 Opc == AMDGPU::V_FMAC_F16_e64) 2936 UseMI.untieRegOperand( 2937 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)); 2938 2939 // ChangingToImmediate adds Src2 back to the instruction. 2940 Src2->ChangeToImmediate(Imm); 2941 2942 // These come before src2. 2943 removeModOperands(UseMI); 2944 UseMI.setDesc(get(NewOpc)); 2945 // It might happen that UseMI was commuted 2946 // and we now have SGPR as SRC1. If so 2 inlined 2947 // constant and SGPR are illegal. 2948 legalizeOperands(UseMI); 2949 2950 bool DeleteDef = MRI->hasOneNonDBGUse(Reg); 2951 if (DeleteDef) 2952 DefMI.eraseFromParent(); 2953 2954 return true; 2955 } 2956 } 2957 2958 return false; 2959 } 2960 2961 static bool 2962 memOpsHaveSameBaseOperands(ArrayRef<const MachineOperand *> BaseOps1, 2963 ArrayRef<const MachineOperand *> BaseOps2) { 2964 if (BaseOps1.size() != BaseOps2.size()) 2965 return false; 2966 for (size_t I = 0, E = BaseOps1.size(); I < E; ++I) { 2967 if (!BaseOps1[I]->isIdenticalTo(*BaseOps2[I])) 2968 return false; 2969 } 2970 return true; 2971 } 2972 2973 static bool offsetsDoNotOverlap(int WidthA, int OffsetA, 2974 int WidthB, int OffsetB) { 2975 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB; 2976 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA; 2977 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB; 2978 return LowOffset + LowWidth <= HighOffset; 2979 } 2980 2981 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, 2982 const MachineInstr &MIb) const { 2983 SmallVector<const MachineOperand *, 4> BaseOps0, BaseOps1; 2984 int64_t Offset0, Offset1; 2985 unsigned Dummy0, Dummy1; 2986 bool Offset0IsScalable, Offset1IsScalable; 2987 if (!getMemOperandsWithOffsetWidth(MIa, BaseOps0, Offset0, Offset0IsScalable, 2988 Dummy0, &RI) || 2989 !getMemOperandsWithOffsetWidth(MIb, BaseOps1, Offset1, Offset1IsScalable, 2990 Dummy1, &RI)) 2991 return false; 2992 2993 if (!memOpsHaveSameBaseOperands(BaseOps0, BaseOps1)) 2994 return false; 2995 2996 if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) { 2997 // FIXME: Handle ds_read2 / ds_write2. 2998 return false; 2999 } 3000 unsigned Width0 = MIa.memoperands().front()->getSize(); 3001 unsigned Width1 = MIb.memoperands().front()->getSize(); 3002 return offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1); 3003 } 3004 3005 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, 3006 const MachineInstr &MIb) const { 3007 assert(MIa.mayLoadOrStore() && 3008 "MIa must load from or modify a memory location"); 3009 assert(MIb.mayLoadOrStore() && 3010 "MIb must load from or modify a memory location"); 3011 3012 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) 3013 return false; 3014 3015 // XXX - Can we relax this between address spaces? 3016 if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef()) 3017 return false; 3018 3019 // TODO: Should we check the address space from the MachineMemOperand? That 3020 // would allow us to distinguish objects we know don't alias based on the 3021 // underlying address space, even if it was lowered to a different one, 3022 // e.g. private accesses lowered to use MUBUF instructions on a scratch 3023 // buffer. 3024 if (isDS(MIa)) { 3025 if (isDS(MIb)) 3026 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3027 3028 return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb); 3029 } 3030 3031 if (isMUBUF(MIa) || isMTBUF(MIa)) { 3032 if (isMUBUF(MIb) || isMTBUF(MIb)) 3033 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3034 3035 return !isFLAT(MIb) && !isSMRD(MIb); 3036 } 3037 3038 if (isSMRD(MIa)) { 3039 if (isSMRD(MIb)) 3040 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3041 3042 return !isFLAT(MIb) && !isMUBUF(MIb) && !isMTBUF(MIb); 3043 } 3044 3045 if (isFLAT(MIa)) { 3046 if (isFLAT(MIb)) 3047 return checkInstOffsetsDoNotOverlap(MIa, MIb); 3048 3049 return false; 3050 } 3051 3052 return false; 3053 } 3054 3055 static int64_t getFoldableImm(const MachineOperand* MO) { 3056 if (!MO->isReg()) 3057 return false; 3058 const MachineFunction *MF = MO->getParent()->getParent()->getParent(); 3059 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3060 auto Def = MRI.getUniqueVRegDef(MO->getReg()); 3061 if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 && 3062 Def->getOperand(1).isImm()) 3063 return Def->getOperand(1).getImm(); 3064 return AMDGPU::NoRegister; 3065 } 3066 3067 static void updateLiveVariables(LiveVariables *LV, MachineInstr &MI, 3068 MachineInstr &NewMI) { 3069 if (LV) { 3070 unsigned NumOps = MI.getNumOperands(); 3071 for (unsigned I = 1; I < NumOps; ++I) { 3072 MachineOperand &Op = MI.getOperand(I); 3073 if (Op.isReg() && Op.isKill()) 3074 LV->replaceKillInstruction(Op.getReg(), MI, NewMI); 3075 } 3076 } 3077 } 3078 3079 MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB, 3080 MachineInstr &MI, 3081 LiveVariables *LV) const { 3082 unsigned Opc = MI.getOpcode(); 3083 bool IsF16 = false; 3084 bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 || 3085 Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64 || 3086 Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3087 bool IsF64 = Opc == AMDGPU::V_FMAC_F64_e32 || Opc == AMDGPU::V_FMAC_F64_e64; 3088 3089 switch (Opc) { 3090 default: 3091 return nullptr; 3092 case AMDGPU::V_MAC_F16_e64: 3093 case AMDGPU::V_FMAC_F16_e64: 3094 IsF16 = true; 3095 LLVM_FALLTHROUGH; 3096 case AMDGPU::V_MAC_F32_e64: 3097 case AMDGPU::V_FMAC_F32_e64: 3098 case AMDGPU::V_FMAC_F64_e64: 3099 break; 3100 case AMDGPU::V_MAC_F16_e32: 3101 case AMDGPU::V_FMAC_F16_e32: 3102 IsF16 = true; 3103 LLVM_FALLTHROUGH; 3104 case AMDGPU::V_MAC_F32_e32: 3105 case AMDGPU::V_FMAC_F32_e32: 3106 case AMDGPU::V_FMAC_F64_e32: { 3107 int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3108 AMDGPU::OpName::src0); 3109 const MachineOperand *Src0 = &MI.getOperand(Src0Idx); 3110 if (!Src0->isReg() && !Src0->isImm()) 3111 return nullptr; 3112 3113 if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0)) 3114 return nullptr; 3115 3116 break; 3117 } 3118 } 3119 3120 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 3121 const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0); 3122 const MachineOperand *Src0Mods = 3123 getNamedOperand(MI, AMDGPU::OpName::src0_modifiers); 3124 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3125 const MachineOperand *Src1Mods = 3126 getNamedOperand(MI, AMDGPU::OpName::src1_modifiers); 3127 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3128 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3129 const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod); 3130 MachineInstrBuilder MIB; 3131 3132 if (!Src0Mods && !Src1Mods && !Clamp && !Omod && !IsF64 && 3133 // If we have an SGPR input, we will violate the constant bus restriction. 3134 (ST.getConstantBusLimit(Opc) > 1 || !Src0->isReg() || 3135 !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) { 3136 if (auto Imm = getFoldableImm(Src2)) { 3137 unsigned NewOpc = 3138 IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32) 3139 : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32); 3140 if (pseudoToMCOpcode(NewOpc) != -1) { 3141 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3142 .add(*Dst) 3143 .add(*Src0) 3144 .add(*Src1) 3145 .addImm(Imm); 3146 updateLiveVariables(LV, MI, *MIB); 3147 return MIB; 3148 } 3149 } 3150 unsigned NewOpc = IsFMA 3151 ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32) 3152 : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32); 3153 if (auto Imm = getFoldableImm(Src1)) { 3154 if (pseudoToMCOpcode(NewOpc) != -1) { 3155 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3156 .add(*Dst) 3157 .add(*Src0) 3158 .addImm(Imm) 3159 .add(*Src2); 3160 updateLiveVariables(LV, MI, *MIB); 3161 return MIB; 3162 } 3163 } 3164 if (auto Imm = getFoldableImm(Src0)) { 3165 if (pseudoToMCOpcode(NewOpc) != -1 && 3166 isOperandLegal( 3167 MI, AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::src0), 3168 Src1)) { 3169 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3170 .add(*Dst) 3171 .add(*Src1) 3172 .addImm(Imm) 3173 .add(*Src2); 3174 updateLiveVariables(LV, MI, *MIB); 3175 return MIB; 3176 } 3177 } 3178 } 3179 3180 unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16_e64 3181 : IsF64 ? AMDGPU::V_FMA_F64_e64 3182 : AMDGPU::V_FMA_F32_e64) 3183 : (IsF16 ? AMDGPU::V_MAD_F16_e64 : AMDGPU::V_MAD_F32_e64); 3184 if (pseudoToMCOpcode(NewOpc) == -1) 3185 return nullptr; 3186 3187 MIB = BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc)) 3188 .add(*Dst) 3189 .addImm(Src0Mods ? Src0Mods->getImm() : 0) 3190 .add(*Src0) 3191 .addImm(Src1Mods ? Src1Mods->getImm() : 0) 3192 .add(*Src1) 3193 .addImm(0) // Src mods 3194 .add(*Src2) 3195 .addImm(Clamp ? Clamp->getImm() : 0) 3196 .addImm(Omod ? Omod->getImm() : 0); 3197 updateLiveVariables(LV, MI, *MIB); 3198 return MIB; 3199 } 3200 3201 // It's not generally safe to move VALU instructions across these since it will 3202 // start using the register as a base index rather than directly. 3203 // XXX - Why isn't hasSideEffects sufficient for these? 3204 static bool changesVGPRIndexingMode(const MachineInstr &MI) { 3205 switch (MI.getOpcode()) { 3206 case AMDGPU::S_SET_GPR_IDX_ON: 3207 case AMDGPU::S_SET_GPR_IDX_MODE: 3208 case AMDGPU::S_SET_GPR_IDX_OFF: 3209 return true; 3210 default: 3211 return false; 3212 } 3213 } 3214 3215 bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI, 3216 const MachineBasicBlock *MBB, 3217 const MachineFunction &MF) const { 3218 // Skipping the check for SP writes in the base implementation. The reason it 3219 // was added was apparently due to compile time concerns. 3220 // 3221 // TODO: Do we really want this barrier? It triggers unnecessary hazard nops 3222 // but is probably avoidable. 3223 3224 // Copied from base implementation. 3225 // Terminators and labels can't be scheduled around. 3226 if (MI.isTerminator() || MI.isPosition()) 3227 return true; 3228 3229 // INLINEASM_BR can jump to another block 3230 if (MI.getOpcode() == TargetOpcode::INLINEASM_BR) 3231 return true; 3232 3233 // Target-independent instructions do not have an implicit-use of EXEC, even 3234 // when they operate on VGPRs. Treating EXEC modifications as scheduling 3235 // boundaries prevents incorrect movements of such instructions. 3236 return MI.modifiesRegister(AMDGPU::EXEC, &RI) || 3237 MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 || 3238 MI.getOpcode() == AMDGPU::S_SETREG_B32 || 3239 changesVGPRIndexingMode(MI); 3240 } 3241 3242 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { 3243 return Opcode == AMDGPU::DS_ORDERED_COUNT || 3244 Opcode == AMDGPU::DS_GWS_INIT || 3245 Opcode == AMDGPU::DS_GWS_SEMA_V || 3246 Opcode == AMDGPU::DS_GWS_SEMA_BR || 3247 Opcode == AMDGPU::DS_GWS_SEMA_P || 3248 Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL || 3249 Opcode == AMDGPU::DS_GWS_BARRIER; 3250 } 3251 3252 bool SIInstrInfo::modifiesModeRegister(const MachineInstr &MI) { 3253 // Skip the full operand and register alias search modifiesRegister 3254 // does. There's only a handful of instructions that touch this, it's only an 3255 // implicit def, and doesn't alias any other registers. 3256 if (const MCPhysReg *ImpDef = MI.getDesc().getImplicitDefs()) { 3257 for (; ImpDef && *ImpDef; ++ImpDef) { 3258 if (*ImpDef == AMDGPU::MODE) 3259 return true; 3260 } 3261 } 3262 3263 return false; 3264 } 3265 3266 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const { 3267 unsigned Opcode = MI.getOpcode(); 3268 3269 if (MI.mayStore() && isSMRD(MI)) 3270 return true; // scalar store or atomic 3271 3272 // This will terminate the function when other lanes may need to continue. 3273 if (MI.isReturn()) 3274 return true; 3275 3276 // These instructions cause shader I/O that may cause hardware lockups 3277 // when executed with an empty EXEC mask. 3278 // 3279 // Note: exp with VM = DONE = 0 is automatically skipped by hardware when 3280 // EXEC = 0, but checking for that case here seems not worth it 3281 // given the typical code patterns. 3282 if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT || 3283 isEXP(Opcode) || 3284 Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP || 3285 Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER) 3286 return true; 3287 3288 if (MI.isCall() || MI.isInlineAsm()) 3289 return true; // conservative assumption 3290 3291 // A mode change is a scalar operation that influences vector instructions. 3292 if (modifiesModeRegister(MI)) 3293 return true; 3294 3295 // These are like SALU instructions in terms of effects, so it's questionable 3296 // whether we should return true for those. 3297 // 3298 // However, executing them with EXEC = 0 causes them to operate on undefined 3299 // data, which we avoid by returning true here. 3300 if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 3301 Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32) 3302 return true; 3303 3304 return false; 3305 } 3306 3307 bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, 3308 const MachineInstr &MI) const { 3309 if (MI.isMetaInstruction()) 3310 return false; 3311 3312 // This won't read exec if this is an SGPR->SGPR copy. 3313 if (MI.isCopyLike()) { 3314 if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) 3315 return true; 3316 3317 // Make sure this isn't copying exec as a normal operand 3318 return MI.readsRegister(AMDGPU::EXEC, &RI); 3319 } 3320 3321 // Make a conservative assumption about the callee. 3322 if (MI.isCall()) 3323 return true; 3324 3325 // Be conservative with any unhandled generic opcodes. 3326 if (!isTargetSpecificOpcode(MI.getOpcode())) 3327 return true; 3328 3329 return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); 3330 } 3331 3332 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { 3333 switch (Imm.getBitWidth()) { 3334 case 1: // This likely will be a condition code mask. 3335 return true; 3336 3337 case 32: 3338 return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(), 3339 ST.hasInv2PiInlineImm()); 3340 case 64: 3341 return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(), 3342 ST.hasInv2PiInlineImm()); 3343 case 16: 3344 return ST.has16BitInsts() && 3345 AMDGPU::isInlinableLiteral16(Imm.getSExtValue(), 3346 ST.hasInv2PiInlineImm()); 3347 default: 3348 llvm_unreachable("invalid bitwidth"); 3349 } 3350 } 3351 3352 bool SIInstrInfo::isInlineConstant(const MachineOperand &MO, 3353 uint8_t OperandType) const { 3354 if (!MO.isImm() || 3355 OperandType < AMDGPU::OPERAND_SRC_FIRST || 3356 OperandType > AMDGPU::OPERAND_SRC_LAST) 3357 return false; 3358 3359 // MachineOperand provides no way to tell the true operand size, since it only 3360 // records a 64-bit value. We need to know the size to determine if a 32-bit 3361 // floating point immediate bit pattern is legal for an integer immediate. It 3362 // would be for any 32-bit integer operand, but would not be for a 64-bit one. 3363 3364 int64_t Imm = MO.getImm(); 3365 switch (OperandType) { 3366 case AMDGPU::OPERAND_REG_IMM_INT32: 3367 case AMDGPU::OPERAND_REG_IMM_FP32: 3368 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3369 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3370 case AMDGPU::OPERAND_REG_IMM_V2FP32: 3371 case AMDGPU::OPERAND_REG_INLINE_C_V2FP32: 3372 case AMDGPU::OPERAND_REG_IMM_V2INT32: 3373 case AMDGPU::OPERAND_REG_INLINE_C_V2INT32: 3374 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3375 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: { 3376 int32_t Trunc = static_cast<int32_t>(Imm); 3377 return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm()); 3378 } 3379 case AMDGPU::OPERAND_REG_IMM_INT64: 3380 case AMDGPU::OPERAND_REG_IMM_FP64: 3381 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3382 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3383 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: 3384 return AMDGPU::isInlinableLiteral64(MO.getImm(), 3385 ST.hasInv2PiInlineImm()); 3386 case AMDGPU::OPERAND_REG_IMM_INT16: 3387 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3388 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3389 // We would expect inline immediates to not be concerned with an integer/fp 3390 // distinction. However, in the case of 16-bit integer operations, the 3391 // "floating point" values appear to not work. It seems read the low 16-bits 3392 // of 32-bit immediates, which happens to always work for the integer 3393 // values. 3394 // 3395 // See llvm bugzilla 46302. 3396 // 3397 // TODO: Theoretically we could use op-sel to use the high bits of the 3398 // 32-bit FP values. 3399 return AMDGPU::isInlinableIntLiteral(Imm); 3400 case AMDGPU::OPERAND_REG_IMM_V2INT16: 3401 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: 3402 case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16: 3403 // This suffers the same problem as the scalar 16-bit cases. 3404 return AMDGPU::isInlinableIntLiteralV216(Imm); 3405 case AMDGPU::OPERAND_REG_IMM_FP16: 3406 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3407 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: { 3408 if (isInt<16>(Imm) || isUInt<16>(Imm)) { 3409 // A few special case instructions have 16-bit operands on subtargets 3410 // where 16-bit instructions are not legal. 3411 // TODO: Do the 32-bit immediates work? We shouldn't really need to handle 3412 // constants in these cases 3413 int16_t Trunc = static_cast<int16_t>(Imm); 3414 return ST.has16BitInsts() && 3415 AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm()); 3416 } 3417 3418 return false; 3419 } 3420 case AMDGPU::OPERAND_REG_IMM_V2FP16: 3421 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: 3422 case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: { 3423 uint32_t Trunc = static_cast<uint32_t>(Imm); 3424 return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm()); 3425 } 3426 default: 3427 llvm_unreachable("invalid bitwidth"); 3428 } 3429 } 3430 3431 bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO, 3432 const MCOperandInfo &OpInfo) const { 3433 switch (MO.getType()) { 3434 case MachineOperand::MO_Register: 3435 return false; 3436 case MachineOperand::MO_Immediate: 3437 return !isInlineConstant(MO, OpInfo); 3438 case MachineOperand::MO_FrameIndex: 3439 case MachineOperand::MO_MachineBasicBlock: 3440 case MachineOperand::MO_ExternalSymbol: 3441 case MachineOperand::MO_GlobalAddress: 3442 case MachineOperand::MO_MCSymbol: 3443 return true; 3444 default: 3445 llvm_unreachable("unexpected operand type"); 3446 } 3447 } 3448 3449 static bool compareMachineOp(const MachineOperand &Op0, 3450 const MachineOperand &Op1) { 3451 if (Op0.getType() != Op1.getType()) 3452 return false; 3453 3454 switch (Op0.getType()) { 3455 case MachineOperand::MO_Register: 3456 return Op0.getReg() == Op1.getReg(); 3457 case MachineOperand::MO_Immediate: 3458 return Op0.getImm() == Op1.getImm(); 3459 default: 3460 llvm_unreachable("Didn't expect to be comparing these operand types"); 3461 } 3462 } 3463 3464 bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo, 3465 const MachineOperand &MO) const { 3466 const MCInstrDesc &InstDesc = MI.getDesc(); 3467 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo]; 3468 3469 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 3470 3471 if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE) 3472 return true; 3473 3474 if (OpInfo.RegClass < 0) 3475 return false; 3476 3477 if (MO.isImm() && isInlineConstant(MO, OpInfo)) { 3478 if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() && 3479 OpNo ==(unsigned)AMDGPU::getNamedOperandIdx(MI.getOpcode(), 3480 AMDGPU::OpName::src2)) 3481 return false; 3482 return RI.opCanUseInlineConstant(OpInfo.OperandType); 3483 } 3484 3485 if (!RI.opCanUseLiteralConstant(OpInfo.OperandType)) 3486 return false; 3487 3488 if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo)) 3489 return true; 3490 3491 return ST.hasVOP3Literal(); 3492 } 3493 3494 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const { 3495 // GFX90A does not have V_MUL_LEGACY_F32_e32. 3496 if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts()) 3497 return false; 3498 3499 int Op32 = AMDGPU::getVOPe32(Opcode); 3500 if (Op32 == -1) 3501 return false; 3502 3503 return pseudoToMCOpcode(Op32) != -1; 3504 } 3505 3506 bool SIInstrInfo::hasModifiers(unsigned Opcode) const { 3507 // The src0_modifier operand is present on all instructions 3508 // that have modifiers. 3509 3510 return AMDGPU::getNamedOperandIdx(Opcode, 3511 AMDGPU::OpName::src0_modifiers) != -1; 3512 } 3513 3514 bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI, 3515 unsigned OpName) const { 3516 const MachineOperand *Mods = getNamedOperand(MI, OpName); 3517 return Mods && Mods->getImm(); 3518 } 3519 3520 bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const { 3521 return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) || 3522 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) || 3523 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) || 3524 hasModifiersSet(MI, AMDGPU::OpName::clamp) || 3525 hasModifiersSet(MI, AMDGPU::OpName::omod); 3526 } 3527 3528 bool SIInstrInfo::canShrink(const MachineInstr &MI, 3529 const MachineRegisterInfo &MRI) const { 3530 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3531 // Can't shrink instruction with three operands. 3532 // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add 3533 // a special case for it. It can only be shrunk if the third operand 3534 // is vcc, and src0_modifiers and src1_modifiers are not set. 3535 // We should handle this the same way we handle vopc, by addding 3536 // a register allocation hint pre-regalloc and then do the shrinking 3537 // post-regalloc. 3538 if (Src2) { 3539 switch (MI.getOpcode()) { 3540 default: return false; 3541 3542 case AMDGPU::V_ADDC_U32_e64: 3543 case AMDGPU::V_SUBB_U32_e64: 3544 case AMDGPU::V_SUBBREV_U32_e64: { 3545 const MachineOperand *Src1 3546 = getNamedOperand(MI, AMDGPU::OpName::src1); 3547 if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg())) 3548 return false; 3549 // Additional verification is needed for sdst/src2. 3550 return true; 3551 } 3552 case AMDGPU::V_MAC_F32_e64: 3553 case AMDGPU::V_MAC_F16_e64: 3554 case AMDGPU::V_FMAC_F32_e64: 3555 case AMDGPU::V_FMAC_F16_e64: 3556 case AMDGPU::V_FMAC_F64_e64: 3557 if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) || 3558 hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)) 3559 return false; 3560 break; 3561 3562 case AMDGPU::V_CNDMASK_B32_e64: 3563 break; 3564 } 3565 } 3566 3567 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3568 if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) || 3569 hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers))) 3570 return false; 3571 3572 // We don't need to check src0, all input types are legal, so just make sure 3573 // src0 isn't using any modifiers. 3574 if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers)) 3575 return false; 3576 3577 // Can it be shrunk to a valid 32 bit opcode? 3578 if (!hasVALU32BitEncoding(MI.getOpcode())) 3579 return false; 3580 3581 // Check output modifiers 3582 return !hasModifiersSet(MI, AMDGPU::OpName::omod) && 3583 !hasModifiersSet(MI, AMDGPU::OpName::clamp); 3584 } 3585 3586 // Set VCC operand with all flags from \p Orig, except for setting it as 3587 // implicit. 3588 static void copyFlagsToImplicitVCC(MachineInstr &MI, 3589 const MachineOperand &Orig) { 3590 3591 for (MachineOperand &Use : MI.implicit_operands()) { 3592 if (Use.isUse() && 3593 (Use.getReg() == AMDGPU::VCC || Use.getReg() == AMDGPU::VCC_LO)) { 3594 Use.setIsUndef(Orig.isUndef()); 3595 Use.setIsKill(Orig.isKill()); 3596 return; 3597 } 3598 } 3599 } 3600 3601 MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI, 3602 unsigned Op32) const { 3603 MachineBasicBlock *MBB = MI.getParent();; 3604 MachineInstrBuilder Inst32 = 3605 BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32)) 3606 .setMIFlags(MI.getFlags()); 3607 3608 // Add the dst operand if the 32-bit encoding also has an explicit $vdst. 3609 // For VOPC instructions, this is replaced by an implicit def of vcc. 3610 int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst); 3611 if (Op32DstIdx != -1) { 3612 // dst 3613 Inst32.add(MI.getOperand(0)); 3614 } else { 3615 assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) || 3616 (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) && 3617 "Unexpected case"); 3618 } 3619 3620 Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0)); 3621 3622 const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1); 3623 if (Src1) 3624 Inst32.add(*Src1); 3625 3626 const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2); 3627 3628 if (Src2) { 3629 int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2); 3630 if (Op32Src2Idx != -1) { 3631 Inst32.add(*Src2); 3632 } else { 3633 // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is 3634 // replaced with an implicit read of vcc or vcc_lo. The implicit read 3635 // of vcc was already added during the initial BuildMI, but we 3636 // 1) may need to change vcc to vcc_lo to preserve the original register 3637 // 2) have to preserve the original flags. 3638 fixImplicitOperands(*Inst32); 3639 copyFlagsToImplicitVCC(*Inst32, *Src2); 3640 } 3641 } 3642 3643 return Inst32; 3644 } 3645 3646 bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI, 3647 const MachineOperand &MO, 3648 const MCOperandInfo &OpInfo) const { 3649 // Literal constants use the constant bus. 3650 //if (isLiteralConstantLike(MO, OpInfo)) 3651 // return true; 3652 if (MO.isImm()) 3653 return !isInlineConstant(MO, OpInfo); 3654 3655 if (!MO.isReg()) 3656 return true; // Misc other operands like FrameIndex 3657 3658 if (!MO.isUse()) 3659 return false; 3660 3661 if (MO.getReg().isVirtual()) 3662 return RI.isSGPRClass(MRI.getRegClass(MO.getReg())); 3663 3664 // Null is free 3665 if (MO.getReg() == AMDGPU::SGPR_NULL) 3666 return false; 3667 3668 // SGPRs use the constant bus 3669 if (MO.isImplicit()) { 3670 return MO.getReg() == AMDGPU::M0 || 3671 MO.getReg() == AMDGPU::VCC || 3672 MO.getReg() == AMDGPU::VCC_LO; 3673 } else { 3674 return AMDGPU::SReg_32RegClass.contains(MO.getReg()) || 3675 AMDGPU::SReg_64RegClass.contains(MO.getReg()); 3676 } 3677 } 3678 3679 static Register findImplicitSGPRRead(const MachineInstr &MI) { 3680 for (const MachineOperand &MO : MI.implicit_operands()) { 3681 // We only care about reads. 3682 if (MO.isDef()) 3683 continue; 3684 3685 switch (MO.getReg()) { 3686 case AMDGPU::VCC: 3687 case AMDGPU::VCC_LO: 3688 case AMDGPU::VCC_HI: 3689 case AMDGPU::M0: 3690 case AMDGPU::FLAT_SCR: 3691 return MO.getReg(); 3692 3693 default: 3694 break; 3695 } 3696 } 3697 3698 return AMDGPU::NoRegister; 3699 } 3700 3701 static bool shouldReadExec(const MachineInstr &MI) { 3702 if (SIInstrInfo::isVALU(MI)) { 3703 switch (MI.getOpcode()) { 3704 case AMDGPU::V_READLANE_B32: 3705 case AMDGPU::V_WRITELANE_B32: 3706 return false; 3707 } 3708 3709 return true; 3710 } 3711 3712 if (MI.isPreISelOpcode() || 3713 SIInstrInfo::isGenericOpcode(MI.getOpcode()) || 3714 SIInstrInfo::isSALU(MI) || 3715 SIInstrInfo::isSMRD(MI)) 3716 return false; 3717 3718 return true; 3719 } 3720 3721 static bool isSubRegOf(const SIRegisterInfo &TRI, 3722 const MachineOperand &SuperVec, 3723 const MachineOperand &SubReg) { 3724 if (SubReg.getReg().isPhysical()) 3725 return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg()); 3726 3727 return SubReg.getSubReg() != AMDGPU::NoSubRegister && 3728 SubReg.getReg() == SuperVec.getReg(); 3729 } 3730 3731 bool SIInstrInfo::verifyInstruction(const MachineInstr &MI, 3732 StringRef &ErrInfo) const { 3733 uint16_t Opcode = MI.getOpcode(); 3734 if (SIInstrInfo::isGenericOpcode(MI.getOpcode())) 3735 return true; 3736 3737 const MachineFunction *MF = MI.getParent()->getParent(); 3738 const MachineRegisterInfo &MRI = MF->getRegInfo(); 3739 3740 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 3741 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1); 3742 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2); 3743 3744 // Make sure the number of operands is correct. 3745 const MCInstrDesc &Desc = get(Opcode); 3746 if (!Desc.isVariadic() && 3747 Desc.getNumOperands() != MI.getNumExplicitOperands()) { 3748 ErrInfo = "Instruction has wrong number of operands."; 3749 return false; 3750 } 3751 3752 if (MI.isInlineAsm()) { 3753 // Verify register classes for inlineasm constraints. 3754 for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands(); 3755 I != E; ++I) { 3756 const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI); 3757 if (!RC) 3758 continue; 3759 3760 const MachineOperand &Op = MI.getOperand(I); 3761 if (!Op.isReg()) 3762 continue; 3763 3764 Register Reg = Op.getReg(); 3765 if (!Reg.isVirtual() && !RC->contains(Reg)) { 3766 ErrInfo = "inlineasm operand has incorrect register class."; 3767 return false; 3768 } 3769 } 3770 3771 return true; 3772 } 3773 3774 if (isMIMG(MI) && MI.memoperands_empty() && MI.mayLoadOrStore()) { 3775 ErrInfo = "missing memory operand from MIMG instruction."; 3776 return false; 3777 } 3778 3779 // Make sure the register classes are correct. 3780 for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) { 3781 const MachineOperand &MO = MI.getOperand(i); 3782 if (MO.isFPImm()) { 3783 ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast " 3784 "all fp values to integers."; 3785 return false; 3786 } 3787 3788 int RegClass = Desc.OpInfo[i].RegClass; 3789 3790 switch (Desc.OpInfo[i].OperandType) { 3791 case MCOI::OPERAND_REGISTER: 3792 if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) { 3793 ErrInfo = "Illegal immediate value for operand."; 3794 return false; 3795 } 3796 break; 3797 case AMDGPU::OPERAND_REG_IMM_INT32: 3798 case AMDGPU::OPERAND_REG_IMM_FP32: 3799 break; 3800 case AMDGPU::OPERAND_REG_INLINE_C_INT32: 3801 case AMDGPU::OPERAND_REG_INLINE_C_FP32: 3802 case AMDGPU::OPERAND_REG_INLINE_C_INT64: 3803 case AMDGPU::OPERAND_REG_INLINE_C_FP64: 3804 case AMDGPU::OPERAND_REG_INLINE_C_INT16: 3805 case AMDGPU::OPERAND_REG_INLINE_C_FP16: 3806 case AMDGPU::OPERAND_REG_INLINE_AC_INT32: 3807 case AMDGPU::OPERAND_REG_INLINE_AC_FP32: 3808 case AMDGPU::OPERAND_REG_INLINE_AC_INT16: 3809 case AMDGPU::OPERAND_REG_INLINE_AC_FP16: 3810 case AMDGPU::OPERAND_REG_INLINE_AC_FP64: { 3811 if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) { 3812 ErrInfo = "Illegal immediate value for operand."; 3813 return false; 3814 } 3815 break; 3816 } 3817 case MCOI::OPERAND_IMMEDIATE: 3818 case AMDGPU::OPERAND_KIMM32: 3819 // Check if this operand is an immediate. 3820 // FrameIndex operands will be replaced by immediates, so they are 3821 // allowed. 3822 if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) { 3823 ErrInfo = "Expected immediate, but got non-immediate"; 3824 return false; 3825 } 3826 LLVM_FALLTHROUGH; 3827 default: 3828 continue; 3829 } 3830 3831 if (!MO.isReg()) 3832 continue; 3833 Register Reg = MO.getReg(); 3834 if (!Reg) 3835 continue; 3836 3837 // FIXME: Ideally we would have separate instruction definitions with the 3838 // aligned register constraint. 3839 // FIXME: We do not verify inline asm operands, but custom inline asm 3840 // verification is broken anyway 3841 if (ST.needsAlignedVGPRs()) { 3842 const TargetRegisterClass *RC = RI.getRegClassForReg(MRI, Reg); 3843 const bool IsVGPR = RI.hasVGPRs(RC); 3844 const bool IsAGPR = !IsVGPR && RI.hasAGPRs(RC); 3845 if ((IsVGPR || IsAGPR) && MO.getSubReg()) { 3846 const TargetRegisterClass *SubRC = 3847 RI.getSubRegClass(RC, MO.getSubReg()); 3848 RC = RI.getCompatibleSubRegClass(RC, SubRC, MO.getSubReg()); 3849 if (RC) 3850 RC = SubRC; 3851 } 3852 3853 // Check that this is the aligned version of the class. 3854 if (!RC || !RI.isProperlyAlignedRC(*RC)) { 3855 ErrInfo = "Subtarget requires even aligned vector registers"; 3856 return false; 3857 } 3858 } 3859 3860 if (RegClass != -1) { 3861 if (Reg.isVirtual()) 3862 continue; 3863 3864 const TargetRegisterClass *RC = RI.getRegClass(RegClass); 3865 if (!RC->contains(Reg)) { 3866 ErrInfo = "Operand has incorrect register class."; 3867 return false; 3868 } 3869 } 3870 } 3871 3872 // Verify SDWA 3873 if (isSDWA(MI)) { 3874 if (!ST.hasSDWA()) { 3875 ErrInfo = "SDWA is not supported on this target"; 3876 return false; 3877 } 3878 3879 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 3880 3881 const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx }; 3882 3883 for (int OpIdx: OpIndicies) { 3884 if (OpIdx == -1) 3885 continue; 3886 const MachineOperand &MO = MI.getOperand(OpIdx); 3887 3888 if (!ST.hasSDWAScalar()) { 3889 // Only VGPRS on VI 3890 if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) { 3891 ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI"; 3892 return false; 3893 } 3894 } else { 3895 // No immediates on GFX9 3896 if (!MO.isReg()) { 3897 ErrInfo = 3898 "Only reg allowed as operands in SDWA instructions on GFX9+"; 3899 return false; 3900 } 3901 } 3902 } 3903 3904 if (!ST.hasSDWAOmod()) { 3905 // No omod allowed on VI 3906 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3907 if (OMod != nullptr && 3908 (!OMod->isImm() || OMod->getImm() != 0)) { 3909 ErrInfo = "OMod not allowed in SDWA instructions on VI"; 3910 return false; 3911 } 3912 } 3913 3914 uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode); 3915 if (isVOPC(BasicOpcode)) { 3916 if (!ST.hasSDWASdst() && DstIdx != -1) { 3917 // Only vcc allowed as dst on VI for VOPC 3918 const MachineOperand &Dst = MI.getOperand(DstIdx); 3919 if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) { 3920 ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI"; 3921 return false; 3922 } 3923 } else if (!ST.hasSDWAOutModsVOPC()) { 3924 // No clamp allowed on GFX9 for VOPC 3925 const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp); 3926 if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) { 3927 ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI"; 3928 return false; 3929 } 3930 3931 // No omod allowed on GFX9 for VOPC 3932 const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod); 3933 if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) { 3934 ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI"; 3935 return false; 3936 } 3937 } 3938 } 3939 3940 const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused); 3941 if (DstUnused && DstUnused->isImm() && 3942 DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) { 3943 const MachineOperand &Dst = MI.getOperand(DstIdx); 3944 if (!Dst.isReg() || !Dst.isTied()) { 3945 ErrInfo = "Dst register should have tied register"; 3946 return false; 3947 } 3948 3949 const MachineOperand &TiedMO = 3950 MI.getOperand(MI.findTiedOperandIdx(DstIdx)); 3951 if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) { 3952 ErrInfo = 3953 "Dst register should be tied to implicit use of preserved register"; 3954 return false; 3955 } else if (TiedMO.getReg().isPhysical() && 3956 Dst.getReg() != TiedMO.getReg()) { 3957 ErrInfo = "Dst register should use same physical register as preserved"; 3958 return false; 3959 } 3960 } 3961 } 3962 3963 // Verify MIMG 3964 if (isMIMG(MI.getOpcode()) && !MI.mayStore()) { 3965 // Ensure that the return type used is large enough for all the options 3966 // being used TFE/LWE require an extra result register. 3967 const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); 3968 if (DMask) { 3969 uint64_t DMaskImm = DMask->getImm(); 3970 uint32_t RegCount = 3971 isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm); 3972 const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); 3973 const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); 3974 const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); 3975 3976 // Adjust for packed 16 bit values 3977 if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) 3978 RegCount >>= 1; 3979 3980 // Adjust if using LWE or TFE 3981 if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) 3982 RegCount += 1; 3983 3984 const uint32_t DstIdx = 3985 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); 3986 const MachineOperand &Dst = MI.getOperand(DstIdx); 3987 if (Dst.isReg()) { 3988 const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); 3989 uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; 3990 if (RegCount > DstSize) { 3991 ErrInfo = "MIMG instruction returns too many registers for dst " 3992 "register class"; 3993 return false; 3994 } 3995 } 3996 } 3997 } 3998 3999 // Verify VOP*. Ignore multiple sgpr operands on writelane. 4000 if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 4001 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) { 4002 // Only look at the true operands. Only a real operand can use the constant 4003 // bus, and we don't want to check pseudo-operands like the source modifier 4004 // flags. 4005 const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx }; 4006 4007 unsigned ConstantBusCount = 0; 4008 bool UsesLiteral = false; 4009 const MachineOperand *LiteralVal = nullptr; 4010 4011 if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) 4012 ++ConstantBusCount; 4013 4014 SmallVector<Register, 2> SGPRsUsed; 4015 Register SGPRUsed; 4016 4017 for (int OpIdx : OpIndices) { 4018 if (OpIdx == -1) 4019 break; 4020 const MachineOperand &MO = MI.getOperand(OpIdx); 4021 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 4022 if (MO.isReg()) { 4023 SGPRUsed = MO.getReg(); 4024 if (llvm::all_of(SGPRsUsed, [SGPRUsed](unsigned SGPR) { 4025 return SGPRUsed != SGPR; 4026 })) { 4027 ++ConstantBusCount; 4028 SGPRsUsed.push_back(SGPRUsed); 4029 } 4030 } else { 4031 if (!UsesLiteral) { 4032 ++ConstantBusCount; 4033 UsesLiteral = true; 4034 LiteralVal = &MO; 4035 } else if (!MO.isIdenticalTo(*LiteralVal)) { 4036 assert(isVOP3(MI)); 4037 ErrInfo = "VOP3 instruction uses more than one literal"; 4038 return false; 4039 } 4040 } 4041 } 4042 } 4043 4044 SGPRUsed = findImplicitSGPRRead(MI); 4045 if (SGPRUsed != AMDGPU::NoRegister) { 4046 // Implicit uses may safely overlap true overands 4047 if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) { 4048 return !RI.regsOverlap(SGPRUsed, SGPR); 4049 })) { 4050 ++ConstantBusCount; 4051 SGPRsUsed.push_back(SGPRUsed); 4052 } 4053 } 4054 4055 // v_writelane_b32 is an exception from constant bus restriction: 4056 // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const 4057 if (ConstantBusCount > ST.getConstantBusLimit(Opcode) && 4058 Opcode != AMDGPU::V_WRITELANE_B32) { 4059 ErrInfo = "VOP* instruction violates constant bus restriction"; 4060 return false; 4061 } 4062 4063 if (isVOP3(MI) && UsesLiteral && !ST.hasVOP3Literal()) { 4064 ErrInfo = "VOP3 instruction uses literal"; 4065 return false; 4066 } 4067 } 4068 4069 // Special case for writelane - this can break the multiple constant bus rule, 4070 // but still can't use more than one SGPR register 4071 if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) { 4072 unsigned SGPRCount = 0; 4073 Register SGPRUsed = AMDGPU::NoRegister; 4074 4075 for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) { 4076 if (OpIdx == -1) 4077 break; 4078 4079 const MachineOperand &MO = MI.getOperand(OpIdx); 4080 4081 if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) { 4082 if (MO.isReg() && MO.getReg() != AMDGPU::M0) { 4083 if (MO.getReg() != SGPRUsed) 4084 ++SGPRCount; 4085 SGPRUsed = MO.getReg(); 4086 } 4087 } 4088 if (SGPRCount > ST.getConstantBusLimit(Opcode)) { 4089 ErrInfo = "WRITELANE instruction violates constant bus restriction"; 4090 return false; 4091 } 4092 } 4093 } 4094 4095 // Verify misc. restrictions on specific instructions. 4096 if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32_e64 || 4097 Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64_e64) { 4098 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4099 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4100 const MachineOperand &Src2 = MI.getOperand(Src2Idx); 4101 if (Src0.isReg() && Src1.isReg() && Src2.isReg()) { 4102 if (!compareMachineOp(Src0, Src1) && 4103 !compareMachineOp(Src0, Src2)) { 4104 ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2"; 4105 return false; 4106 } 4107 } 4108 if ((getNamedOperand(MI, AMDGPU::OpName::src0_modifiers)->getImm() & 4109 SISrcMods::ABS) || 4110 (getNamedOperand(MI, AMDGPU::OpName::src1_modifiers)->getImm() & 4111 SISrcMods::ABS) || 4112 (getNamedOperand(MI, AMDGPU::OpName::src2_modifiers)->getImm() & 4113 SISrcMods::ABS)) { 4114 ErrInfo = "ABS not allowed in VOP3B instructions"; 4115 return false; 4116 } 4117 } 4118 4119 if (isSOP2(MI) || isSOPC(MI)) { 4120 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4121 const MachineOperand &Src1 = MI.getOperand(Src1Idx); 4122 unsigned Immediates = 0; 4123 4124 if (!Src0.isReg() && 4125 !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType)) 4126 Immediates++; 4127 if (!Src1.isReg() && 4128 !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType)) 4129 Immediates++; 4130 4131 if (Immediates > 1) { 4132 ErrInfo = "SOP2/SOPC instruction requires too many immediate constants"; 4133 return false; 4134 } 4135 } 4136 4137 if (isSOPK(MI)) { 4138 auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16); 4139 if (Desc.isBranch()) { 4140 if (!Op->isMBB()) { 4141 ErrInfo = "invalid branch target for SOPK instruction"; 4142 return false; 4143 } 4144 } else { 4145 uint64_t Imm = Op->getImm(); 4146 if (sopkIsZext(MI)) { 4147 if (!isUInt<16>(Imm)) { 4148 ErrInfo = "invalid immediate for SOPK instruction"; 4149 return false; 4150 } 4151 } else { 4152 if (!isInt<16>(Imm)) { 4153 ErrInfo = "invalid immediate for SOPK instruction"; 4154 return false; 4155 } 4156 } 4157 } 4158 } 4159 4160 if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 || 4161 Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 || 4162 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4163 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) { 4164 const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 || 4165 Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64; 4166 4167 const unsigned StaticNumOps = Desc.getNumOperands() + 4168 Desc.getNumImplicitUses(); 4169 const unsigned NumImplicitOps = IsDst ? 2 : 1; 4170 4171 // Allow additional implicit operands. This allows a fixup done by the post 4172 // RA scheduler where the main implicit operand is killed and implicit-defs 4173 // are added for sub-registers that remain live after this instruction. 4174 if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) { 4175 ErrInfo = "missing implicit register operands"; 4176 return false; 4177 } 4178 4179 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4180 if (IsDst) { 4181 if (!Dst->isUse()) { 4182 ErrInfo = "v_movreld_b32 vdst should be a use operand"; 4183 return false; 4184 } 4185 4186 unsigned UseOpIdx; 4187 if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) || 4188 UseOpIdx != StaticNumOps + 1) { 4189 ErrInfo = "movrel implicit operands should be tied"; 4190 return false; 4191 } 4192 } 4193 4194 const MachineOperand &Src0 = MI.getOperand(Src0Idx); 4195 const MachineOperand &ImpUse 4196 = MI.getOperand(StaticNumOps + NumImplicitOps - 1); 4197 if (!ImpUse.isReg() || !ImpUse.isUse() || 4198 !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) { 4199 ErrInfo = "src0 should be subreg of implicit vector use"; 4200 return false; 4201 } 4202 } 4203 4204 // Make sure we aren't losing exec uses in the td files. This mostly requires 4205 // being careful when using let Uses to try to add other use registers. 4206 if (shouldReadExec(MI)) { 4207 if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) { 4208 ErrInfo = "VALU instruction does not implicitly read exec mask"; 4209 return false; 4210 } 4211 } 4212 4213 if (isSMRD(MI)) { 4214 if (MI.mayStore()) { 4215 // The register offset form of scalar stores may only use m0 as the 4216 // soffset register. 4217 const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff); 4218 if (Soff && Soff->getReg() != AMDGPU::M0) { 4219 ErrInfo = "scalar stores must use m0 as offset register"; 4220 return false; 4221 } 4222 } 4223 } 4224 4225 if (isFLAT(MI) && !ST.hasFlatInstOffsets()) { 4226 const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 4227 if (Offset->getImm() != 0) { 4228 ErrInfo = "subtarget does not support offsets in flat instructions"; 4229 return false; 4230 } 4231 } 4232 4233 if (isMIMG(MI)) { 4234 const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim); 4235 if (DimOp) { 4236 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode, 4237 AMDGPU::OpName::vaddr0); 4238 int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc); 4239 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode); 4240 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = 4241 AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); 4242 const AMDGPU::MIMGDimInfo *Dim = 4243 AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm()); 4244 4245 if (!Dim) { 4246 ErrInfo = "dim is out of range"; 4247 return false; 4248 } 4249 4250 bool IsA16 = false; 4251 if (ST.hasR128A16()) { 4252 const MachineOperand *R128A16 = getNamedOperand(MI, AMDGPU::OpName::r128); 4253 IsA16 = R128A16->getImm() != 0; 4254 } else if (ST.hasGFX10A16()) { 4255 const MachineOperand *A16 = getNamedOperand(MI, AMDGPU::OpName::a16); 4256 IsA16 = A16->getImm() != 0; 4257 } 4258 4259 bool IsNSA = SRsrcIdx - VAddr0Idx > 1; 4260 4261 unsigned AddrWords = 4262 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim, IsA16, ST.hasG16()); 4263 4264 unsigned VAddrWords; 4265 if (IsNSA) { 4266 VAddrWords = SRsrcIdx - VAddr0Idx; 4267 } else { 4268 const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx); 4269 VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32; 4270 if (AddrWords > 8) 4271 AddrWords = 16; 4272 else if (AddrWords > 5) 4273 AddrWords = 8; 4274 } 4275 4276 if (VAddrWords != AddrWords) { 4277 LLVM_DEBUG(dbgs() << "bad vaddr size, expected " << AddrWords 4278 << " but got " << VAddrWords << "\n"); 4279 ErrInfo = "bad vaddr size"; 4280 return false; 4281 } 4282 } 4283 } 4284 4285 const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl); 4286 if (DppCt) { 4287 using namespace AMDGPU::DPP; 4288 4289 unsigned DC = DppCt->getImm(); 4290 if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 || 4291 DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST || 4292 (DC >= DppCtrl::DPP_UNUSED4_FIRST && DC <= DppCtrl::DPP_UNUSED4_LAST) || 4293 (DC >= DppCtrl::DPP_UNUSED5_FIRST && DC <= DppCtrl::DPP_UNUSED5_LAST) || 4294 (DC >= DppCtrl::DPP_UNUSED6_FIRST && DC <= DppCtrl::DPP_UNUSED6_LAST) || 4295 (DC >= DppCtrl::DPP_UNUSED7_FIRST && DC <= DppCtrl::DPP_UNUSED7_LAST) || 4296 (DC >= DppCtrl::DPP_UNUSED8_FIRST && DC <= DppCtrl::DPP_UNUSED8_LAST)) { 4297 ErrInfo = "Invalid dpp_ctrl value"; 4298 return false; 4299 } 4300 if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 && 4301 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4302 ErrInfo = "Invalid dpp_ctrl value: " 4303 "wavefront shifts are not supported on GFX10+"; 4304 return false; 4305 } 4306 if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 && 4307 ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 4308 ErrInfo = "Invalid dpp_ctrl value: " 4309 "broadcasts are not supported on GFX10+"; 4310 return false; 4311 } 4312 if (DC >= DppCtrl::ROW_SHARE_FIRST && DC <= DppCtrl::ROW_XMASK_LAST && 4313 ST.getGeneration() < AMDGPUSubtarget::GFX10) { 4314 if (DC >= DppCtrl::ROW_NEWBCAST_FIRST && 4315 DC <= DppCtrl::ROW_NEWBCAST_LAST && 4316 !ST.hasGFX90AInsts()) { 4317 ErrInfo = "Invalid dpp_ctrl value: " 4318 "row_newbroadcast/row_share is not supported before " 4319 "GFX90A/GFX10"; 4320 return false; 4321 } else if (DC > DppCtrl::ROW_NEWBCAST_LAST || !ST.hasGFX90AInsts()) { 4322 ErrInfo = "Invalid dpp_ctrl value: " 4323 "row_share and row_xmask are not supported before GFX10"; 4324 return false; 4325 } 4326 } 4327 4328 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst); 4329 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0); 4330 4331 if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO && 4332 ((DstIdx >= 0 && 4333 (Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64RegClassID || 4334 Desc.OpInfo[DstIdx].RegClass == AMDGPU::VReg_64_Align2RegClassID)) || 4335 ((Src0Idx >= 0 && 4336 (Desc.OpInfo[Src0Idx].RegClass == AMDGPU::VReg_64RegClassID || 4337 Desc.OpInfo[Src0Idx].RegClass == 4338 AMDGPU::VReg_64_Align2RegClassID)))) && 4339 !AMDGPU::isLegal64BitDPPControl(DC)) { 4340 ErrInfo = "Invalid dpp_ctrl value: " 4341 "64 bit dpp only support row_newbcast"; 4342 return false; 4343 } 4344 } 4345 4346 if ((MI.mayStore() || MI.mayLoad()) && !isVGPRSpill(MI)) { 4347 const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst); 4348 uint16_t DataNameIdx = isDS(Opcode) ? AMDGPU::OpName::data0 4349 : AMDGPU::OpName::vdata; 4350 const MachineOperand *Data = getNamedOperand(MI, DataNameIdx); 4351 const MachineOperand *Data2 = getNamedOperand(MI, AMDGPU::OpName::data1); 4352 if (Data && !Data->isReg()) 4353 Data = nullptr; 4354 4355 if (ST.hasGFX90AInsts()) { 4356 if (Dst && Data && 4357 (RI.isAGPR(MRI, Dst->getReg()) != RI.isAGPR(MRI, Data->getReg()))) { 4358 ErrInfo = "Invalid register class: " 4359 "vdata and vdst should be both VGPR or AGPR"; 4360 return false; 4361 } 4362 if (Data && Data2 && 4363 (RI.isAGPR(MRI, Data->getReg()) != RI.isAGPR(MRI, Data2->getReg()))) { 4364 ErrInfo = "Invalid register class: " 4365 "both data operands should be VGPR or AGPR"; 4366 return false; 4367 } 4368 } else { 4369 if ((Dst && RI.isAGPR(MRI, Dst->getReg())) || 4370 (Data && RI.isAGPR(MRI, Data->getReg())) || 4371 (Data2 && RI.isAGPR(MRI, Data2->getReg()))) { 4372 ErrInfo = "Invalid register class: " 4373 "agpr loads and stores not supported on this GPU"; 4374 return false; 4375 } 4376 } 4377 } 4378 4379 if (ST.needsAlignedVGPRs() && 4380 (MI.getOpcode() == AMDGPU::DS_GWS_INIT || 4381 MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR || 4382 MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) { 4383 const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0); 4384 Register Reg = Op->getReg(); 4385 bool Aligned = true; 4386 if (Reg.isPhysical()) { 4387 Aligned = !(RI.getHWRegIndex(Reg) & 1); 4388 } else { 4389 const TargetRegisterClass &RC = *MRI.getRegClass(Reg); 4390 Aligned = RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) && 4391 !(RI.getChannelFromSubReg(Op->getSubReg()) & 1); 4392 } 4393 4394 if (!Aligned) { 4395 ErrInfo = "Subtarget requires even aligned vector registers " 4396 "for DS_GWS instructions"; 4397 return false; 4398 } 4399 } 4400 4401 return true; 4402 } 4403 4404 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const { 4405 switch (MI.getOpcode()) { 4406 default: return AMDGPU::INSTRUCTION_LIST_END; 4407 case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE; 4408 case AMDGPU::COPY: return AMDGPU::COPY; 4409 case AMDGPU::PHI: return AMDGPU::PHI; 4410 case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG; 4411 case AMDGPU::WQM: return AMDGPU::WQM; 4412 case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM; 4413 case AMDGPU::STRICT_WWM: return AMDGPU::STRICT_WWM; 4414 case AMDGPU::STRICT_WQM: return AMDGPU::STRICT_WQM; 4415 case AMDGPU::S_MOV_B32: { 4416 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4417 return MI.getOperand(1).isReg() || 4418 RI.isAGPR(MRI, MI.getOperand(0).getReg()) ? 4419 AMDGPU::COPY : AMDGPU::V_MOV_B32_e32; 4420 } 4421 case AMDGPU::S_ADD_I32: 4422 return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32; 4423 case AMDGPU::S_ADDC_U32: 4424 return AMDGPU::V_ADDC_U32_e32; 4425 case AMDGPU::S_SUB_I32: 4426 return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_CO_U32_e32; 4427 // FIXME: These are not consistently handled, and selected when the carry is 4428 // used. 4429 case AMDGPU::S_ADD_U32: 4430 return AMDGPU::V_ADD_CO_U32_e32; 4431 case AMDGPU::S_SUB_U32: 4432 return AMDGPU::V_SUB_CO_U32_e32; 4433 case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32; 4434 case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32_e64; 4435 case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32_e64; 4436 case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32_e64; 4437 case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64; 4438 case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64; 4439 case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64; 4440 case AMDGPU::S_XNOR_B32: 4441 return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END; 4442 case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64; 4443 case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64; 4444 case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64; 4445 case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64; 4446 case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32; 4447 case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64_e64; 4448 case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32; 4449 case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64_e64; 4450 case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32; 4451 case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64_e64; 4452 case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32_e64; 4453 case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32_e64; 4454 case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32_e64; 4455 case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32_e64; 4456 case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64; 4457 case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32; 4458 case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32; 4459 case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32; 4460 case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32; 4461 case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32; 4462 case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32; 4463 case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32; 4464 case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32; 4465 case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32; 4466 case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32; 4467 case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32; 4468 case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32; 4469 case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32; 4470 case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32; 4471 case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32; 4472 case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32; 4473 case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32; 4474 case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64; 4475 case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32; 4476 case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32; 4477 case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64; 4478 case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ; 4479 case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ; 4480 } 4481 llvm_unreachable( 4482 "Unexpected scalar opcode without corresponding vector one!"); 4483 } 4484 4485 static unsigned adjustAllocatableRegClass(const GCNSubtarget &ST, 4486 const MachineRegisterInfo &MRI, 4487 const MCInstrDesc &TID, 4488 unsigned RCID, 4489 bool IsAllocatable) { 4490 if ((IsAllocatable || !ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 4491 (TID.mayLoad() || TID.mayStore() || 4492 (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::MIMG)))) { 4493 switch (RCID) { 4494 case AMDGPU::AV_32RegClassID: return AMDGPU::VGPR_32RegClassID; 4495 case AMDGPU::AV_64RegClassID: return AMDGPU::VReg_64RegClassID; 4496 case AMDGPU::AV_96RegClassID: return AMDGPU::VReg_96RegClassID; 4497 case AMDGPU::AV_128RegClassID: return AMDGPU::VReg_128RegClassID; 4498 case AMDGPU::AV_160RegClassID: return AMDGPU::VReg_160RegClassID; 4499 default: 4500 break; 4501 } 4502 } 4503 return RCID; 4504 } 4505 4506 const TargetRegisterClass *SIInstrInfo::getRegClass(const MCInstrDesc &TID, 4507 unsigned OpNum, const TargetRegisterInfo *TRI, 4508 const MachineFunction &MF) 4509 const { 4510 if (OpNum >= TID.getNumOperands()) 4511 return nullptr; 4512 auto RegClass = TID.OpInfo[OpNum].RegClass; 4513 bool IsAllocatable = false; 4514 if (TID.TSFlags & (SIInstrFlags::DS | SIInstrFlags::FLAT)) { 4515 // vdst and vdata should be both VGPR or AGPR, same for the DS instructions 4516 // with two data operands. Request register class constainted to VGPR only 4517 // of both operands present as Machine Copy Propagation can not check this 4518 // constraint and possibly other passes too. 4519 // 4520 // The check is limited to FLAT and DS because atomics in non-flat encoding 4521 // have their vdst and vdata tied to be the same register. 4522 const int VDstIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4523 AMDGPU::OpName::vdst); 4524 const int DataIdx = AMDGPU::getNamedOperandIdx(TID.Opcode, 4525 (TID.TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0 4526 : AMDGPU::OpName::vdata); 4527 if (DataIdx != -1) { 4528 IsAllocatable = VDstIdx != -1 || 4529 AMDGPU::getNamedOperandIdx(TID.Opcode, 4530 AMDGPU::OpName::data1) != -1; 4531 } 4532 } 4533 RegClass = adjustAllocatableRegClass(ST, MF.getRegInfo(), TID, RegClass, 4534 IsAllocatable); 4535 return RI.getRegClass(RegClass); 4536 } 4537 4538 const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI, 4539 unsigned OpNo) const { 4540 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 4541 const MCInstrDesc &Desc = get(MI.getOpcode()); 4542 if (MI.isVariadic() || OpNo >= Desc.getNumOperands() || 4543 Desc.OpInfo[OpNo].RegClass == -1) { 4544 Register Reg = MI.getOperand(OpNo).getReg(); 4545 4546 if (Reg.isVirtual()) 4547 return MRI.getRegClass(Reg); 4548 return RI.getPhysRegClass(Reg); 4549 } 4550 4551 unsigned RCID = Desc.OpInfo[OpNo].RegClass; 4552 RCID = adjustAllocatableRegClass(ST, MRI, Desc, RCID, true); 4553 return RI.getRegClass(RCID); 4554 } 4555 4556 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const { 4557 MachineBasicBlock::iterator I = MI; 4558 MachineBasicBlock *MBB = MI.getParent(); 4559 MachineOperand &MO = MI.getOperand(OpIdx); 4560 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 4561 unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass; 4562 const TargetRegisterClass *RC = RI.getRegClass(RCID); 4563 unsigned Size = RI.getRegSizeInBits(*RC); 4564 unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32; 4565 if (MO.isReg()) 4566 Opcode = AMDGPU::COPY; 4567 else if (RI.isSGPRClass(RC)) 4568 Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32; 4569 4570 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC); 4571 const TargetRegisterClass *VRC64 = RI.getVGPR64Class(); 4572 if (RI.getCommonSubClass(VRC64, VRC)) 4573 VRC = VRC64; 4574 else 4575 VRC = &AMDGPU::VGPR_32RegClass; 4576 4577 Register Reg = MRI.createVirtualRegister(VRC); 4578 DebugLoc DL = MBB->findDebugLoc(I); 4579 BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO); 4580 MO.ChangeToRegister(Reg, false); 4581 } 4582 4583 unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI, 4584 MachineRegisterInfo &MRI, 4585 MachineOperand &SuperReg, 4586 const TargetRegisterClass *SuperRC, 4587 unsigned SubIdx, 4588 const TargetRegisterClass *SubRC) 4589 const { 4590 MachineBasicBlock *MBB = MI->getParent(); 4591 DebugLoc DL = MI->getDebugLoc(); 4592 Register SubReg = MRI.createVirtualRegister(SubRC); 4593 4594 if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) { 4595 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4596 .addReg(SuperReg.getReg(), 0, SubIdx); 4597 return SubReg; 4598 } 4599 4600 // Just in case the super register is itself a sub-register, copy it to a new 4601 // value so we don't need to worry about merging its subreg index with the 4602 // SubIdx passed to this function. The register coalescer should be able to 4603 // eliminate this extra copy. 4604 Register NewSuperReg = MRI.createVirtualRegister(SuperRC); 4605 4606 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg) 4607 .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg()); 4608 4609 BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg) 4610 .addReg(NewSuperReg, 0, SubIdx); 4611 4612 return SubReg; 4613 } 4614 4615 MachineOperand SIInstrInfo::buildExtractSubRegOrImm( 4616 MachineBasicBlock::iterator MII, 4617 MachineRegisterInfo &MRI, 4618 MachineOperand &Op, 4619 const TargetRegisterClass *SuperRC, 4620 unsigned SubIdx, 4621 const TargetRegisterClass *SubRC) const { 4622 if (Op.isImm()) { 4623 if (SubIdx == AMDGPU::sub0) 4624 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm())); 4625 if (SubIdx == AMDGPU::sub1) 4626 return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32)); 4627 4628 llvm_unreachable("Unhandled register index for immediate"); 4629 } 4630 4631 unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC, 4632 SubIdx, SubRC); 4633 return MachineOperand::CreateReg(SubReg, false); 4634 } 4635 4636 // Change the order of operands from (0, 1, 2) to (0, 2, 1) 4637 void SIInstrInfo::swapOperands(MachineInstr &Inst) const { 4638 assert(Inst.getNumExplicitOperands() == 3); 4639 MachineOperand Op1 = Inst.getOperand(1); 4640 Inst.RemoveOperand(1); 4641 Inst.addOperand(Op1); 4642 } 4643 4644 bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, 4645 const MCOperandInfo &OpInfo, 4646 const MachineOperand &MO) const { 4647 if (!MO.isReg()) 4648 return false; 4649 4650 Register Reg = MO.getReg(); 4651 4652 const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); 4653 if (Reg.isPhysical()) 4654 return DRC->contains(Reg); 4655 4656 const TargetRegisterClass *RC = MRI.getRegClass(Reg); 4657 4658 if (MO.getSubReg()) { 4659 const MachineFunction *MF = MO.getParent()->getParent()->getParent(); 4660 const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); 4661 if (!SuperRC) 4662 return false; 4663 4664 DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); 4665 if (!DRC) 4666 return false; 4667 } 4668 return RC->hasSuperClassEq(DRC); 4669 } 4670 4671 bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, 4672 const MCOperandInfo &OpInfo, 4673 const MachineOperand &MO) const { 4674 if (MO.isReg()) 4675 return isLegalRegOperand(MRI, OpInfo, MO); 4676 4677 // Handle non-register types that are treated like immediates. 4678 assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal()); 4679 return true; 4680 } 4681 4682 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx, 4683 const MachineOperand *MO) const { 4684 const MachineFunction &MF = *MI.getParent()->getParent(); 4685 const MachineRegisterInfo &MRI = MF.getRegInfo(); 4686 const MCInstrDesc &InstDesc = MI.getDesc(); 4687 const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx]; 4688 const TargetRegisterClass *DefinedRC = 4689 OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr; 4690 if (!MO) 4691 MO = &MI.getOperand(OpIdx); 4692 4693 int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode()); 4694 int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4695 if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) { 4696 if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--) 4697 return false; 4698 4699 SmallDenseSet<RegSubRegPair> SGPRsUsed; 4700 if (MO->isReg()) 4701 SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg())); 4702 4703 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { 4704 if (i == OpIdx) 4705 continue; 4706 const MachineOperand &Op = MI.getOperand(i); 4707 if (Op.isReg()) { 4708 RegSubRegPair SGPR(Op.getReg(), Op.getSubReg()); 4709 if (!SGPRsUsed.count(SGPR) && 4710 usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) { 4711 if (--ConstantBusLimit <= 0) 4712 return false; 4713 SGPRsUsed.insert(SGPR); 4714 } 4715 } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) { 4716 if (--ConstantBusLimit <= 0) 4717 return false; 4718 } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) && 4719 isLiteralConstantLike(Op, InstDesc.OpInfo[i])) { 4720 if (!VOP3LiteralLimit--) 4721 return false; 4722 if (--ConstantBusLimit <= 0) 4723 return false; 4724 } 4725 } 4726 } 4727 4728 if (MO->isReg()) { 4729 assert(DefinedRC); 4730 if (!isLegalRegOperand(MRI, OpInfo, *MO)) 4731 return false; 4732 bool IsAGPR = RI.isAGPR(MRI, MO->getReg()); 4733 if (IsAGPR && !ST.hasMAIInsts()) 4734 return false; 4735 unsigned Opc = MI.getOpcode(); 4736 if (IsAGPR && 4737 (!ST.hasGFX90AInsts() || !MRI.reservedRegsFrozen()) && 4738 (MI.mayLoad() || MI.mayStore() || isDS(Opc) || isMIMG(Opc))) 4739 return false; 4740 // Atomics should have both vdst and vdata either vgpr or agpr. 4741 const int VDstIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst); 4742 const int DataIdx = AMDGPU::getNamedOperandIdx(Opc, 4743 isDS(Opc) ? AMDGPU::OpName::data0 : AMDGPU::OpName::vdata); 4744 if ((int)OpIdx == VDstIdx && DataIdx != -1 && 4745 MI.getOperand(DataIdx).isReg() && 4746 RI.isAGPR(MRI, MI.getOperand(DataIdx).getReg()) != IsAGPR) 4747 return false; 4748 if ((int)OpIdx == DataIdx) { 4749 if (VDstIdx != -1 && 4750 RI.isAGPR(MRI, MI.getOperand(VDstIdx).getReg()) != IsAGPR) 4751 return false; 4752 // DS instructions with 2 src operands also must have tied RC. 4753 const int Data1Idx = AMDGPU::getNamedOperandIdx(Opc, 4754 AMDGPU::OpName::data1); 4755 if (Data1Idx != -1 && MI.getOperand(Data1Idx).isReg() && 4756 RI.isAGPR(MRI, MI.getOperand(Data1Idx).getReg()) != IsAGPR) 4757 return false; 4758 } 4759 if (Opc == AMDGPU::V_ACCVGPR_WRITE_B32_e64 && 4760 (int)OpIdx == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) && 4761 RI.isSGPRReg(MRI, MO->getReg())) 4762 return false; 4763 return true; 4764 } 4765 4766 // Handle non-register types that are treated like immediates. 4767 assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal()); 4768 4769 if (!DefinedRC) { 4770 // This operand expects an immediate. 4771 return true; 4772 } 4773 4774 return isImmOperandLegal(MI, OpIdx, *MO); 4775 } 4776 4777 void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI, 4778 MachineInstr &MI) const { 4779 unsigned Opc = MI.getOpcode(); 4780 const MCInstrDesc &InstrDesc = get(Opc); 4781 4782 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 4783 MachineOperand &Src0 = MI.getOperand(Src0Idx); 4784 4785 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 4786 MachineOperand &Src1 = MI.getOperand(Src1Idx); 4787 4788 // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32 4789 // we need to only have one constant bus use before GFX10. 4790 bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister; 4791 if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 && 4792 Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) || 4793 isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx]))) 4794 legalizeOpWithMove(MI, Src0Idx); 4795 4796 // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for 4797 // both the value to write (src0) and lane select (src1). Fix up non-SGPR 4798 // src0/src1 with V_READFIRSTLANE. 4799 if (Opc == AMDGPU::V_WRITELANE_B32) { 4800 const DebugLoc &DL = MI.getDebugLoc(); 4801 if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) { 4802 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4803 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4804 .add(Src0); 4805 Src0.ChangeToRegister(Reg, false); 4806 } 4807 if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) { 4808 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4809 const DebugLoc &DL = MI.getDebugLoc(); 4810 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4811 .add(Src1); 4812 Src1.ChangeToRegister(Reg, false); 4813 } 4814 return; 4815 } 4816 4817 // No VOP2 instructions support AGPRs. 4818 if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg())) 4819 legalizeOpWithMove(MI, Src0Idx); 4820 4821 if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg())) 4822 legalizeOpWithMove(MI, Src1Idx); 4823 4824 // VOP2 src0 instructions support all operand types, so we don't need to check 4825 // their legality. If src1 is already legal, we don't need to do anything. 4826 if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1)) 4827 return; 4828 4829 // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for 4830 // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane 4831 // select is uniform. 4832 if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() && 4833 RI.isVGPR(MRI, Src1.getReg())) { 4834 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4835 const DebugLoc &DL = MI.getDebugLoc(); 4836 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4837 .add(Src1); 4838 Src1.ChangeToRegister(Reg, false); 4839 return; 4840 } 4841 4842 // We do not use commuteInstruction here because it is too aggressive and will 4843 // commute if it is possible. We only want to commute here if it improves 4844 // legality. This can be called a fairly large number of times so don't waste 4845 // compile time pointlessly swapping and checking legality again. 4846 if (HasImplicitSGPR || !MI.isCommutable()) { 4847 legalizeOpWithMove(MI, Src1Idx); 4848 return; 4849 } 4850 4851 // If src0 can be used as src1, commuting will make the operands legal. 4852 // Otherwise we have to give up and insert a move. 4853 // 4854 // TODO: Other immediate-like operand kinds could be commuted if there was a 4855 // MachineOperand::ChangeTo* for them. 4856 if ((!Src1.isImm() && !Src1.isReg()) || 4857 !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) { 4858 legalizeOpWithMove(MI, Src1Idx); 4859 return; 4860 } 4861 4862 int CommutedOpc = commuteOpcode(MI); 4863 if (CommutedOpc == -1) { 4864 legalizeOpWithMove(MI, Src1Idx); 4865 return; 4866 } 4867 4868 MI.setDesc(get(CommutedOpc)); 4869 4870 Register Src0Reg = Src0.getReg(); 4871 unsigned Src0SubReg = Src0.getSubReg(); 4872 bool Src0Kill = Src0.isKill(); 4873 4874 if (Src1.isImm()) 4875 Src0.ChangeToImmediate(Src1.getImm()); 4876 else if (Src1.isReg()) { 4877 Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill()); 4878 Src0.setSubReg(Src1.getSubReg()); 4879 } else 4880 llvm_unreachable("Should only have register or immediate operands"); 4881 4882 Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill); 4883 Src1.setSubReg(Src0SubReg); 4884 fixImplicitOperands(MI); 4885 } 4886 4887 // Legalize VOP3 operands. All operand types are supported for any operand 4888 // but only one literal constant and only starting from GFX10. 4889 void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI, 4890 MachineInstr &MI) const { 4891 unsigned Opc = MI.getOpcode(); 4892 4893 int VOP3Idx[3] = { 4894 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0), 4895 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1), 4896 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2) 4897 }; 4898 4899 if (Opc == AMDGPU::V_PERMLANE16_B32_e64 || 4900 Opc == AMDGPU::V_PERMLANEX16_B32_e64) { 4901 // src1 and src2 must be scalar 4902 MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]); 4903 MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]); 4904 const DebugLoc &DL = MI.getDebugLoc(); 4905 if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) { 4906 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4907 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4908 .add(Src1); 4909 Src1.ChangeToRegister(Reg, false); 4910 } 4911 if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) { 4912 Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 4913 BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg) 4914 .add(Src2); 4915 Src2.ChangeToRegister(Reg, false); 4916 } 4917 } 4918 4919 // Find the one SGPR operand we are allowed to use. 4920 int ConstantBusLimit = ST.getConstantBusLimit(Opc); 4921 int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0; 4922 SmallDenseSet<unsigned> SGPRsUsed; 4923 Register SGPRReg = findUsedSGPR(MI, VOP3Idx); 4924 if (SGPRReg != AMDGPU::NoRegister) { 4925 SGPRsUsed.insert(SGPRReg); 4926 --ConstantBusLimit; 4927 } 4928 4929 for (unsigned i = 0; i < 3; ++i) { 4930 int Idx = VOP3Idx[i]; 4931 if (Idx == -1) 4932 break; 4933 MachineOperand &MO = MI.getOperand(Idx); 4934 4935 if (!MO.isReg()) { 4936 if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx])) 4937 continue; 4938 4939 if (LiteralLimit > 0 && ConstantBusLimit > 0) { 4940 --LiteralLimit; 4941 --ConstantBusLimit; 4942 continue; 4943 } 4944 4945 --LiteralLimit; 4946 --ConstantBusLimit; 4947 legalizeOpWithMove(MI, Idx); 4948 continue; 4949 } 4950 4951 if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) && 4952 !isOperandLegal(MI, Idx, &MO)) { 4953 legalizeOpWithMove(MI, Idx); 4954 continue; 4955 } 4956 4957 if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg()))) 4958 continue; // VGPRs are legal 4959 4960 // We can use one SGPR in each VOP3 instruction prior to GFX10 4961 // and two starting from GFX10. 4962 if (SGPRsUsed.count(MO.getReg())) 4963 continue; 4964 if (ConstantBusLimit > 0) { 4965 SGPRsUsed.insert(MO.getReg()); 4966 --ConstantBusLimit; 4967 continue; 4968 } 4969 4970 // If we make it this far, then the operand is not legal and we must 4971 // legalize it. 4972 legalizeOpWithMove(MI, Idx); 4973 } 4974 } 4975 4976 Register SIInstrInfo::readlaneVGPRToSGPR(Register SrcReg, MachineInstr &UseMI, 4977 MachineRegisterInfo &MRI) const { 4978 const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg); 4979 const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC); 4980 Register DstReg = MRI.createVirtualRegister(SRC); 4981 unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32; 4982 4983 if (RI.hasAGPRs(VRC)) { 4984 VRC = RI.getEquivalentVGPRClass(VRC); 4985 Register NewSrcReg = MRI.createVirtualRegister(VRC); 4986 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4987 get(TargetOpcode::COPY), NewSrcReg) 4988 .addReg(SrcReg); 4989 SrcReg = NewSrcReg; 4990 } 4991 4992 if (SubRegs == 1) { 4993 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 4994 get(AMDGPU::V_READFIRSTLANE_B32), DstReg) 4995 .addReg(SrcReg); 4996 return DstReg; 4997 } 4998 4999 SmallVector<unsigned, 8> SRegs; 5000 for (unsigned i = 0; i < SubRegs; ++i) { 5001 Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5002 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5003 get(AMDGPU::V_READFIRSTLANE_B32), SGPR) 5004 .addReg(SrcReg, 0, RI.getSubRegFromChannel(i)); 5005 SRegs.push_back(SGPR); 5006 } 5007 5008 MachineInstrBuilder MIB = 5009 BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(), 5010 get(AMDGPU::REG_SEQUENCE), DstReg); 5011 for (unsigned i = 0; i < SubRegs; ++i) { 5012 MIB.addReg(SRegs[i]); 5013 MIB.addImm(RI.getSubRegFromChannel(i)); 5014 } 5015 return DstReg; 5016 } 5017 5018 void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI, 5019 MachineInstr &MI) const { 5020 5021 // If the pointer is store in VGPRs, then we need to move them to 5022 // SGPRs using v_readfirstlane. This is safe because we only select 5023 // loads with uniform pointers to SMRD instruction so we know the 5024 // pointer value is uniform. 5025 MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase); 5026 if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) { 5027 Register SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI); 5028 SBase->setReg(SGPR); 5029 } 5030 MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff); 5031 if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) { 5032 Register SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI); 5033 SOff->setReg(SGPR); 5034 } 5035 } 5036 5037 bool SIInstrInfo::moveFlatAddrToVGPR(MachineInstr &Inst) const { 5038 unsigned Opc = Inst.getOpcode(); 5039 int OldSAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr); 5040 if (OldSAddrIdx < 0) 5041 return false; 5042 5043 assert(isSegmentSpecificFLAT(Inst)); 5044 5045 int NewOpc = AMDGPU::getGlobalVaddrOp(Opc); 5046 if (NewOpc < 0) 5047 NewOpc = AMDGPU::getFlatScratchInstSVfromSS(Opc); 5048 if (NewOpc < 0) 5049 return false; 5050 5051 MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo(); 5052 MachineOperand &SAddr = Inst.getOperand(OldSAddrIdx); 5053 if (RI.isSGPRReg(MRI, SAddr.getReg())) 5054 return false; 5055 5056 int NewVAddrIdx = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vaddr); 5057 if (NewVAddrIdx < 0) 5058 return false; 5059 5060 int OldVAddrIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr); 5061 5062 // Check vaddr, it shall be zero or absent. 5063 MachineInstr *VAddrDef = nullptr; 5064 if (OldVAddrIdx >= 0) { 5065 MachineOperand &VAddr = Inst.getOperand(OldVAddrIdx); 5066 VAddrDef = MRI.getUniqueVRegDef(VAddr.getReg()); 5067 if (!VAddrDef || VAddrDef->getOpcode() != AMDGPU::V_MOV_B32_e32 || 5068 !VAddrDef->getOperand(1).isImm() || 5069 VAddrDef->getOperand(1).getImm() != 0) 5070 return false; 5071 } 5072 5073 const MCInstrDesc &NewDesc = get(NewOpc); 5074 Inst.setDesc(NewDesc); 5075 5076 // Callers expect interator to be valid after this call, so modify the 5077 // instruction in place. 5078 if (OldVAddrIdx == NewVAddrIdx) { 5079 MachineOperand &NewVAddr = Inst.getOperand(NewVAddrIdx); 5080 // Clear use list from the old vaddr holding a zero register. 5081 MRI.removeRegOperandFromUseList(&NewVAddr); 5082 MRI.moveOperands(&NewVAddr, &SAddr, 1); 5083 Inst.RemoveOperand(OldSAddrIdx); 5084 // Update the use list with the pointer we have just moved from vaddr to 5085 // saddr poisition. Otherwise new vaddr will be missing from the use list. 5086 MRI.removeRegOperandFromUseList(&NewVAddr); 5087 MRI.addRegOperandToUseList(&NewVAddr); 5088 } else { 5089 assert(OldSAddrIdx == NewVAddrIdx); 5090 5091 if (OldVAddrIdx >= 0) { 5092 int NewVDstIn = AMDGPU::getNamedOperandIdx(NewOpc, 5093 AMDGPU::OpName::vdst_in); 5094 5095 // RemoveOperand doesn't try to fixup tied operand indexes at it goes, so 5096 // it asserts. Untie the operands for now and retie them afterwards. 5097 if (NewVDstIn != -1) { 5098 int OldVDstIn = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in); 5099 Inst.untieRegOperand(OldVDstIn); 5100 } 5101 5102 Inst.RemoveOperand(OldVAddrIdx); 5103 5104 if (NewVDstIn != -1) { 5105 int NewVDst = AMDGPU::getNamedOperandIdx(NewOpc, AMDGPU::OpName::vdst); 5106 Inst.tieOperands(NewVDst, NewVDstIn); 5107 } 5108 } 5109 } 5110 5111 if (VAddrDef && MRI.use_nodbg_empty(VAddrDef->getOperand(0).getReg())) 5112 VAddrDef->eraseFromParent(); 5113 5114 return true; 5115 } 5116 5117 // FIXME: Remove this when SelectionDAG is obsoleted. 5118 void SIInstrInfo::legalizeOperandsFLAT(MachineRegisterInfo &MRI, 5119 MachineInstr &MI) const { 5120 if (!isSegmentSpecificFLAT(MI)) 5121 return; 5122 5123 // Fixup SGPR operands in VGPRs. We only select these when the DAG divergence 5124 // thinks they are uniform, so a readfirstlane should be valid. 5125 MachineOperand *SAddr = getNamedOperand(MI, AMDGPU::OpName::saddr); 5126 if (!SAddr || RI.isSGPRClass(MRI.getRegClass(SAddr->getReg()))) 5127 return; 5128 5129 if (moveFlatAddrToVGPR(MI)) 5130 return; 5131 5132 Register ToSGPR = readlaneVGPRToSGPR(SAddr->getReg(), MI, MRI); 5133 SAddr->setReg(ToSGPR); 5134 } 5135 5136 void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB, 5137 MachineBasicBlock::iterator I, 5138 const TargetRegisterClass *DstRC, 5139 MachineOperand &Op, 5140 MachineRegisterInfo &MRI, 5141 const DebugLoc &DL) const { 5142 Register OpReg = Op.getReg(); 5143 unsigned OpSubReg = Op.getSubReg(); 5144 5145 const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg( 5146 RI.getRegClassForReg(MRI, OpReg), OpSubReg); 5147 5148 // Check if operand is already the correct register class. 5149 if (DstRC == OpRC) 5150 return; 5151 5152 Register DstReg = MRI.createVirtualRegister(DstRC); 5153 MachineInstr *Copy = 5154 BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op); 5155 5156 Op.setReg(DstReg); 5157 Op.setSubReg(0); 5158 5159 MachineInstr *Def = MRI.getVRegDef(OpReg); 5160 if (!Def) 5161 return; 5162 5163 // Try to eliminate the copy if it is copying an immediate value. 5164 if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass) 5165 FoldImmediate(*Copy, *Def, OpReg, &MRI); 5166 5167 bool ImpDef = Def->isImplicitDef(); 5168 while (!ImpDef && Def && Def->isCopy()) { 5169 if (Def->getOperand(1).getReg().isPhysical()) 5170 break; 5171 Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg()); 5172 ImpDef = Def && Def->isImplicitDef(); 5173 } 5174 if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) && 5175 !ImpDef) 5176 Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); 5177 } 5178 5179 // Emit the actual waterfall loop, executing the wrapped instruction for each 5180 // unique value of \p Rsrc across all lanes. In the best case we execute 1 5181 // iteration, in the worst case we execute 64 (once per lane). 5182 static void 5183 emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI, 5184 MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, 5185 const DebugLoc &DL, MachineOperand &Rsrc) { 5186 MachineFunction &MF = *OrigBB.getParent(); 5187 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5188 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5189 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5190 unsigned SaveExecOpc = 5191 ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64; 5192 unsigned XorTermOpc = 5193 ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term; 5194 unsigned AndOpc = 5195 ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64; 5196 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5197 5198 MachineBasicBlock::iterator I = LoopBB.begin(); 5199 5200 SmallVector<Register, 8> ReadlanePieces; 5201 Register CondReg = AMDGPU::NoRegister; 5202 5203 Register VRsrc = Rsrc.getReg(); 5204 unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef()); 5205 5206 unsigned RegSize = TRI->getRegSizeInBits(Rsrc.getReg(), MRI); 5207 unsigned NumSubRegs = RegSize / 32; 5208 assert(NumSubRegs % 2 == 0 && NumSubRegs <= 32 && "Unhandled register size"); 5209 5210 for (unsigned Idx = 0; Idx < NumSubRegs; Idx += 2) { 5211 5212 Register CurRegLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5213 Register CurRegHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5214 5215 // Read the next variant <- also loop target. 5216 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegLo) 5217 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx)); 5218 5219 // Read the next variant <- also loop target. 5220 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), CurRegHi) 5221 .addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx + 1)); 5222 5223 ReadlanePieces.push_back(CurRegLo); 5224 ReadlanePieces.push_back(CurRegHi); 5225 5226 // Comparison is to be done as 64-bit. 5227 Register CurReg = MRI.createVirtualRegister(&AMDGPU::SGPR_64RegClass); 5228 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), CurReg) 5229 .addReg(CurRegLo) 5230 .addImm(AMDGPU::sub0) 5231 .addReg(CurRegHi) 5232 .addImm(AMDGPU::sub1); 5233 5234 Register NewCondReg = MRI.createVirtualRegister(BoolXExecRC); 5235 auto Cmp = 5236 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), NewCondReg) 5237 .addReg(CurReg); 5238 if (NumSubRegs <= 2) 5239 Cmp.addReg(VRsrc); 5240 else 5241 Cmp.addReg(VRsrc, VRsrcUndef, TRI->getSubRegFromChannel(Idx, 2)); 5242 5243 // Combine the comparision results with AND. 5244 if (CondReg == AMDGPU::NoRegister) // First. 5245 CondReg = NewCondReg; 5246 else { // If not the first, we create an AND. 5247 Register AndReg = MRI.createVirtualRegister(BoolXExecRC); 5248 BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndReg) 5249 .addReg(CondReg) 5250 .addReg(NewCondReg); 5251 CondReg = AndReg; 5252 } 5253 } // End for loop. 5254 5255 auto SRsrcRC = TRI->getEquivalentSGPRClass(MRI.getRegClass(VRsrc)); 5256 Register SRsrc = MRI.createVirtualRegister(SRsrcRC); 5257 5258 // Build scalar Rsrc. 5259 auto Merge = BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc); 5260 unsigned Channel = 0; 5261 for (Register Piece : ReadlanePieces) { 5262 Merge.addReg(Piece) 5263 .addImm(TRI->getSubRegFromChannel(Channel++)); 5264 } 5265 5266 // Update Rsrc operand to use the SGPR Rsrc. 5267 Rsrc.setReg(SRsrc); 5268 Rsrc.setIsKill(true); 5269 5270 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5271 MRI.setSimpleHint(SaveExec, CondReg); 5272 5273 // Update EXEC to matching lanes, saving original to SaveExec. 5274 BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec) 5275 .addReg(CondReg, RegState::Kill); 5276 5277 // The original instruction is here; we insert the terminators after it. 5278 I = LoopBB.end(); 5279 5280 // Update EXEC, switch all done bits to 0 and all todo bits to 1. 5281 BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec) 5282 .addReg(Exec) 5283 .addReg(SaveExec); 5284 5285 BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB); 5286 } 5287 5288 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register 5289 // with SGPRs by iterating over all unique values across all lanes. 5290 // Returns the loop basic block that now contains \p MI. 5291 static MachineBasicBlock * 5292 loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, 5293 MachineOperand &Rsrc, MachineDominatorTree *MDT, 5294 MachineBasicBlock::iterator Begin = nullptr, 5295 MachineBasicBlock::iterator End = nullptr) { 5296 MachineBasicBlock &MBB = *MI.getParent(); 5297 MachineFunction &MF = *MBB.getParent(); 5298 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 5299 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 5300 MachineRegisterInfo &MRI = MF.getRegInfo(); 5301 if (!Begin.isValid()) 5302 Begin = &MI; 5303 if (!End.isValid()) { 5304 End = &MI; 5305 ++End; 5306 } 5307 const DebugLoc &DL = MI.getDebugLoc(); 5308 unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; 5309 unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; 5310 const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5311 5312 Register SaveExec = MRI.createVirtualRegister(BoolXExecRC); 5313 5314 // Save the EXEC mask 5315 BuildMI(MBB, Begin, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec); 5316 5317 // Killed uses in the instruction we are waterfalling around will be 5318 // incorrect due to the added control-flow. 5319 MachineBasicBlock::iterator AfterMI = MI; 5320 ++AfterMI; 5321 for (auto I = Begin; I != AfterMI; I++) { 5322 for (auto &MO : I->uses()) { 5323 if (MO.isReg() && MO.isUse()) { 5324 MRI.clearKillFlags(MO.getReg()); 5325 } 5326 } 5327 } 5328 5329 // To insert the loop we need to split the block. Move everything after this 5330 // point to a new block, and insert a new empty block between the two. 5331 MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock(); 5332 MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock(); 5333 MachineFunction::iterator MBBI(MBB); 5334 ++MBBI; 5335 5336 MF.insert(MBBI, LoopBB); 5337 MF.insert(MBBI, RemainderBB); 5338 5339 LoopBB->addSuccessor(LoopBB); 5340 LoopBB->addSuccessor(RemainderBB); 5341 5342 // Move Begin to MI to the LoopBB, and the remainder of the block to 5343 // RemainderBB. 5344 RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB); 5345 RemainderBB->splice(RemainderBB->begin(), &MBB, End, MBB.end()); 5346 LoopBB->splice(LoopBB->begin(), &MBB, Begin, MBB.end()); 5347 5348 MBB.addSuccessor(LoopBB); 5349 5350 // Update dominators. We know that MBB immediately dominates LoopBB, that 5351 // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately 5352 // dominates all of the successors transferred to it from MBB that MBB used 5353 // to properly dominate. 5354 if (MDT) { 5355 MDT->addNewBlock(LoopBB, &MBB); 5356 MDT->addNewBlock(RemainderBB, LoopBB); 5357 for (auto &Succ : RemainderBB->successors()) { 5358 if (MDT->properlyDominates(&MBB, Succ)) { 5359 MDT->changeImmediateDominator(Succ, RemainderBB); 5360 } 5361 } 5362 } 5363 5364 emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc); 5365 5366 // Restore the EXEC mask 5367 MachineBasicBlock::iterator First = RemainderBB->begin(); 5368 BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec); 5369 return LoopBB; 5370 } 5371 5372 // Extract pointer from Rsrc and return a zero-value Rsrc replacement. 5373 static std::tuple<unsigned, unsigned> 5374 extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) { 5375 MachineBasicBlock &MBB = *MI.getParent(); 5376 MachineFunction &MF = *MBB.getParent(); 5377 MachineRegisterInfo &MRI = MF.getRegInfo(); 5378 5379 // Extract the ptr from the resource descriptor. 5380 unsigned RsrcPtr = 5381 TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass, 5382 AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass); 5383 5384 // Create an empty resource descriptor 5385 Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 5386 Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5387 Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); 5388 Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass); 5389 uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat(); 5390 5391 // Zero64 = 0 5392 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64) 5393 .addImm(0); 5394 5395 // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0} 5396 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo) 5397 .addImm(RsrcDataFormat & 0xFFFFFFFF); 5398 5399 // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32} 5400 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi) 5401 .addImm(RsrcDataFormat >> 32); 5402 5403 // NewSRsrc = {Zero64, SRsrcFormat} 5404 BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc) 5405 .addReg(Zero64) 5406 .addImm(AMDGPU::sub0_sub1) 5407 .addReg(SRsrcFormatLo) 5408 .addImm(AMDGPU::sub2) 5409 .addReg(SRsrcFormatHi) 5410 .addImm(AMDGPU::sub3); 5411 5412 return std::make_tuple(RsrcPtr, NewSRsrc); 5413 } 5414 5415 MachineBasicBlock * 5416 SIInstrInfo::legalizeOperands(MachineInstr &MI, 5417 MachineDominatorTree *MDT) const { 5418 MachineFunction &MF = *MI.getParent()->getParent(); 5419 MachineRegisterInfo &MRI = MF.getRegInfo(); 5420 MachineBasicBlock *CreatedBB = nullptr; 5421 5422 // Legalize VOP2 5423 if (isVOP2(MI) || isVOPC(MI)) { 5424 legalizeOperandsVOP2(MRI, MI); 5425 return CreatedBB; 5426 } 5427 5428 // Legalize VOP3 5429 if (isVOP3(MI)) { 5430 legalizeOperandsVOP3(MRI, MI); 5431 return CreatedBB; 5432 } 5433 5434 // Legalize SMRD 5435 if (isSMRD(MI)) { 5436 legalizeOperandsSMRD(MRI, MI); 5437 return CreatedBB; 5438 } 5439 5440 // Legalize FLAT 5441 if (isFLAT(MI)) { 5442 legalizeOperandsFLAT(MRI, MI); 5443 return CreatedBB; 5444 } 5445 5446 // Legalize REG_SEQUENCE and PHI 5447 // The register class of the operands much be the same type as the register 5448 // class of the output. 5449 if (MI.getOpcode() == AMDGPU::PHI) { 5450 const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr; 5451 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 5452 if (!MI.getOperand(i).isReg() || !MI.getOperand(i).getReg().isVirtual()) 5453 continue; 5454 const TargetRegisterClass *OpRC = 5455 MRI.getRegClass(MI.getOperand(i).getReg()); 5456 if (RI.hasVectorRegisters(OpRC)) { 5457 VRC = OpRC; 5458 } else { 5459 SRC = OpRC; 5460 } 5461 } 5462 5463 // If any of the operands are VGPR registers, then they all most be 5464 // otherwise we will create illegal VGPR->SGPR copies when legalizing 5465 // them. 5466 if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) { 5467 if (!VRC) { 5468 assert(SRC); 5469 if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) { 5470 VRC = &AMDGPU::VReg_1RegClass; 5471 } else 5472 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5473 ? RI.getEquivalentAGPRClass(SRC) 5474 : RI.getEquivalentVGPRClass(SRC); 5475 } else { 5476 VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) 5477 ? RI.getEquivalentAGPRClass(VRC) 5478 : RI.getEquivalentVGPRClass(VRC); 5479 } 5480 RC = VRC; 5481 } else { 5482 RC = SRC; 5483 } 5484 5485 // Update all the operands so they have the same type. 5486 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5487 MachineOperand &Op = MI.getOperand(I); 5488 if (!Op.isReg() || !Op.getReg().isVirtual()) 5489 continue; 5490 5491 // MI is a PHI instruction. 5492 MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB(); 5493 MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator(); 5494 5495 // Avoid creating no-op copies with the same src and dst reg class. These 5496 // confuse some of the machine passes. 5497 legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc()); 5498 } 5499 } 5500 5501 // REG_SEQUENCE doesn't really require operand legalization, but if one has a 5502 // VGPR dest type and SGPR sources, insert copies so all operands are 5503 // VGPRs. This seems to help operand folding / the register coalescer. 5504 if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) { 5505 MachineBasicBlock *MBB = MI.getParent(); 5506 const TargetRegisterClass *DstRC = getOpRegClass(MI, 0); 5507 if (RI.hasVGPRs(DstRC)) { 5508 // Update all the operands so they are VGPR register classes. These may 5509 // not be the same register class because REG_SEQUENCE supports mixing 5510 // subregister index types e.g. sub0_sub1 + sub2 + sub3 5511 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { 5512 MachineOperand &Op = MI.getOperand(I); 5513 if (!Op.isReg() || !Op.getReg().isVirtual()) 5514 continue; 5515 5516 const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg()); 5517 const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC); 5518 if (VRC == OpRC) 5519 continue; 5520 5521 legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc()); 5522 Op.setIsKill(); 5523 } 5524 } 5525 5526 return CreatedBB; 5527 } 5528 5529 // Legalize INSERT_SUBREG 5530 // src0 must have the same register class as dst 5531 if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) { 5532 Register Dst = MI.getOperand(0).getReg(); 5533 Register Src0 = MI.getOperand(1).getReg(); 5534 const TargetRegisterClass *DstRC = MRI.getRegClass(Dst); 5535 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0); 5536 if (DstRC != Src0RC) { 5537 MachineBasicBlock *MBB = MI.getParent(); 5538 MachineOperand &Op = MI.getOperand(1); 5539 legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc()); 5540 } 5541 return CreatedBB; 5542 } 5543 5544 // Legalize SI_INIT_M0 5545 if (MI.getOpcode() == AMDGPU::SI_INIT_M0) { 5546 MachineOperand &Src = MI.getOperand(0); 5547 if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg()))) 5548 Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI)); 5549 return CreatedBB; 5550 } 5551 5552 // Legalize MIMG and MUBUF/MTBUF for shaders. 5553 // 5554 // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via 5555 // scratch memory access. In both cases, the legalization never involves 5556 // conversion to the addr64 form. 5557 if (isMIMG(MI) || (AMDGPU::isGraphics(MF.getFunction().getCallingConv()) && 5558 (isMUBUF(MI) || isMTBUF(MI)))) { 5559 MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc); 5560 if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) 5561 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SRsrc, MDT); 5562 5563 MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp); 5564 if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) 5565 CreatedBB = loadSRsrcFromVGPR(*this, MI, *SSamp, MDT); 5566 5567 return CreatedBB; 5568 } 5569 5570 // Legalize SI_CALL 5571 if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) { 5572 MachineOperand *Dest = &MI.getOperand(0); 5573 if (!RI.isSGPRClass(MRI.getRegClass(Dest->getReg()))) { 5574 // Move everything between ADJCALLSTACKUP and ADJCALLSTACKDOWN and 5575 // following copies, we also need to move copies from and to physical 5576 // registers into the loop block. 5577 unsigned FrameSetupOpcode = getCallFrameSetupOpcode(); 5578 unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode(); 5579 5580 // Also move the copies to physical registers into the loop block 5581 MachineBasicBlock &MBB = *MI.getParent(); 5582 MachineBasicBlock::iterator Start(&MI); 5583 while (Start->getOpcode() != FrameSetupOpcode) 5584 --Start; 5585 MachineBasicBlock::iterator End(&MI); 5586 while (End->getOpcode() != FrameDestroyOpcode) 5587 ++End; 5588 // Also include following copies of the return value 5589 ++End; 5590 while (End != MBB.end() && End->isCopy() && End->getOperand(1).isReg() && 5591 MI.definesRegister(End->getOperand(1).getReg())) 5592 ++End; 5593 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Dest, MDT, Start, End); 5594 } 5595 } 5596 5597 // Legalize MUBUF* instructions. 5598 int RsrcIdx = 5599 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc); 5600 if (RsrcIdx != -1) { 5601 // We have an MUBUF instruction 5602 MachineOperand *Rsrc = &MI.getOperand(RsrcIdx); 5603 unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass; 5604 if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()), 5605 RI.getRegClass(RsrcRC))) { 5606 // The operands are legal. 5607 // FIXME: We may need to legalize operands besided srsrc. 5608 return CreatedBB; 5609 } 5610 5611 // Legalize a VGPR Rsrc. 5612 // 5613 // If the instruction is _ADDR64, we can avoid a waterfall by extracting 5614 // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using 5615 // a zero-value SRsrc. 5616 // 5617 // If the instruction is _OFFSET (both idxen and offen disabled), and we 5618 // support ADDR64 instructions, we can convert to ADDR64 and do the same as 5619 // above. 5620 // 5621 // Otherwise we are on non-ADDR64 hardware, and/or we have 5622 // idxen/offen/bothen and we fall back to a waterfall loop. 5623 5624 MachineBasicBlock &MBB = *MI.getParent(); 5625 5626 MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 5627 if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) { 5628 // This is already an ADDR64 instruction so we need to add the pointer 5629 // extracted from the resource descriptor to the current value of VAddr. 5630 Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5631 Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 5632 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5633 5634 const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5635 Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC); 5636 Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC); 5637 5638 unsigned RsrcPtr, NewSRsrc; 5639 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5640 5641 // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0 5642 const DebugLoc &DL = MI.getDebugLoc(); 5643 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_CO_U32_e64), NewVAddrLo) 5644 .addDef(CondReg0) 5645 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5646 .addReg(VAddr->getReg(), 0, AMDGPU::sub0) 5647 .addImm(0); 5648 5649 // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1 5650 BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi) 5651 .addDef(CondReg1, RegState::Dead) 5652 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5653 .addReg(VAddr->getReg(), 0, AMDGPU::sub1) 5654 .addReg(CondReg0, RegState::Kill) 5655 .addImm(0); 5656 5657 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5658 BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr) 5659 .addReg(NewVAddrLo) 5660 .addImm(AMDGPU::sub0) 5661 .addReg(NewVAddrHi) 5662 .addImm(AMDGPU::sub1); 5663 5664 VAddr->setReg(NewVAddr); 5665 Rsrc->setReg(NewSRsrc); 5666 } else if (!VAddr && ST.hasAddr64()) { 5667 // This instructions is the _OFFSET variant, so we need to convert it to 5668 // ADDR64. 5669 assert(ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS && 5670 "FIXME: Need to emit flat atomics here"); 5671 5672 unsigned RsrcPtr, NewSRsrc; 5673 std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc); 5674 5675 Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 5676 MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata); 5677 MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset); 5678 MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset); 5679 unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode()); 5680 5681 // Atomics rith return have have an additional tied operand and are 5682 // missing some of the special bits. 5683 MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in); 5684 MachineInstr *Addr64; 5685 5686 if (!VDataIn) { 5687 // Regular buffer load / store. 5688 MachineInstrBuilder MIB = 5689 BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5690 .add(*VData) 5691 .addReg(NewVAddr) 5692 .addReg(NewSRsrc) 5693 .add(*SOffset) 5694 .add(*Offset); 5695 5696 if (const MachineOperand *CPol = 5697 getNamedOperand(MI, AMDGPU::OpName::cpol)) { 5698 MIB.addImm(CPol->getImm()); 5699 } 5700 5701 if (const MachineOperand *TFE = 5702 getNamedOperand(MI, AMDGPU::OpName::tfe)) { 5703 MIB.addImm(TFE->getImm()); 5704 } 5705 5706 MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz)); 5707 5708 MIB.cloneMemRefs(MI); 5709 Addr64 = MIB; 5710 } else { 5711 // Atomics with return. 5712 Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode)) 5713 .add(*VData) 5714 .add(*VDataIn) 5715 .addReg(NewVAddr) 5716 .addReg(NewSRsrc) 5717 .add(*SOffset) 5718 .add(*Offset) 5719 .addImm(getNamedImmOperand(MI, AMDGPU::OpName::cpol)) 5720 .cloneMemRefs(MI); 5721 } 5722 5723 MI.removeFromParent(); 5724 5725 // NewVaddr = {NewVaddrHi, NewVaddrLo} 5726 BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE), 5727 NewVAddr) 5728 .addReg(RsrcPtr, 0, AMDGPU::sub0) 5729 .addImm(AMDGPU::sub0) 5730 .addReg(RsrcPtr, 0, AMDGPU::sub1) 5731 .addImm(AMDGPU::sub1); 5732 } else { 5733 // This is another variant; legalize Rsrc with waterfall loop from VGPRs 5734 // to SGPRs. 5735 CreatedBB = loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT); 5736 return CreatedBB; 5737 } 5738 } 5739 return CreatedBB; 5740 } 5741 5742 MachineBasicBlock *SIInstrInfo::moveToVALU(MachineInstr &TopInst, 5743 MachineDominatorTree *MDT) const { 5744 SetVectorType Worklist; 5745 Worklist.insert(&TopInst); 5746 MachineBasicBlock *CreatedBB = nullptr; 5747 MachineBasicBlock *CreatedBBTmp = nullptr; 5748 5749 while (!Worklist.empty()) { 5750 MachineInstr &Inst = *Worklist.pop_back_val(); 5751 MachineBasicBlock *MBB = Inst.getParent(); 5752 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); 5753 5754 unsigned Opcode = Inst.getOpcode(); 5755 unsigned NewOpcode = getVALUOp(Inst); 5756 5757 // Handle some special cases 5758 switch (Opcode) { 5759 default: 5760 break; 5761 case AMDGPU::S_ADD_U64_PSEUDO: 5762 case AMDGPU::S_SUB_U64_PSEUDO: 5763 splitScalar64BitAddSub(Worklist, Inst, MDT); 5764 Inst.eraseFromParent(); 5765 continue; 5766 case AMDGPU::S_ADD_I32: 5767 case AMDGPU::S_SUB_I32: { 5768 // FIXME: The u32 versions currently selected use the carry. 5769 bool Changed; 5770 std::tie(Changed, CreatedBBTmp) = moveScalarAddSub(Worklist, Inst, MDT); 5771 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5772 CreatedBB = CreatedBBTmp; 5773 if (Changed) 5774 continue; 5775 5776 // Default handling 5777 break; 5778 } 5779 case AMDGPU::S_AND_B64: 5780 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT); 5781 Inst.eraseFromParent(); 5782 continue; 5783 5784 case AMDGPU::S_OR_B64: 5785 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT); 5786 Inst.eraseFromParent(); 5787 continue; 5788 5789 case AMDGPU::S_XOR_B64: 5790 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT); 5791 Inst.eraseFromParent(); 5792 continue; 5793 5794 case AMDGPU::S_NAND_B64: 5795 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT); 5796 Inst.eraseFromParent(); 5797 continue; 5798 5799 case AMDGPU::S_NOR_B64: 5800 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT); 5801 Inst.eraseFromParent(); 5802 continue; 5803 5804 case AMDGPU::S_XNOR_B64: 5805 if (ST.hasDLInsts()) 5806 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT); 5807 else 5808 splitScalar64BitXnor(Worklist, Inst, MDT); 5809 Inst.eraseFromParent(); 5810 continue; 5811 5812 case AMDGPU::S_ANDN2_B64: 5813 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT); 5814 Inst.eraseFromParent(); 5815 continue; 5816 5817 case AMDGPU::S_ORN2_B64: 5818 splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT); 5819 Inst.eraseFromParent(); 5820 continue; 5821 5822 case AMDGPU::S_BREV_B64: 5823 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_BREV_B32, true); 5824 Inst.eraseFromParent(); 5825 continue; 5826 5827 case AMDGPU::S_NOT_B64: 5828 splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32); 5829 Inst.eraseFromParent(); 5830 continue; 5831 5832 case AMDGPU::S_BCNT1_I32_B64: 5833 splitScalar64BitBCNT(Worklist, Inst); 5834 Inst.eraseFromParent(); 5835 continue; 5836 5837 case AMDGPU::S_BFE_I64: 5838 splitScalar64BitBFE(Worklist, Inst); 5839 Inst.eraseFromParent(); 5840 continue; 5841 5842 case AMDGPU::S_LSHL_B32: 5843 if (ST.hasOnlyRevVALUShifts()) { 5844 NewOpcode = AMDGPU::V_LSHLREV_B32_e64; 5845 swapOperands(Inst); 5846 } 5847 break; 5848 case AMDGPU::S_ASHR_I32: 5849 if (ST.hasOnlyRevVALUShifts()) { 5850 NewOpcode = AMDGPU::V_ASHRREV_I32_e64; 5851 swapOperands(Inst); 5852 } 5853 break; 5854 case AMDGPU::S_LSHR_B32: 5855 if (ST.hasOnlyRevVALUShifts()) { 5856 NewOpcode = AMDGPU::V_LSHRREV_B32_e64; 5857 swapOperands(Inst); 5858 } 5859 break; 5860 case AMDGPU::S_LSHL_B64: 5861 if (ST.hasOnlyRevVALUShifts()) { 5862 NewOpcode = AMDGPU::V_LSHLREV_B64_e64; 5863 swapOperands(Inst); 5864 } 5865 break; 5866 case AMDGPU::S_ASHR_I64: 5867 if (ST.hasOnlyRevVALUShifts()) { 5868 NewOpcode = AMDGPU::V_ASHRREV_I64_e64; 5869 swapOperands(Inst); 5870 } 5871 break; 5872 case AMDGPU::S_LSHR_B64: 5873 if (ST.hasOnlyRevVALUShifts()) { 5874 NewOpcode = AMDGPU::V_LSHRREV_B64_e64; 5875 swapOperands(Inst); 5876 } 5877 break; 5878 5879 case AMDGPU::S_ABS_I32: 5880 lowerScalarAbs(Worklist, Inst); 5881 Inst.eraseFromParent(); 5882 continue; 5883 5884 case AMDGPU::S_CBRANCH_SCC0: 5885 case AMDGPU::S_CBRANCH_SCC1: 5886 // Clear unused bits of vcc 5887 if (ST.isWave32()) 5888 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32), 5889 AMDGPU::VCC_LO) 5890 .addReg(AMDGPU::EXEC_LO) 5891 .addReg(AMDGPU::VCC_LO); 5892 else 5893 BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64), 5894 AMDGPU::VCC) 5895 .addReg(AMDGPU::EXEC) 5896 .addReg(AMDGPU::VCC); 5897 break; 5898 5899 case AMDGPU::S_BFE_U64: 5900 case AMDGPU::S_BFM_B64: 5901 llvm_unreachable("Moving this op to VALU not implemented"); 5902 5903 case AMDGPU::S_PACK_LL_B32_B16: 5904 case AMDGPU::S_PACK_LH_B32_B16: 5905 case AMDGPU::S_PACK_HH_B32_B16: 5906 movePackToVALU(Worklist, MRI, Inst); 5907 Inst.eraseFromParent(); 5908 continue; 5909 5910 case AMDGPU::S_XNOR_B32: 5911 lowerScalarXnor(Worklist, Inst); 5912 Inst.eraseFromParent(); 5913 continue; 5914 5915 case AMDGPU::S_NAND_B32: 5916 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32); 5917 Inst.eraseFromParent(); 5918 continue; 5919 5920 case AMDGPU::S_NOR_B32: 5921 splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32); 5922 Inst.eraseFromParent(); 5923 continue; 5924 5925 case AMDGPU::S_ANDN2_B32: 5926 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32); 5927 Inst.eraseFromParent(); 5928 continue; 5929 5930 case AMDGPU::S_ORN2_B32: 5931 splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32); 5932 Inst.eraseFromParent(); 5933 continue; 5934 5935 // TODO: remove as soon as everything is ready 5936 // to replace VGPR to SGPR copy with V_READFIRSTLANEs. 5937 // S_ADD/SUB_CO_PSEUDO as well as S_UADDO/USUBO_PSEUDO 5938 // can only be selected from the uniform SDNode. 5939 case AMDGPU::S_ADD_CO_PSEUDO: 5940 case AMDGPU::S_SUB_CO_PSEUDO: { 5941 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) 5942 ? AMDGPU::V_ADDC_U32_e64 5943 : AMDGPU::V_SUBB_U32_e64; 5944 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 5945 5946 Register CarryInReg = Inst.getOperand(4).getReg(); 5947 if (!MRI.constrainRegClass(CarryInReg, CarryRC)) { 5948 Register NewCarryReg = MRI.createVirtualRegister(CarryRC); 5949 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(AMDGPU::COPY), NewCarryReg) 5950 .addReg(CarryInReg); 5951 } 5952 5953 Register CarryOutReg = Inst.getOperand(1).getReg(); 5954 5955 Register DestReg = MRI.createVirtualRegister(RI.getEquivalentVGPRClass( 5956 MRI.getRegClass(Inst.getOperand(0).getReg()))); 5957 MachineInstr *CarryOp = 5958 BuildMI(*MBB, &Inst, Inst.getDebugLoc(), get(Opc), DestReg) 5959 .addReg(CarryOutReg, RegState::Define) 5960 .add(Inst.getOperand(2)) 5961 .add(Inst.getOperand(3)) 5962 .addReg(CarryInReg) 5963 .addImm(0); 5964 CreatedBBTmp = legalizeOperands(*CarryOp); 5965 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5966 CreatedBB = CreatedBBTmp; 5967 MRI.replaceRegWith(Inst.getOperand(0).getReg(), DestReg); 5968 addUsersToMoveToVALUWorklist(DestReg, MRI, Worklist); 5969 Inst.eraseFromParent(); 5970 } 5971 continue; 5972 case AMDGPU::S_UADDO_PSEUDO: 5973 case AMDGPU::S_USUBO_PSEUDO: { 5974 const DebugLoc &DL = Inst.getDebugLoc(); 5975 MachineOperand &Dest0 = Inst.getOperand(0); 5976 MachineOperand &Dest1 = Inst.getOperand(1); 5977 MachineOperand &Src0 = Inst.getOperand(2); 5978 MachineOperand &Src1 = Inst.getOperand(3); 5979 5980 unsigned Opc = (Inst.getOpcode() == AMDGPU::S_UADDO_PSEUDO) 5981 ? AMDGPU::V_ADD_CO_U32_e64 5982 : AMDGPU::V_SUB_CO_U32_e64; 5983 const TargetRegisterClass *NewRC = 5984 RI.getEquivalentVGPRClass(MRI.getRegClass(Dest0.getReg())); 5985 Register DestReg = MRI.createVirtualRegister(NewRC); 5986 MachineInstr *NewInstr = BuildMI(*MBB, &Inst, DL, get(Opc), DestReg) 5987 .addReg(Dest1.getReg(), RegState::Define) 5988 .add(Src0) 5989 .add(Src1) 5990 .addImm(0); // clamp bit 5991 5992 CreatedBBTmp = legalizeOperands(*NewInstr, MDT); 5993 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 5994 CreatedBB = CreatedBBTmp; 5995 5996 MRI.replaceRegWith(Dest0.getReg(), DestReg); 5997 addUsersToMoveToVALUWorklist(NewInstr->getOperand(0).getReg(), MRI, 5998 Worklist); 5999 Inst.eraseFromParent(); 6000 } 6001 continue; 6002 6003 case AMDGPU::S_CSELECT_B32: 6004 case AMDGPU::S_CSELECT_B64: 6005 lowerSelect(Worklist, Inst, MDT); 6006 Inst.eraseFromParent(); 6007 continue; 6008 } 6009 6010 if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) { 6011 // We cannot move this instruction to the VALU, so we should try to 6012 // legalize its operands instead. 6013 CreatedBBTmp = legalizeOperands(Inst, MDT); 6014 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6015 CreatedBB = CreatedBBTmp; 6016 continue; 6017 } 6018 6019 // Use the new VALU Opcode. 6020 const MCInstrDesc &NewDesc = get(NewOpcode); 6021 Inst.setDesc(NewDesc); 6022 6023 // Remove any references to SCC. Vector instructions can't read from it, and 6024 // We're just about to add the implicit use / defs of VCC, and we don't want 6025 // both. 6026 for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) { 6027 MachineOperand &Op = Inst.getOperand(i); 6028 if (Op.isReg() && Op.getReg() == AMDGPU::SCC) { 6029 // Only propagate through live-def of SCC. 6030 if (Op.isDef() && !Op.isDead()) 6031 addSCCDefUsersToVALUWorklist(Op, Inst, Worklist); 6032 if (Op.isUse()) 6033 addSCCDefsToVALUWorklist(Op, Worklist); 6034 Inst.RemoveOperand(i); 6035 } 6036 } 6037 6038 if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) { 6039 // We are converting these to a BFE, so we need to add the missing 6040 // operands for the size and offset. 6041 unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16; 6042 Inst.addOperand(MachineOperand::CreateImm(0)); 6043 Inst.addOperand(MachineOperand::CreateImm(Size)); 6044 6045 } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) { 6046 // The VALU version adds the second operand to the result, so insert an 6047 // extra 0 operand. 6048 Inst.addOperand(MachineOperand::CreateImm(0)); 6049 } 6050 6051 Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent()); 6052 fixImplicitOperands(Inst); 6053 6054 if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) { 6055 const MachineOperand &OffsetWidthOp = Inst.getOperand(2); 6056 // If we need to move this to VGPRs, we need to unpack the second operand 6057 // back into the 2 separate ones for bit offset and width. 6058 assert(OffsetWidthOp.isImm() && 6059 "Scalar BFE is only implemented for constant width and offset"); 6060 uint32_t Imm = OffsetWidthOp.getImm(); 6061 6062 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6063 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6064 Inst.RemoveOperand(2); // Remove old immediate. 6065 Inst.addOperand(MachineOperand::CreateImm(Offset)); 6066 Inst.addOperand(MachineOperand::CreateImm(BitWidth)); 6067 } 6068 6069 bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef(); 6070 unsigned NewDstReg = AMDGPU::NoRegister; 6071 if (HasDst) { 6072 Register DstReg = Inst.getOperand(0).getReg(); 6073 if (DstReg.isPhysical()) 6074 continue; 6075 6076 // Update the destination register class. 6077 const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst); 6078 if (!NewDstRC) 6079 continue; 6080 6081 if (Inst.isCopy() && Inst.getOperand(1).getReg().isVirtual() && 6082 NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) { 6083 // Instead of creating a copy where src and dst are the same register 6084 // class, we just replace all uses of dst with src. These kinds of 6085 // copies interfere with the heuristics MachineSink uses to decide 6086 // whether or not to split a critical edge. Since the pass assumes 6087 // that copies will end up as machine instructions and not be 6088 // eliminated. 6089 addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist); 6090 MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg()); 6091 MRI.clearKillFlags(Inst.getOperand(1).getReg()); 6092 Inst.getOperand(0).setReg(DstReg); 6093 6094 // Make sure we don't leave around a dead VGPR->SGPR copy. Normally 6095 // these are deleted later, but at -O0 it would leave a suspicious 6096 // looking illegal copy of an undef register. 6097 for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I) 6098 Inst.RemoveOperand(I); 6099 Inst.setDesc(get(AMDGPU::IMPLICIT_DEF)); 6100 continue; 6101 } 6102 6103 NewDstReg = MRI.createVirtualRegister(NewDstRC); 6104 MRI.replaceRegWith(DstReg, NewDstReg); 6105 } 6106 6107 // Legalize the operands 6108 CreatedBBTmp = legalizeOperands(Inst, MDT); 6109 if (CreatedBBTmp && TopInst.getParent() == CreatedBBTmp) 6110 CreatedBB = CreatedBBTmp; 6111 6112 if (HasDst) 6113 addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist); 6114 } 6115 return CreatedBB; 6116 } 6117 6118 // Add/sub require special handling to deal with carry outs. 6119 std::pair<bool, MachineBasicBlock *> 6120 SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst, 6121 MachineDominatorTree *MDT) const { 6122 if (ST.hasAddNoCarry()) { 6123 // Assume there is no user of scc since we don't select this in that case. 6124 // Since scc isn't used, it doesn't really matter if the i32 or u32 variant 6125 // is used. 6126 6127 MachineBasicBlock &MBB = *Inst.getParent(); 6128 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6129 6130 Register OldDstReg = Inst.getOperand(0).getReg(); 6131 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6132 6133 unsigned Opc = Inst.getOpcode(); 6134 assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32); 6135 6136 unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ? 6137 AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64; 6138 6139 assert(Inst.getOperand(3).getReg() == AMDGPU::SCC); 6140 Inst.RemoveOperand(3); 6141 6142 Inst.setDesc(get(NewOpc)); 6143 Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit 6144 Inst.addImplicitDefUseOperands(*MBB.getParent()); 6145 MRI.replaceRegWith(OldDstReg, ResultReg); 6146 MachineBasicBlock *NewBB = legalizeOperands(Inst, MDT); 6147 6148 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6149 return std::make_pair(true, NewBB); 6150 } 6151 6152 return std::make_pair(false, nullptr); 6153 } 6154 6155 void SIInstrInfo::lowerSelect(SetVectorType &Worklist, MachineInstr &Inst, 6156 MachineDominatorTree *MDT) const { 6157 6158 MachineBasicBlock &MBB = *Inst.getParent(); 6159 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6160 MachineBasicBlock::iterator MII = Inst; 6161 DebugLoc DL = Inst.getDebugLoc(); 6162 6163 MachineOperand &Dest = Inst.getOperand(0); 6164 MachineOperand &Src0 = Inst.getOperand(1); 6165 MachineOperand &Src1 = Inst.getOperand(2); 6166 MachineOperand &Cond = Inst.getOperand(3); 6167 6168 Register SCCSource = Cond.getReg(); 6169 // Find SCC def, and if that is a copy (SCC = COPY reg) then use reg instead. 6170 if (!Cond.isUndef()) { 6171 for (MachineInstr &CandI : 6172 make_range(std::next(MachineBasicBlock::reverse_iterator(Inst)), 6173 Inst.getParent()->rend())) { 6174 if (CandI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != 6175 -1) { 6176 if (CandI.isCopy() && CandI.getOperand(0).getReg() == AMDGPU::SCC) { 6177 SCCSource = CandI.getOperand(1).getReg(); 6178 } 6179 break; 6180 } 6181 } 6182 } 6183 6184 // If this is a trivial select where the condition is effectively not SCC 6185 // (SCCSource is a source of copy to SCC), then the select is semantically 6186 // equivalent to copying SCCSource. Hence, there is no need to create 6187 // V_CNDMASK, we can just use that and bail out. 6188 if ((SCCSource != AMDGPU::SCC) && Src0.isImm() && (Src0.getImm() == -1) && 6189 Src1.isImm() && (Src1.getImm() == 0)) { 6190 MRI.replaceRegWith(Dest.getReg(), SCCSource); 6191 return; 6192 } 6193 6194 const TargetRegisterClass *TC = ST.getWavefrontSize() == 64 6195 ? &AMDGPU::SReg_64_XEXECRegClass 6196 : &AMDGPU::SReg_32_XM0_XEXECRegClass; 6197 Register CopySCC = MRI.createVirtualRegister(TC); 6198 6199 if (SCCSource == AMDGPU::SCC) { 6200 // Insert a trivial select instead of creating a copy, because a copy from 6201 // SCC would semantically mean just copying a single bit, but we may need 6202 // the result to be a vector condition mask that needs preserving. 6203 unsigned Opcode = (ST.getWavefrontSize() == 64) ? AMDGPU::S_CSELECT_B64 6204 : AMDGPU::S_CSELECT_B32; 6205 auto NewSelect = 6206 BuildMI(MBB, MII, DL, get(Opcode), CopySCC).addImm(-1).addImm(0); 6207 NewSelect->getOperand(3).setIsUndef(Cond.isUndef()); 6208 } else { 6209 BuildMI(MBB, MII, DL, get(AMDGPU::COPY), CopySCC).addReg(SCCSource); 6210 } 6211 6212 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6213 6214 auto UpdatedInst = 6215 BuildMI(MBB, MII, DL, get(AMDGPU::V_CNDMASK_B32_e64), ResultReg) 6216 .addImm(0) 6217 .add(Src1) // False 6218 .addImm(0) 6219 .add(Src0) // True 6220 .addReg(CopySCC); 6221 6222 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6223 legalizeOperands(*UpdatedInst, MDT); 6224 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6225 } 6226 6227 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist, 6228 MachineInstr &Inst) const { 6229 MachineBasicBlock &MBB = *Inst.getParent(); 6230 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6231 MachineBasicBlock::iterator MII = Inst; 6232 DebugLoc DL = Inst.getDebugLoc(); 6233 6234 MachineOperand &Dest = Inst.getOperand(0); 6235 MachineOperand &Src = Inst.getOperand(1); 6236 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6237 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6238 6239 unsigned SubOp = ST.hasAddNoCarry() ? 6240 AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_CO_U32_e32; 6241 6242 BuildMI(MBB, MII, DL, get(SubOp), TmpReg) 6243 .addImm(0) 6244 .addReg(Src.getReg()); 6245 6246 BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg) 6247 .addReg(Src.getReg()) 6248 .addReg(TmpReg); 6249 6250 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6251 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6252 } 6253 6254 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist, 6255 MachineInstr &Inst) const { 6256 MachineBasicBlock &MBB = *Inst.getParent(); 6257 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6258 MachineBasicBlock::iterator MII = Inst; 6259 const DebugLoc &DL = Inst.getDebugLoc(); 6260 6261 MachineOperand &Dest = Inst.getOperand(0); 6262 MachineOperand &Src0 = Inst.getOperand(1); 6263 MachineOperand &Src1 = Inst.getOperand(2); 6264 6265 if (ST.hasDLInsts()) { 6266 Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6267 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL); 6268 legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL); 6269 6270 BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest) 6271 .add(Src0) 6272 .add(Src1); 6273 6274 MRI.replaceRegWith(Dest.getReg(), NewDest); 6275 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6276 } else { 6277 // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can 6278 // invert either source and then perform the XOR. If either source is a 6279 // scalar register, then we can leave the inversion on the scalar unit to 6280 // acheive a better distrubution of scalar and vector instructions. 6281 bool Src0IsSGPR = Src0.isReg() && 6282 RI.isSGPRClass(MRI.getRegClass(Src0.getReg())); 6283 bool Src1IsSGPR = Src1.isReg() && 6284 RI.isSGPRClass(MRI.getRegClass(Src1.getReg())); 6285 MachineInstr *Xor; 6286 Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6287 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6288 6289 // Build a pair of scalar instructions and add them to the work list. 6290 // The next iteration over the work list will lower these to the vector 6291 // unit as necessary. 6292 if (Src0IsSGPR) { 6293 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0); 6294 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6295 .addReg(Temp) 6296 .add(Src1); 6297 } else if (Src1IsSGPR) { 6298 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1); 6299 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest) 6300 .add(Src0) 6301 .addReg(Temp); 6302 } else { 6303 Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp) 6304 .add(Src0) 6305 .add(Src1); 6306 MachineInstr *Not = 6307 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp); 6308 Worklist.insert(Not); 6309 } 6310 6311 MRI.replaceRegWith(Dest.getReg(), NewDest); 6312 6313 Worklist.insert(Xor); 6314 6315 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6316 } 6317 } 6318 6319 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist, 6320 MachineInstr &Inst, 6321 unsigned Opcode) const { 6322 MachineBasicBlock &MBB = *Inst.getParent(); 6323 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6324 MachineBasicBlock::iterator MII = Inst; 6325 const DebugLoc &DL = Inst.getDebugLoc(); 6326 6327 MachineOperand &Dest = Inst.getOperand(0); 6328 MachineOperand &Src0 = Inst.getOperand(1); 6329 MachineOperand &Src1 = Inst.getOperand(2); 6330 6331 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6332 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); 6333 6334 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm) 6335 .add(Src0) 6336 .add(Src1); 6337 6338 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest) 6339 .addReg(Interm); 6340 6341 Worklist.insert(&Op); 6342 Worklist.insert(&Not); 6343 6344 MRI.replaceRegWith(Dest.getReg(), NewDest); 6345 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6346 } 6347 6348 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist, 6349 MachineInstr &Inst, 6350 unsigned Opcode) const { 6351 MachineBasicBlock &MBB = *Inst.getParent(); 6352 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6353 MachineBasicBlock::iterator MII = Inst; 6354 const DebugLoc &DL = Inst.getDebugLoc(); 6355 6356 MachineOperand &Dest = Inst.getOperand(0); 6357 MachineOperand &Src0 = Inst.getOperand(1); 6358 MachineOperand &Src1 = Inst.getOperand(2); 6359 6360 Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6361 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); 6362 6363 MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm) 6364 .add(Src1); 6365 6366 MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest) 6367 .add(Src0) 6368 .addReg(Interm); 6369 6370 Worklist.insert(&Not); 6371 Worklist.insert(&Op); 6372 6373 MRI.replaceRegWith(Dest.getReg(), NewDest); 6374 addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist); 6375 } 6376 6377 void SIInstrInfo::splitScalar64BitUnaryOp( 6378 SetVectorType &Worklist, MachineInstr &Inst, 6379 unsigned Opcode, bool Swap) const { 6380 MachineBasicBlock &MBB = *Inst.getParent(); 6381 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6382 6383 MachineOperand &Dest = Inst.getOperand(0); 6384 MachineOperand &Src0 = Inst.getOperand(1); 6385 DebugLoc DL = Inst.getDebugLoc(); 6386 6387 MachineBasicBlock::iterator MII = Inst; 6388 6389 const MCInstrDesc &InstDesc = get(Opcode); 6390 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6391 MRI.getRegClass(Src0.getReg()) : 6392 &AMDGPU::SGPR_32RegClass; 6393 6394 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6395 6396 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6397 AMDGPU::sub0, Src0SubRC); 6398 6399 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6400 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6401 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6402 6403 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6404 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0); 6405 6406 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6407 AMDGPU::sub1, Src0SubRC); 6408 6409 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6410 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1); 6411 6412 if (Swap) 6413 std::swap(DestSub0, DestSub1); 6414 6415 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6416 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6417 .addReg(DestSub0) 6418 .addImm(AMDGPU::sub0) 6419 .addReg(DestSub1) 6420 .addImm(AMDGPU::sub1); 6421 6422 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6423 6424 Worklist.insert(&LoHalf); 6425 Worklist.insert(&HiHalf); 6426 6427 // We don't need to legalizeOperands here because for a single operand, src0 6428 // will support any kind of input. 6429 6430 // Move all users of this moved value. 6431 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6432 } 6433 6434 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist, 6435 MachineInstr &Inst, 6436 MachineDominatorTree *MDT) const { 6437 bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO); 6438 6439 MachineBasicBlock &MBB = *Inst.getParent(); 6440 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6441 const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID); 6442 6443 Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6444 Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6445 Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6446 6447 Register CarryReg = MRI.createVirtualRegister(CarryRC); 6448 Register DeadCarryReg = MRI.createVirtualRegister(CarryRC); 6449 6450 MachineOperand &Dest = Inst.getOperand(0); 6451 MachineOperand &Src0 = Inst.getOperand(1); 6452 MachineOperand &Src1 = Inst.getOperand(2); 6453 const DebugLoc &DL = Inst.getDebugLoc(); 6454 MachineBasicBlock::iterator MII = Inst; 6455 6456 const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg()); 6457 const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg()); 6458 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6459 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6460 6461 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6462 AMDGPU::sub0, Src0SubRC); 6463 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6464 AMDGPU::sub0, Src1SubRC); 6465 6466 6467 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6468 AMDGPU::sub1, Src0SubRC); 6469 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6470 AMDGPU::sub1, Src1SubRC); 6471 6472 unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_CO_U32_e64 : AMDGPU::V_SUB_CO_U32_e64; 6473 MachineInstr *LoHalf = 6474 BuildMI(MBB, MII, DL, get(LoOpc), DestSub0) 6475 .addReg(CarryReg, RegState::Define) 6476 .add(SrcReg0Sub0) 6477 .add(SrcReg1Sub0) 6478 .addImm(0); // clamp bit 6479 6480 unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; 6481 MachineInstr *HiHalf = 6482 BuildMI(MBB, MII, DL, get(HiOpc), DestSub1) 6483 .addReg(DeadCarryReg, RegState::Define | RegState::Dead) 6484 .add(SrcReg0Sub1) 6485 .add(SrcReg1Sub1) 6486 .addReg(CarryReg, RegState::Kill) 6487 .addImm(0); // clamp bit 6488 6489 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6490 .addReg(DestSub0) 6491 .addImm(AMDGPU::sub0) 6492 .addReg(DestSub1) 6493 .addImm(AMDGPU::sub1); 6494 6495 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6496 6497 // Try to legalize the operands in case we need to swap the order to keep it 6498 // valid. 6499 legalizeOperands(*LoHalf, MDT); 6500 legalizeOperands(*HiHalf, MDT); 6501 6502 // Move all users of this moved vlaue. 6503 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6504 } 6505 6506 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist, 6507 MachineInstr &Inst, unsigned Opcode, 6508 MachineDominatorTree *MDT) const { 6509 MachineBasicBlock &MBB = *Inst.getParent(); 6510 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6511 6512 MachineOperand &Dest = Inst.getOperand(0); 6513 MachineOperand &Src0 = Inst.getOperand(1); 6514 MachineOperand &Src1 = Inst.getOperand(2); 6515 DebugLoc DL = Inst.getDebugLoc(); 6516 6517 MachineBasicBlock::iterator MII = Inst; 6518 6519 const MCInstrDesc &InstDesc = get(Opcode); 6520 const TargetRegisterClass *Src0RC = Src0.isReg() ? 6521 MRI.getRegClass(Src0.getReg()) : 6522 &AMDGPU::SGPR_32RegClass; 6523 6524 const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0); 6525 const TargetRegisterClass *Src1RC = Src1.isReg() ? 6526 MRI.getRegClass(Src1.getReg()) : 6527 &AMDGPU::SGPR_32RegClass; 6528 6529 const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0); 6530 6531 MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6532 AMDGPU::sub0, Src0SubRC); 6533 MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6534 AMDGPU::sub0, Src1SubRC); 6535 MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC, 6536 AMDGPU::sub1, Src0SubRC); 6537 MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC, 6538 AMDGPU::sub1, Src1SubRC); 6539 6540 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6541 const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC); 6542 const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0); 6543 6544 Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC); 6545 MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0) 6546 .add(SrcReg0Sub0) 6547 .add(SrcReg1Sub0); 6548 6549 Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC); 6550 MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1) 6551 .add(SrcReg0Sub1) 6552 .add(SrcReg1Sub1); 6553 6554 Register FullDestReg = MRI.createVirtualRegister(NewDestRC); 6555 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg) 6556 .addReg(DestSub0) 6557 .addImm(AMDGPU::sub0) 6558 .addReg(DestSub1) 6559 .addImm(AMDGPU::sub1); 6560 6561 MRI.replaceRegWith(Dest.getReg(), FullDestReg); 6562 6563 Worklist.insert(&LoHalf); 6564 Worklist.insert(&HiHalf); 6565 6566 // Move all users of this moved vlaue. 6567 addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist); 6568 } 6569 6570 void SIInstrInfo::splitScalar64BitXnor(SetVectorType &Worklist, 6571 MachineInstr &Inst, 6572 MachineDominatorTree *MDT) const { 6573 MachineBasicBlock &MBB = *Inst.getParent(); 6574 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6575 6576 MachineOperand &Dest = Inst.getOperand(0); 6577 MachineOperand &Src0 = Inst.getOperand(1); 6578 MachineOperand &Src1 = Inst.getOperand(2); 6579 const DebugLoc &DL = Inst.getDebugLoc(); 6580 6581 MachineBasicBlock::iterator MII = Inst; 6582 6583 const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg()); 6584 6585 Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 6586 6587 MachineOperand* Op0; 6588 MachineOperand* Op1; 6589 6590 if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg())) { 6591 Op0 = &Src0; 6592 Op1 = &Src1; 6593 } else { 6594 Op0 = &Src1; 6595 Op1 = &Src0; 6596 } 6597 6598 BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B64), Interm) 6599 .add(*Op0); 6600 6601 Register NewDest = MRI.createVirtualRegister(DestRC); 6602 6603 MachineInstr &Xor = *BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B64), NewDest) 6604 .addReg(Interm) 6605 .add(*Op1); 6606 6607 MRI.replaceRegWith(Dest.getReg(), NewDest); 6608 6609 Worklist.insert(&Xor); 6610 } 6611 6612 void SIInstrInfo::splitScalar64BitBCNT( 6613 SetVectorType &Worklist, MachineInstr &Inst) const { 6614 MachineBasicBlock &MBB = *Inst.getParent(); 6615 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6616 6617 MachineBasicBlock::iterator MII = Inst; 6618 const DebugLoc &DL = Inst.getDebugLoc(); 6619 6620 MachineOperand &Dest = Inst.getOperand(0); 6621 MachineOperand &Src = Inst.getOperand(1); 6622 6623 const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64); 6624 const TargetRegisterClass *SrcRC = Src.isReg() ? 6625 MRI.getRegClass(Src.getReg()) : 6626 &AMDGPU::SGPR_32RegClass; 6627 6628 Register MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6629 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6630 6631 const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0); 6632 6633 MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6634 AMDGPU::sub0, SrcSubRC); 6635 MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC, 6636 AMDGPU::sub1, SrcSubRC); 6637 6638 BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0); 6639 6640 BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg); 6641 6642 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6643 6644 // We don't need to legalize operands here. src0 for etiher instruction can be 6645 // an SGPR, and the second input is unused or determined here. 6646 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6647 } 6648 6649 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist, 6650 MachineInstr &Inst) const { 6651 MachineBasicBlock &MBB = *Inst.getParent(); 6652 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 6653 MachineBasicBlock::iterator MII = Inst; 6654 const DebugLoc &DL = Inst.getDebugLoc(); 6655 6656 MachineOperand &Dest = Inst.getOperand(0); 6657 uint32_t Imm = Inst.getOperand(2).getImm(); 6658 uint32_t Offset = Imm & 0x3f; // Extract bits [5:0]. 6659 uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16]. 6660 6661 (void) Offset; 6662 6663 // Only sext_inreg cases handled. 6664 assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 && 6665 Offset == 0 && "Not implemented"); 6666 6667 if (BitWidth < 32) { 6668 Register MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6669 Register MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6670 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6671 6672 BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32_e64), MidRegLo) 6673 .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0) 6674 .addImm(0) 6675 .addImm(BitWidth); 6676 6677 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi) 6678 .addImm(31) 6679 .addReg(MidRegLo); 6680 6681 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6682 .addReg(MidRegLo) 6683 .addImm(AMDGPU::sub0) 6684 .addReg(MidRegHi) 6685 .addImm(AMDGPU::sub1); 6686 6687 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6688 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6689 return; 6690 } 6691 6692 MachineOperand &Src = Inst.getOperand(1); 6693 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6694 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass); 6695 6696 BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg) 6697 .addImm(31) 6698 .addReg(Src.getReg(), 0, AMDGPU::sub0); 6699 6700 BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg) 6701 .addReg(Src.getReg(), 0, AMDGPU::sub0) 6702 .addImm(AMDGPU::sub0) 6703 .addReg(TmpReg) 6704 .addImm(AMDGPU::sub1); 6705 6706 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6707 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6708 } 6709 6710 void SIInstrInfo::addUsersToMoveToVALUWorklist( 6711 Register DstReg, 6712 MachineRegisterInfo &MRI, 6713 SetVectorType &Worklist) const { 6714 for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg), 6715 E = MRI.use_end(); I != E;) { 6716 MachineInstr &UseMI = *I->getParent(); 6717 6718 unsigned OpNo = 0; 6719 6720 switch (UseMI.getOpcode()) { 6721 case AMDGPU::COPY: 6722 case AMDGPU::WQM: 6723 case AMDGPU::SOFT_WQM: 6724 case AMDGPU::STRICT_WWM: 6725 case AMDGPU::STRICT_WQM: 6726 case AMDGPU::REG_SEQUENCE: 6727 case AMDGPU::PHI: 6728 case AMDGPU::INSERT_SUBREG: 6729 break; 6730 default: 6731 OpNo = I.getOperandNo(); 6732 break; 6733 } 6734 6735 if (!RI.hasVectorRegisters(getOpRegClass(UseMI, OpNo))) { 6736 Worklist.insert(&UseMI); 6737 6738 do { 6739 ++I; 6740 } while (I != E && I->getParent() == &UseMI); 6741 } else { 6742 ++I; 6743 } 6744 } 6745 } 6746 6747 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist, 6748 MachineRegisterInfo &MRI, 6749 MachineInstr &Inst) const { 6750 Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6751 MachineBasicBlock *MBB = Inst.getParent(); 6752 MachineOperand &Src0 = Inst.getOperand(1); 6753 MachineOperand &Src1 = Inst.getOperand(2); 6754 const DebugLoc &DL = Inst.getDebugLoc(); 6755 6756 switch (Inst.getOpcode()) { 6757 case AMDGPU::S_PACK_LL_B32_B16: { 6758 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6759 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6760 6761 // FIXME: Can do a lot better if we know the high bits of src0 or src1 are 6762 // 0. 6763 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6764 .addImm(0xffff); 6765 6766 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg) 6767 .addReg(ImmReg, RegState::Kill) 6768 .add(Src0); 6769 6770 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32_e64), ResultReg) 6771 .add(Src1) 6772 .addImm(16) 6773 .addReg(TmpReg, RegState::Kill); 6774 break; 6775 } 6776 case AMDGPU::S_PACK_LH_B32_B16: { 6777 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6778 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6779 .addImm(0xffff); 6780 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32_e64), ResultReg) 6781 .addReg(ImmReg, RegState::Kill) 6782 .add(Src0) 6783 .add(Src1); 6784 break; 6785 } 6786 case AMDGPU::S_PACK_HH_B32_B16: { 6787 Register ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6788 Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); 6789 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg) 6790 .addImm(16) 6791 .add(Src0); 6792 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg) 6793 .addImm(0xffff0000); 6794 BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32_e64), ResultReg) 6795 .add(Src1) 6796 .addReg(ImmReg, RegState::Kill) 6797 .addReg(TmpReg, RegState::Kill); 6798 break; 6799 } 6800 default: 6801 llvm_unreachable("unhandled s_pack_* instruction"); 6802 } 6803 6804 MachineOperand &Dest = Inst.getOperand(0); 6805 MRI.replaceRegWith(Dest.getReg(), ResultReg); 6806 addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist); 6807 } 6808 6809 void SIInstrInfo::addSCCDefUsersToVALUWorklist(MachineOperand &Op, 6810 MachineInstr &SCCDefInst, 6811 SetVectorType &Worklist) const { 6812 bool SCCUsedImplicitly = false; 6813 6814 // Ensure that def inst defines SCC, which is still live. 6815 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isDef() && 6816 !Op.isDead() && Op.getParent() == &SCCDefInst); 6817 SmallVector<MachineInstr *, 4> CopyToDelete; 6818 // This assumes that all the users of SCC are in the same block 6819 // as the SCC def. 6820 for (MachineInstr &MI : // Skip the def inst itself. 6821 make_range(std::next(MachineBasicBlock::iterator(SCCDefInst)), 6822 SCCDefInst.getParent()->end())) { 6823 // Check if SCC is used first. 6824 if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1) { 6825 if (MI.isCopy()) { 6826 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6827 Register DestReg = MI.getOperand(0).getReg(); 6828 6829 for (auto &User : MRI.use_nodbg_instructions(DestReg)) { 6830 if ((User.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO) || 6831 (User.getOpcode() == AMDGPU::S_SUB_CO_PSEUDO)) { 6832 User.getOperand(4).setReg(RI.getVCC()); 6833 Worklist.insert(&User); 6834 } else if (User.getOpcode() == AMDGPU::V_CNDMASK_B32_e64) { 6835 User.getOperand(5).setReg(RI.getVCC()); 6836 // No need to add to Worklist. 6837 } 6838 } 6839 CopyToDelete.push_back(&MI); 6840 } else { 6841 if (MI.getOpcode() == AMDGPU::S_CSELECT_B32 || 6842 MI.getOpcode() == AMDGPU::S_CSELECT_B64) { 6843 // This is an implicit use of SCC and it is really expected by 6844 // the SCC users to handle. 6845 // We cannot preserve the edge to the user so add the explicit 6846 // copy: SCC = COPY VCC. 6847 // The copy will be cleaned up during the processing of the user 6848 // in lowerSelect. 6849 SCCUsedImplicitly = true; 6850 } 6851 6852 Worklist.insert(&MI); 6853 } 6854 } 6855 // Exit if we find another SCC def. 6856 if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1) 6857 break; 6858 } 6859 for (auto &Copy : CopyToDelete) 6860 Copy->eraseFromParent(); 6861 6862 if (SCCUsedImplicitly) { 6863 BuildMI(*SCCDefInst.getParent(), std::next(SCCDefInst.getIterator()), 6864 SCCDefInst.getDebugLoc(), get(AMDGPU::COPY), AMDGPU::SCC) 6865 .addReg(RI.getVCC()); 6866 } 6867 } 6868 6869 // Instructions that use SCC may be converted to VALU instructions. When that 6870 // happens, the SCC register is changed to VCC_LO. The instruction that defines 6871 // SCC must be changed to an instruction that defines VCC. This function makes 6872 // sure that the instruction that defines SCC is added to the moveToVALU 6873 // worklist. 6874 void SIInstrInfo::addSCCDefsToVALUWorklist(MachineOperand &Op, 6875 SetVectorType &Worklist) const { 6876 assert(Op.isReg() && Op.getReg() == AMDGPU::SCC && Op.isUse()); 6877 6878 MachineInstr *SCCUseInst = Op.getParent(); 6879 // Look for a preceeding instruction that either defines VCC or SCC. If VCC 6880 // then there is nothing to do because the defining instruction has been 6881 // converted to a VALU already. If SCC then that instruction needs to be 6882 // converted to a VALU. 6883 for (MachineInstr &MI : 6884 make_range(std::next(MachineBasicBlock::reverse_iterator(SCCUseInst)), 6885 SCCUseInst->getParent()->rend())) { 6886 if (MI.modifiesRegister(AMDGPU::VCC, &RI)) 6887 break; 6888 if (MI.definesRegister(AMDGPU::SCC, &RI)) { 6889 Worklist.insert(&MI); 6890 break; 6891 } 6892 } 6893 } 6894 6895 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass( 6896 const MachineInstr &Inst) const { 6897 const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0); 6898 6899 switch (Inst.getOpcode()) { 6900 // For target instructions, getOpRegClass just returns the virtual register 6901 // class associated with the operand, so we need to find an equivalent VGPR 6902 // register class in order to move the instruction to the VALU. 6903 case AMDGPU::COPY: 6904 case AMDGPU::PHI: 6905 case AMDGPU::REG_SEQUENCE: 6906 case AMDGPU::INSERT_SUBREG: 6907 case AMDGPU::WQM: 6908 case AMDGPU::SOFT_WQM: 6909 case AMDGPU::STRICT_WWM: 6910 case AMDGPU::STRICT_WQM: { 6911 const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1); 6912 if (RI.hasAGPRs(SrcRC)) { 6913 if (RI.hasAGPRs(NewDstRC)) 6914 return nullptr; 6915 6916 switch (Inst.getOpcode()) { 6917 case AMDGPU::PHI: 6918 case AMDGPU::REG_SEQUENCE: 6919 case AMDGPU::INSERT_SUBREG: 6920 NewDstRC = RI.getEquivalentAGPRClass(NewDstRC); 6921 break; 6922 default: 6923 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6924 } 6925 6926 if (!NewDstRC) 6927 return nullptr; 6928 } else { 6929 if (RI.hasVGPRs(NewDstRC) || NewDstRC == &AMDGPU::VReg_1RegClass) 6930 return nullptr; 6931 6932 NewDstRC = RI.getEquivalentVGPRClass(NewDstRC); 6933 if (!NewDstRC) 6934 return nullptr; 6935 } 6936 6937 return NewDstRC; 6938 } 6939 default: 6940 return NewDstRC; 6941 } 6942 } 6943 6944 // Find the one SGPR operand we are allowed to use. 6945 Register SIInstrInfo::findUsedSGPR(const MachineInstr &MI, 6946 int OpIndices[3]) const { 6947 const MCInstrDesc &Desc = MI.getDesc(); 6948 6949 // Find the one SGPR operand we are allowed to use. 6950 // 6951 // First we need to consider the instruction's operand requirements before 6952 // legalizing. Some operands are required to be SGPRs, such as implicit uses 6953 // of VCC, but we are still bound by the constant bus requirement to only use 6954 // one. 6955 // 6956 // If the operand's class is an SGPR, we can never move it. 6957 6958 Register SGPRReg = findImplicitSGPRRead(MI); 6959 if (SGPRReg != AMDGPU::NoRegister) 6960 return SGPRReg; 6961 6962 Register UsedSGPRs[3] = { AMDGPU::NoRegister }; 6963 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); 6964 6965 for (unsigned i = 0; i < 3; ++i) { 6966 int Idx = OpIndices[i]; 6967 if (Idx == -1) 6968 break; 6969 6970 const MachineOperand &MO = MI.getOperand(Idx); 6971 if (!MO.isReg()) 6972 continue; 6973 6974 // Is this operand statically required to be an SGPR based on the operand 6975 // constraints? 6976 const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass); 6977 bool IsRequiredSGPR = RI.isSGPRClass(OpRC); 6978 if (IsRequiredSGPR) 6979 return MO.getReg(); 6980 6981 // If this could be a VGPR or an SGPR, Check the dynamic register class. 6982 Register Reg = MO.getReg(); 6983 const TargetRegisterClass *RegRC = MRI.getRegClass(Reg); 6984 if (RI.isSGPRClass(RegRC)) 6985 UsedSGPRs[i] = Reg; 6986 } 6987 6988 // We don't have a required SGPR operand, so we have a bit more freedom in 6989 // selecting operands to move. 6990 6991 // Try to select the most used SGPR. If an SGPR is equal to one of the 6992 // others, we choose that. 6993 // 6994 // e.g. 6995 // V_FMA_F32 v0, s0, s0, s0 -> No moves 6996 // V_FMA_F32 v0, s0, s1, s0 -> Move s1 6997 6998 // TODO: If some of the operands are 64-bit SGPRs and some 32, we should 6999 // prefer those. 7000 7001 if (UsedSGPRs[0] != AMDGPU::NoRegister) { 7002 if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2]) 7003 SGPRReg = UsedSGPRs[0]; 7004 } 7005 7006 if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) { 7007 if (UsedSGPRs[1] == UsedSGPRs[2]) 7008 SGPRReg = UsedSGPRs[1]; 7009 } 7010 7011 return SGPRReg; 7012 } 7013 7014 MachineOperand *SIInstrInfo::getNamedOperand(MachineInstr &MI, 7015 unsigned OperandName) const { 7016 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName); 7017 if (Idx == -1) 7018 return nullptr; 7019 7020 return &MI.getOperand(Idx); 7021 } 7022 7023 uint64_t SIInstrInfo::getDefaultRsrcDataFormat() const { 7024 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) { 7025 return (AMDGPU::MTBUFFormat::UFMT_32_FLOAT << 44) | 7026 (1ULL << 56) | // RESOURCE_LEVEL = 1 7027 (3ULL << 60); // OOB_SELECT = 3 7028 } 7029 7030 uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT; 7031 if (ST.isAmdHsaOS()) { 7032 // Set ATC = 1. GFX9 doesn't have this bit. 7033 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) 7034 RsrcDataFormat |= (1ULL << 56); 7035 7036 // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this. 7037 // BTW, it disables TC L2 and therefore decreases performance. 7038 if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS) 7039 RsrcDataFormat |= (2ULL << 59); 7040 } 7041 7042 return RsrcDataFormat; 7043 } 7044 7045 uint64_t SIInstrInfo::getScratchRsrcWords23() const { 7046 uint64_t Rsrc23 = getDefaultRsrcDataFormat() | 7047 AMDGPU::RSRC_TID_ENABLE | 7048 0xffffffff; // Size; 7049 7050 // GFX9 doesn't have ELEMENT_SIZE. 7051 if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) { 7052 uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize(true)) - 1; 7053 Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT; 7054 } 7055 7056 // IndexStride = 64 / 32. 7057 uint64_t IndexStride = ST.getWavefrontSize() == 64 ? 3 : 2; 7058 Rsrc23 |= IndexStride << AMDGPU::RSRC_INDEX_STRIDE_SHIFT; 7059 7060 // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17]. 7061 // Clear them unless we want a huge stride. 7062 if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && 7063 ST.getGeneration() <= AMDGPUSubtarget::GFX9) 7064 Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT; 7065 7066 return Rsrc23; 7067 } 7068 7069 bool SIInstrInfo::isLowLatencyInstruction(const MachineInstr &MI) const { 7070 unsigned Opc = MI.getOpcode(); 7071 7072 return isSMRD(Opc); 7073 } 7074 7075 bool SIInstrInfo::isHighLatencyDef(int Opc) const { 7076 return get(Opc).mayLoad() && 7077 (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc)); 7078 } 7079 7080 unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI, 7081 int &FrameIndex) const { 7082 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr); 7083 if (!Addr || !Addr->isFI()) 7084 return AMDGPU::NoRegister; 7085 7086 assert(!MI.memoperands_empty() && 7087 (*MI.memoperands_begin())->getAddrSpace() == AMDGPUAS::PRIVATE_ADDRESS); 7088 7089 FrameIndex = Addr->getIndex(); 7090 return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg(); 7091 } 7092 7093 unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI, 7094 int &FrameIndex) const { 7095 const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr); 7096 assert(Addr && Addr->isFI()); 7097 FrameIndex = Addr->getIndex(); 7098 return getNamedOperand(MI, AMDGPU::OpName::data)->getReg(); 7099 } 7100 7101 unsigned SIInstrInfo::isLoadFromStackSlot(const MachineInstr &MI, 7102 int &FrameIndex) const { 7103 if (!MI.mayLoad()) 7104 return AMDGPU::NoRegister; 7105 7106 if (isMUBUF(MI) || isVGPRSpill(MI)) 7107 return isStackAccess(MI, FrameIndex); 7108 7109 if (isSGPRSpill(MI)) 7110 return isSGPRStackAccess(MI, FrameIndex); 7111 7112 return AMDGPU::NoRegister; 7113 } 7114 7115 unsigned SIInstrInfo::isStoreToStackSlot(const MachineInstr &MI, 7116 int &FrameIndex) const { 7117 if (!MI.mayStore()) 7118 return AMDGPU::NoRegister; 7119 7120 if (isMUBUF(MI) || isVGPRSpill(MI)) 7121 return isStackAccess(MI, FrameIndex); 7122 7123 if (isSGPRSpill(MI)) 7124 return isSGPRStackAccess(MI, FrameIndex); 7125 7126 return AMDGPU::NoRegister; 7127 } 7128 7129 unsigned SIInstrInfo::getInstBundleSize(const MachineInstr &MI) const { 7130 unsigned Size = 0; 7131 MachineBasicBlock::const_instr_iterator I = MI.getIterator(); 7132 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end(); 7133 while (++I != E && I->isInsideBundle()) { 7134 assert(!I->isBundle() && "No nested bundle!"); 7135 Size += getInstSizeInBytes(*I); 7136 } 7137 7138 return Size; 7139 } 7140 7141 unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { 7142 unsigned Opc = MI.getOpcode(); 7143 const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc); 7144 unsigned DescSize = Desc.getSize(); 7145 7146 // If we have a definitive size, we can use it. Otherwise we need to inspect 7147 // the operands to know the size. 7148 if (isFixedSize(MI)) { 7149 unsigned Size = DescSize; 7150 7151 // If we hit the buggy offset, an extra nop will be inserted in MC so 7152 // estimate the worst case. 7153 if (MI.isBranch() && ST.hasOffset3fBug()) 7154 Size += 4; 7155 7156 return Size; 7157 } 7158 7159 // 4-byte instructions may have a 32-bit literal encoded after them. Check 7160 // operands that coud ever be literals. 7161 if (isVALU(MI) || isSALU(MI)) { 7162 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); 7163 if (Src0Idx == -1) 7164 return DescSize; // No operands. 7165 7166 if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx])) 7167 return isVOP3(MI) ? 12 : (DescSize + 4); 7168 7169 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1); 7170 if (Src1Idx == -1) 7171 return DescSize; 7172 7173 if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx])) 7174 return isVOP3(MI) ? 12 : (DescSize + 4); 7175 7176 int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2); 7177 if (Src2Idx == -1) 7178 return DescSize; 7179 7180 if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx])) 7181 return isVOP3(MI) ? 12 : (DescSize + 4); 7182 7183 return DescSize; 7184 } 7185 7186 // Check whether we have extra NSA words. 7187 if (isMIMG(MI)) { 7188 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0); 7189 if (VAddr0Idx < 0) 7190 return 8; 7191 7192 int RSrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc); 7193 return 8 + 4 * ((RSrcIdx - VAddr0Idx + 2) / 4); 7194 } 7195 7196 switch (Opc) { 7197 case TargetOpcode::IMPLICIT_DEF: 7198 case TargetOpcode::KILL: 7199 case TargetOpcode::DBG_VALUE: 7200 case TargetOpcode::EH_LABEL: 7201 return 0; 7202 case TargetOpcode::BUNDLE: 7203 return getInstBundleSize(MI); 7204 case TargetOpcode::INLINEASM: 7205 case TargetOpcode::INLINEASM_BR: { 7206 const MachineFunction *MF = MI.getParent()->getParent(); 7207 const char *AsmStr = MI.getOperand(0).getSymbolName(); 7208 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo(), &ST); 7209 } 7210 default: 7211 return DescSize; 7212 } 7213 } 7214 7215 bool SIInstrInfo::mayAccessFlatAddressSpace(const MachineInstr &MI) const { 7216 if (!isFLAT(MI)) 7217 return false; 7218 7219 if (MI.memoperands_empty()) 7220 return true; 7221 7222 for (const MachineMemOperand *MMO : MI.memoperands()) { 7223 if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS) 7224 return true; 7225 } 7226 return false; 7227 } 7228 7229 bool SIInstrInfo::isNonUniformBranchInstr(MachineInstr &Branch) const { 7230 return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO; 7231 } 7232 7233 void SIInstrInfo::convertNonUniformIfRegion(MachineBasicBlock *IfEntry, 7234 MachineBasicBlock *IfEnd) const { 7235 MachineBasicBlock::iterator TI = IfEntry->getFirstTerminator(); 7236 assert(TI != IfEntry->end()); 7237 7238 MachineInstr *Branch = &(*TI); 7239 MachineFunction *MF = IfEntry->getParent(); 7240 MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo(); 7241 7242 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7243 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7244 MachineInstr *SIIF = 7245 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg) 7246 .add(Branch->getOperand(0)) 7247 .add(Branch->getOperand(1)); 7248 MachineInstr *SIEND = 7249 BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF)) 7250 .addReg(DstReg); 7251 7252 IfEntry->erase(TI); 7253 IfEntry->insert(IfEntry->end(), SIIF); 7254 IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND); 7255 } 7256 } 7257 7258 void SIInstrInfo::convertNonUniformLoopRegion( 7259 MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const { 7260 MachineBasicBlock::iterator TI = LoopEnd->getFirstTerminator(); 7261 // We expect 2 terminators, one conditional and one unconditional. 7262 assert(TI != LoopEnd->end()); 7263 7264 MachineInstr *Branch = &(*TI); 7265 MachineFunction *MF = LoopEnd->getParent(); 7266 MachineRegisterInfo &MRI = LoopEnd->getParent()->getRegInfo(); 7267 7268 if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) { 7269 7270 Register DstReg = MRI.createVirtualRegister(RI.getBoolRC()); 7271 Register BackEdgeReg = MRI.createVirtualRegister(RI.getBoolRC()); 7272 MachineInstrBuilder HeaderPHIBuilder = 7273 BuildMI(*(MF), Branch->getDebugLoc(), get(TargetOpcode::PHI), DstReg); 7274 for (MachineBasicBlock::pred_iterator PI = LoopEntry->pred_begin(), 7275 E = LoopEntry->pred_end(); 7276 PI != E; ++PI) { 7277 if (*PI == LoopEnd) { 7278 HeaderPHIBuilder.addReg(BackEdgeReg); 7279 } else { 7280 MachineBasicBlock *PMBB = *PI; 7281 Register ZeroReg = MRI.createVirtualRegister(RI.getBoolRC()); 7282 materializeImmediate(*PMBB, PMBB->getFirstTerminator(), DebugLoc(), 7283 ZeroReg, 0); 7284 HeaderPHIBuilder.addReg(ZeroReg); 7285 } 7286 HeaderPHIBuilder.addMBB(*PI); 7287 } 7288 MachineInstr *HeaderPhi = HeaderPHIBuilder; 7289 MachineInstr *SIIFBREAK = BuildMI(*(MF), Branch->getDebugLoc(), 7290 get(AMDGPU::SI_IF_BREAK), BackEdgeReg) 7291 .addReg(DstReg) 7292 .add(Branch->getOperand(0)); 7293 MachineInstr *SILOOP = 7294 BuildMI(*(MF), Branch->getDebugLoc(), get(AMDGPU::SI_LOOP)) 7295 .addReg(BackEdgeReg) 7296 .addMBB(LoopEntry); 7297 7298 LoopEntry->insert(LoopEntry->begin(), HeaderPhi); 7299 LoopEnd->erase(TI); 7300 LoopEnd->insert(LoopEnd->end(), SIIFBREAK); 7301 LoopEnd->insert(LoopEnd->end(), SILOOP); 7302 } 7303 } 7304 7305 ArrayRef<std::pair<int, const char *>> 7306 SIInstrInfo::getSerializableTargetIndices() const { 7307 static const std::pair<int, const char *> TargetIndices[] = { 7308 {AMDGPU::TI_CONSTDATA_START, "amdgpu-constdata-start"}, 7309 {AMDGPU::TI_SCRATCH_RSRC_DWORD0, "amdgpu-scratch-rsrc-dword0"}, 7310 {AMDGPU::TI_SCRATCH_RSRC_DWORD1, "amdgpu-scratch-rsrc-dword1"}, 7311 {AMDGPU::TI_SCRATCH_RSRC_DWORD2, "amdgpu-scratch-rsrc-dword2"}, 7312 {AMDGPU::TI_SCRATCH_RSRC_DWORD3, "amdgpu-scratch-rsrc-dword3"}}; 7313 return makeArrayRef(TargetIndices); 7314 } 7315 7316 /// This is used by the post-RA scheduler (SchedulePostRAList.cpp). The 7317 /// post-RA version of misched uses CreateTargetMIHazardRecognizer. 7318 ScheduleHazardRecognizer * 7319 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, 7320 const ScheduleDAG *DAG) const { 7321 return new GCNHazardRecognizer(DAG->MF); 7322 } 7323 7324 /// This is the hazard recognizer used at -O0 by the PostRAHazardRecognizer 7325 /// pass. 7326 ScheduleHazardRecognizer * 7327 SIInstrInfo::CreateTargetPostRAHazardRecognizer(const MachineFunction &MF) const { 7328 return new GCNHazardRecognizer(MF); 7329 } 7330 7331 std::pair<unsigned, unsigned> 7332 SIInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { 7333 return std::make_pair(TF & MO_MASK, TF & ~MO_MASK); 7334 } 7335 7336 ArrayRef<std::pair<unsigned, const char *>> 7337 SIInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { 7338 static const std::pair<unsigned, const char *> TargetFlags[] = { 7339 { MO_GOTPCREL, "amdgpu-gotprel" }, 7340 { MO_GOTPCREL32_LO, "amdgpu-gotprel32-lo" }, 7341 { MO_GOTPCREL32_HI, "amdgpu-gotprel32-hi" }, 7342 { MO_REL32_LO, "amdgpu-rel32-lo" }, 7343 { MO_REL32_HI, "amdgpu-rel32-hi" }, 7344 { MO_ABS32_LO, "amdgpu-abs32-lo" }, 7345 { MO_ABS32_HI, "amdgpu-abs32-hi" }, 7346 }; 7347 7348 return makeArrayRef(TargetFlags); 7349 } 7350 7351 bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI) const { 7352 return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY && 7353 MI.modifiesRegister(AMDGPU::EXEC, &RI); 7354 } 7355 7356 MachineInstrBuilder 7357 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7358 MachineBasicBlock::iterator I, 7359 const DebugLoc &DL, 7360 Register DestReg) const { 7361 if (ST.hasAddNoCarry()) 7362 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg); 7363 7364 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); 7365 Register UnusedCarry = MRI.createVirtualRegister(RI.getBoolRC()); 7366 MRI.setRegAllocationHint(UnusedCarry, 0, RI.getVCC()); 7367 7368 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7369 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7370 } 7371 7372 MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, 7373 MachineBasicBlock::iterator I, 7374 const DebugLoc &DL, 7375 Register DestReg, 7376 RegScavenger &RS) const { 7377 if (ST.hasAddNoCarry()) 7378 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); 7379 7380 // If available, prefer to use vcc. 7381 Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) 7382 ? Register(RI.getVCC()) 7383 : RS.scavengeRegister(RI.getBoolRC(), I, 0, false); 7384 7385 // TODO: Users need to deal with this. 7386 if (!UnusedCarry.isValid()) 7387 return MachineInstrBuilder(); 7388 7389 return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_CO_U32_e64), DestReg) 7390 .addReg(UnusedCarry, RegState::Define | RegState::Dead); 7391 } 7392 7393 bool SIInstrInfo::isKillTerminator(unsigned Opcode) { 7394 switch (Opcode) { 7395 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR: 7396 case AMDGPU::SI_KILL_I1_TERMINATOR: 7397 return true; 7398 default: 7399 return false; 7400 } 7401 } 7402 7403 const MCInstrDesc &SIInstrInfo::getKillTerminatorFromPseudo(unsigned Opcode) const { 7404 switch (Opcode) { 7405 case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO: 7406 return get(AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR); 7407 case AMDGPU::SI_KILL_I1_PSEUDO: 7408 return get(AMDGPU::SI_KILL_I1_TERMINATOR); 7409 default: 7410 llvm_unreachable("invalid opcode, expected SI_KILL_*_PSEUDO"); 7411 } 7412 } 7413 7414 void SIInstrInfo::fixImplicitOperands(MachineInstr &MI) const { 7415 if (!ST.isWave32()) 7416 return; 7417 7418 for (auto &Op : MI.implicit_operands()) { 7419 if (Op.isReg() && Op.getReg() == AMDGPU::VCC) 7420 Op.setReg(AMDGPU::VCC_LO); 7421 } 7422 } 7423 7424 bool SIInstrInfo::isBufferSMRD(const MachineInstr &MI) const { 7425 if (!isSMRD(MI)) 7426 return false; 7427 7428 // Check that it is using a buffer resource. 7429 int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::sbase); 7430 if (Idx == -1) // e.g. s_memtime 7431 return false; 7432 7433 const auto RCID = MI.getDesc().OpInfo[Idx].RegClass; 7434 return RI.getRegClass(RCID)->hasSubClassEq(&AMDGPU::SGPR_128RegClass); 7435 } 7436 7437 // Depending on the used address space and instructions, some immediate offsets 7438 // are allowed and some are not. 7439 // In general, flat instruction offsets can only be non-negative, global and 7440 // scratch instruction offsets can also be negative. 7441 // 7442 // There are several bugs related to these offsets: 7443 // On gfx10.1, flat instructions that go into the global address space cannot 7444 // use an offset. 7445 // 7446 // For scratch instructions, the address can be either an SGPR or a VGPR. 7447 // The following offsets can be used, depending on the architecture (x means 7448 // cannot be used): 7449 // +----------------------------+------+------+ 7450 // | Address-Mode | SGPR | VGPR | 7451 // +----------------------------+------+------+ 7452 // | gfx9 | | | 7453 // | negative, 4-aligned offset | x | ok | 7454 // | negative, unaligned offset | x | ok | 7455 // +----------------------------+------+------+ 7456 // | gfx10 | | | 7457 // | negative, 4-aligned offset | ok | ok | 7458 // | negative, unaligned offset | ok | x | 7459 // +----------------------------+------+------+ 7460 // | gfx10.3 | | | 7461 // | negative, 4-aligned offset | ok | ok | 7462 // | negative, unaligned offset | ok | ok | 7463 // +----------------------------+------+------+ 7464 // 7465 // This function ignores the addressing mode, so if an offset cannot be used in 7466 // one addressing mode, it is considered illegal. 7467 bool SIInstrInfo::isLegalFLATOffset(int64_t Offset, unsigned AddrSpace, 7468 uint64_t FlatVariant) const { 7469 // TODO: Should 0 be special cased? 7470 if (!ST.hasFlatInstOffsets()) 7471 return false; 7472 7473 if (ST.hasFlatSegmentOffsetBug() && FlatVariant == SIInstrFlags::FLAT && 7474 (AddrSpace == AMDGPUAS::FLAT_ADDRESS || 7475 AddrSpace == AMDGPUAS::GLOBAL_ADDRESS)) 7476 return false; 7477 7478 bool Signed = FlatVariant != SIInstrFlags::FLAT; 7479 if (ST.hasNegativeScratchOffsetBug() && 7480 FlatVariant == SIInstrFlags::FlatScratch) 7481 Signed = false; 7482 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7483 FlatVariant == SIInstrFlags::FlatScratch && Offset < 0 && 7484 (Offset % 4) != 0) { 7485 return false; 7486 } 7487 7488 unsigned N = AMDGPU::getNumFlatOffsetBits(ST, Signed); 7489 return Signed ? isIntN(N, Offset) : isUIntN(N, Offset); 7490 } 7491 7492 // See comment on SIInstrInfo::isLegalFLATOffset for what is legal and what not. 7493 std::pair<int64_t, int64_t> 7494 SIInstrInfo::splitFlatOffset(int64_t COffsetVal, unsigned AddrSpace, 7495 uint64_t FlatVariant) const { 7496 int64_t RemainderOffset = COffsetVal; 7497 int64_t ImmField = 0; 7498 bool Signed = FlatVariant != SIInstrFlags::FLAT; 7499 if (ST.hasNegativeScratchOffsetBug() && 7500 FlatVariant == SIInstrFlags::FlatScratch) 7501 Signed = false; 7502 7503 const unsigned NumBits = AMDGPU::getNumFlatOffsetBits(ST, Signed); 7504 if (Signed) { 7505 // Use signed division by a power of two to truncate towards 0. 7506 int64_t D = 1LL << (NumBits - 1); 7507 RemainderOffset = (COffsetVal / D) * D; 7508 ImmField = COffsetVal - RemainderOffset; 7509 7510 if (ST.hasNegativeUnalignedScratchOffsetBug() && 7511 FlatVariant == SIInstrFlags::FlatScratch && ImmField < 0 && 7512 (ImmField % 4) != 0) { 7513 // Make ImmField a multiple of 4 7514 RemainderOffset += ImmField % 4; 7515 ImmField -= ImmField % 4; 7516 } 7517 } else if (COffsetVal >= 0) { 7518 ImmField = COffsetVal & maskTrailingOnes<uint64_t>(NumBits); 7519 RemainderOffset = COffsetVal - ImmField; 7520 } 7521 7522 assert(isLegalFLATOffset(ImmField, AddrSpace, FlatVariant)); 7523 assert(RemainderOffset + ImmField == COffsetVal); 7524 return {ImmField, RemainderOffset}; 7525 } 7526 7527 // This must be kept in sync with the SIEncodingFamily class in SIInstrInfo.td 7528 enum SIEncodingFamily { 7529 SI = 0, 7530 VI = 1, 7531 SDWA = 2, 7532 SDWA9 = 3, 7533 GFX80 = 4, 7534 GFX9 = 5, 7535 GFX10 = 6, 7536 SDWA10 = 7, 7537 GFX90A = 8 7538 }; 7539 7540 static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) { 7541 switch (ST.getGeneration()) { 7542 default: 7543 break; 7544 case AMDGPUSubtarget::SOUTHERN_ISLANDS: 7545 case AMDGPUSubtarget::SEA_ISLANDS: 7546 return SIEncodingFamily::SI; 7547 case AMDGPUSubtarget::VOLCANIC_ISLANDS: 7548 case AMDGPUSubtarget::GFX9: 7549 return SIEncodingFamily::VI; 7550 case AMDGPUSubtarget::GFX10: 7551 return SIEncodingFamily::GFX10; 7552 } 7553 llvm_unreachable("Unknown subtarget generation!"); 7554 } 7555 7556 bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { 7557 switch(MCOp) { 7558 // These opcodes use indirect register addressing so 7559 // they need special handling by codegen (currently missing). 7560 // Therefore it is too risky to allow these opcodes 7561 // to be selected by dpp combiner or sdwa peepholer. 7562 case AMDGPU::V_MOVRELS_B32_dpp_gfx10: 7563 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: 7564 case AMDGPU::V_MOVRELD_B32_dpp_gfx10: 7565 case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: 7566 case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: 7567 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: 7568 case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: 7569 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: 7570 return true; 7571 default: 7572 return false; 7573 } 7574 } 7575 7576 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { 7577 SIEncodingFamily Gen = subtargetEncodingFamily(ST); 7578 7579 if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 && 7580 ST.getGeneration() == AMDGPUSubtarget::GFX9) 7581 Gen = SIEncodingFamily::GFX9; 7582 7583 // Adjust the encoding family to GFX80 for D16 buffer instructions when the 7584 // subtarget has UnpackedD16VMem feature. 7585 // TODO: remove this when we discard GFX80 encoding. 7586 if (ST.hasUnpackedD16VMem() && (get(Opcode).TSFlags & SIInstrFlags::D16Buf)) 7587 Gen = SIEncodingFamily::GFX80; 7588 7589 if (get(Opcode).TSFlags & SIInstrFlags::SDWA) { 7590 switch (ST.getGeneration()) { 7591 default: 7592 Gen = SIEncodingFamily::SDWA; 7593 break; 7594 case AMDGPUSubtarget::GFX9: 7595 Gen = SIEncodingFamily::SDWA9; 7596 break; 7597 case AMDGPUSubtarget::GFX10: 7598 Gen = SIEncodingFamily::SDWA10; 7599 break; 7600 } 7601 } 7602 7603 int MCOp = AMDGPU::getMCOpcode(Opcode, Gen); 7604 7605 // -1 means that Opcode is already a native instruction. 7606 if (MCOp == -1) 7607 return Opcode; 7608 7609 if (ST.hasGFX90AInsts()) { 7610 uint16_t NMCOp = (uint16_t)-1; 7611 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX90A); 7612 if (NMCOp == (uint16_t)-1) 7613 NMCOp = AMDGPU::getMCOpcode(Opcode, SIEncodingFamily::GFX9); 7614 if (NMCOp != (uint16_t)-1) 7615 MCOp = NMCOp; 7616 } 7617 7618 // (uint16_t)-1 means that Opcode is a pseudo instruction that has 7619 // no encoding in the given subtarget generation. 7620 if (MCOp == (uint16_t)-1) 7621 return -1; 7622 7623 if (isAsmOnlyOpcode(MCOp)) 7624 return -1; 7625 7626 return MCOp; 7627 } 7628 7629 static 7630 TargetInstrInfo::RegSubRegPair getRegOrUndef(const MachineOperand &RegOpnd) { 7631 assert(RegOpnd.isReg()); 7632 return RegOpnd.isUndef() ? TargetInstrInfo::RegSubRegPair() : 7633 getRegSubRegPair(RegOpnd); 7634 } 7635 7636 TargetInstrInfo::RegSubRegPair 7637 llvm::getRegSequenceSubReg(MachineInstr &MI, unsigned SubReg) { 7638 assert(MI.isRegSequence()); 7639 for (unsigned I = 0, E = (MI.getNumOperands() - 1)/ 2; I < E; ++I) 7640 if (MI.getOperand(1 + 2 * I + 1).getImm() == SubReg) { 7641 auto &RegOp = MI.getOperand(1 + 2 * I); 7642 return getRegOrUndef(RegOp); 7643 } 7644 return TargetInstrInfo::RegSubRegPair(); 7645 } 7646 7647 // Try to find the definition of reg:subreg in subreg-manipulation pseudos 7648 // Following a subreg of reg:subreg isn't supported 7649 static bool followSubRegDef(MachineInstr &MI, 7650 TargetInstrInfo::RegSubRegPair &RSR) { 7651 if (!RSR.SubReg) 7652 return false; 7653 switch (MI.getOpcode()) { 7654 default: break; 7655 case AMDGPU::REG_SEQUENCE: 7656 RSR = getRegSequenceSubReg(MI, RSR.SubReg); 7657 return true; 7658 // EXTRACT_SUBREG ins't supported as this would follow a subreg of subreg 7659 case AMDGPU::INSERT_SUBREG: 7660 if (RSR.SubReg == (unsigned)MI.getOperand(3).getImm()) 7661 // inserted the subreg we're looking for 7662 RSR = getRegOrUndef(MI.getOperand(2)); 7663 else { // the subreg in the rest of the reg 7664 auto R1 = getRegOrUndef(MI.getOperand(1)); 7665 if (R1.SubReg) // subreg of subreg isn't supported 7666 return false; 7667 RSR.Reg = R1.Reg; 7668 } 7669 return true; 7670 } 7671 return false; 7672 } 7673 7674 MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, 7675 MachineRegisterInfo &MRI) { 7676 assert(MRI.isSSA()); 7677 if (!P.Reg.isVirtual()) 7678 return nullptr; 7679 7680 auto RSR = P; 7681 auto *DefInst = MRI.getVRegDef(RSR.Reg); 7682 while (auto *MI = DefInst) { 7683 DefInst = nullptr; 7684 switch (MI->getOpcode()) { 7685 case AMDGPU::COPY: 7686 case AMDGPU::V_MOV_B32_e32: { 7687 auto &Op1 = MI->getOperand(1); 7688 if (Op1.isReg() && Op1.getReg().isVirtual()) { 7689 if (Op1.isUndef()) 7690 return nullptr; 7691 RSR = getRegSubRegPair(Op1); 7692 DefInst = MRI.getVRegDef(RSR.Reg); 7693 } 7694 break; 7695 } 7696 default: 7697 if (followSubRegDef(*MI, RSR)) { 7698 if (!RSR.Reg) 7699 return nullptr; 7700 DefInst = MRI.getVRegDef(RSR.Reg); 7701 } 7702 } 7703 if (!DefInst) 7704 return MI; 7705 } 7706 return nullptr; 7707 } 7708 7709 bool llvm::execMayBeModifiedBeforeUse(const MachineRegisterInfo &MRI, 7710 Register VReg, 7711 const MachineInstr &DefMI, 7712 const MachineInstr &UseMI) { 7713 assert(MRI.isSSA() && "Must be run on SSA"); 7714 7715 auto *TRI = MRI.getTargetRegisterInfo(); 7716 auto *DefBB = DefMI.getParent(); 7717 7718 // Don't bother searching between blocks, although it is possible this block 7719 // doesn't modify exec. 7720 if (UseMI.getParent() != DefBB) 7721 return true; 7722 7723 const int MaxInstScan = 20; 7724 int NumInst = 0; 7725 7726 // Stop scan at the use. 7727 auto E = UseMI.getIterator(); 7728 for (auto I = std::next(DefMI.getIterator()); I != E; ++I) { 7729 if (I->isDebugInstr()) 7730 continue; 7731 7732 if (++NumInst > MaxInstScan) 7733 return true; 7734 7735 if (I->modifiesRegister(AMDGPU::EXEC, TRI)) 7736 return true; 7737 } 7738 7739 return false; 7740 } 7741 7742 bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI, 7743 Register VReg, 7744 const MachineInstr &DefMI) { 7745 assert(MRI.isSSA() && "Must be run on SSA"); 7746 7747 auto *TRI = MRI.getTargetRegisterInfo(); 7748 auto *DefBB = DefMI.getParent(); 7749 7750 const int MaxUseScan = 10; 7751 int NumUse = 0; 7752 7753 for (auto &Use : MRI.use_nodbg_operands(VReg)) { 7754 auto &UseInst = *Use.getParent(); 7755 // Don't bother searching between blocks, although it is possible this block 7756 // doesn't modify exec. 7757 if (UseInst.getParent() != DefBB) 7758 return true; 7759 7760 if (++NumUse > MaxUseScan) 7761 return true; 7762 } 7763 7764 if (NumUse == 0) 7765 return false; 7766 7767 const int MaxInstScan = 20; 7768 int NumInst = 0; 7769 7770 // Stop scan when we have seen all the uses. 7771 for (auto I = std::next(DefMI.getIterator()); ; ++I) { 7772 assert(I != DefBB->end()); 7773 7774 if (I->isDebugInstr()) 7775 continue; 7776 7777 if (++NumInst > MaxInstScan) 7778 return true; 7779 7780 for (const MachineOperand &Op : I->operands()) { 7781 // We don't check reg masks here as they're used only on calls: 7782 // 1. EXEC is only considered const within one BB 7783 // 2. Call should be a terminator instruction if present in a BB 7784 7785 if (!Op.isReg()) 7786 continue; 7787 7788 Register Reg = Op.getReg(); 7789 if (Op.isUse()) { 7790 if (Reg == VReg && --NumUse == 0) 7791 return false; 7792 } else if (TRI->regsOverlap(Reg, AMDGPU::EXEC)) 7793 return true; 7794 } 7795 } 7796 } 7797 7798 MachineInstr *SIInstrInfo::createPHIDestinationCopy( 7799 MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt, 7800 const DebugLoc &DL, Register Src, Register Dst) const { 7801 auto Cur = MBB.begin(); 7802 if (Cur != MBB.end()) 7803 do { 7804 if (!Cur->isPHI() && Cur->readsRegister(Dst)) 7805 return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src); 7806 ++Cur; 7807 } while (Cur != MBB.end() && Cur != LastPHIIt); 7808 7809 return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src, 7810 Dst); 7811 } 7812 7813 MachineInstr *SIInstrInfo::createPHISourceCopy( 7814 MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, 7815 const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { 7816 if (InsPt != MBB.end() && 7817 (InsPt->getOpcode() == AMDGPU::SI_IF || 7818 InsPt->getOpcode() == AMDGPU::SI_ELSE || 7819 InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && 7820 InsPt->definesRegister(Src)) { 7821 InsPt++; 7822 return BuildMI(MBB, InsPt, DL, 7823 get(ST.isWave32() ? AMDGPU::S_MOV_B32_term 7824 : AMDGPU::S_MOV_B64_term), 7825 Dst) 7826 .addReg(Src, 0, SrcSubReg) 7827 .addReg(AMDGPU::EXEC, RegState::Implicit); 7828 } 7829 return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg, 7830 Dst); 7831 } 7832 7833 bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } 7834 7835 MachineInstr *SIInstrInfo::foldMemoryOperandImpl( 7836 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, 7837 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, 7838 VirtRegMap *VRM) const { 7839 // This is a bit of a hack (copied from AArch64). Consider this instruction: 7840 // 7841 // %0:sreg_32 = COPY $m0 7842 // 7843 // We explicitly chose SReg_32 for the virtual register so such a copy might 7844 // be eliminated by RegisterCoalescer. However, that may not be possible, and 7845 // %0 may even spill. We can't spill $m0 normally (it would require copying to 7846 // a numbered SGPR anyway), and since it is in the SReg_32 register class, 7847 // TargetInstrInfo::foldMemoryOperand() is going to try. 7848 // A similar issue also exists with spilling and reloading $exec registers. 7849 // 7850 // To prevent that, constrain the %0 register class here. 7851 if (MI.isFullCopy()) { 7852 Register DstReg = MI.getOperand(0).getReg(); 7853 Register SrcReg = MI.getOperand(1).getReg(); 7854 if ((DstReg.isVirtual() || SrcReg.isVirtual()) && 7855 (DstReg.isVirtual() != SrcReg.isVirtual())) { 7856 MachineRegisterInfo &MRI = MF.getRegInfo(); 7857 Register VirtReg = DstReg.isVirtual() ? DstReg : SrcReg; 7858 const TargetRegisterClass *RC = MRI.getRegClass(VirtReg); 7859 if (RC->hasSuperClassEq(&AMDGPU::SReg_32RegClass)) { 7860 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_32_XM0_XEXECRegClass); 7861 return nullptr; 7862 } else if (RC->hasSuperClassEq(&AMDGPU::SReg_64RegClass)) { 7863 MRI.constrainRegClass(VirtReg, &AMDGPU::SReg_64_XEXECRegClass); 7864 return nullptr; 7865 } 7866 } 7867 } 7868 7869 return nullptr; 7870 } 7871 7872 unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, 7873 const MachineInstr &MI, 7874 unsigned *PredCost) const { 7875 if (MI.isBundle()) { 7876 MachineBasicBlock::const_instr_iterator I(MI.getIterator()); 7877 MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); 7878 unsigned Lat = 0, Count = 0; 7879 for (++I; I != E && I->isBundledWithPred(); ++I) { 7880 ++Count; 7881 Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); 7882 } 7883 return Lat + Count - 1; 7884 } 7885 7886 return SchedModel.computeInstrLatency(&MI); 7887 } 7888 7889 unsigned SIInstrInfo::getDSShaderTypeValue(const MachineFunction &MF) { 7890 switch (MF.getFunction().getCallingConv()) { 7891 case CallingConv::AMDGPU_PS: 7892 return 1; 7893 case CallingConv::AMDGPU_VS: 7894 return 2; 7895 case CallingConv::AMDGPU_GS: 7896 return 3; 7897 case CallingConv::AMDGPU_HS: 7898 case CallingConv::AMDGPU_LS: 7899 case CallingConv::AMDGPU_ES: 7900 report_fatal_error("ds_ordered_count unsupported for this calling conv"); 7901 case CallingConv::AMDGPU_CS: 7902 case CallingConv::AMDGPU_KERNEL: 7903 case CallingConv::C: 7904 case CallingConv::Fast: 7905 default: 7906 // Assume other calling conventions are various compute callable functions 7907 return 0; 7908 } 7909 } 7910