1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //==-----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// Defines an instruction selector for the AMDGPU target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "AMDGPU.h" 15 #include "AMDGPUArgumentUsageInfo.h" 16 #include "AMDGPUISelLowering.h" // For AMDGPUISD 17 #include "AMDGPUInstrInfo.h" 18 #include "AMDGPUPerfHintAnalysis.h" 19 #include "AMDGPUSubtarget.h" 20 #include "AMDGPUTargetMachine.h" 21 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 22 #include "SIDefines.h" 23 #include "SIISelLowering.h" 24 #include "SIInstrInfo.h" 25 #include "SIMachineFunctionInfo.h" 26 #include "SIRegisterInfo.h" 27 #include "llvm/ADT/APInt.h" 28 #include "llvm/ADT/SmallVector.h" 29 #include "llvm/ADT/StringRef.h" 30 #include "llvm/Analysis/LegacyDivergenceAnalysis.h" 31 #include "llvm/Analysis/ValueTracking.h" 32 #include "llvm/CodeGen/FunctionLoweringInfo.h" 33 #include "llvm/CodeGen/ISDOpcodes.h" 34 #include "llvm/CodeGen/MachineFunction.h" 35 #include "llvm/CodeGen/MachineRegisterInfo.h" 36 #include "llvm/CodeGen/SelectionDAG.h" 37 #include "llvm/CodeGen/SelectionDAGISel.h" 38 #include "llvm/CodeGen/SelectionDAGNodes.h" 39 #include "llvm/CodeGen/ValueTypes.h" 40 #include "llvm/IR/BasicBlock.h" 41 #include "llvm/InitializePasses.h" 42 #ifdef EXPENSIVE_CHECKS 43 #include "llvm/IR/Dominators.h" 44 #endif 45 #include "llvm/IR/Instruction.h" 46 #include "llvm/MC/MCInstrDesc.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/CodeGen.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MachineValueType.h" 51 #include "llvm/Support/MathExtras.h" 52 #include <cassert> 53 #include <cstdint> 54 #include <new> 55 #include <vector> 56 57 #define DEBUG_TYPE "isel" 58 59 using namespace llvm; 60 61 namespace llvm { 62 63 class R600InstrInfo; 64 65 } // end namespace llvm 66 67 //===----------------------------------------------------------------------===// 68 // Instruction Selector Implementation 69 //===----------------------------------------------------------------------===// 70 71 namespace { 72 73 static bool isNullConstantOrUndef(SDValue V) { 74 if (V.isUndef()) 75 return true; 76 77 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); 78 return Const != nullptr && Const->isNullValue(); 79 } 80 81 static bool getConstantValue(SDValue N, uint32_t &Out) { 82 // This is only used for packed vectors, where ussing 0 for undef should 83 // always be good. 84 if (N.isUndef()) { 85 Out = 0; 86 return true; 87 } 88 89 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 90 Out = C->getAPIntValue().getSExtValue(); 91 return true; 92 } 93 94 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 95 Out = C->getValueAPF().bitcastToAPInt().getSExtValue(); 96 return true; 97 } 98 99 return false; 100 } 101 102 // TODO: Handle undef as zero 103 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG, 104 bool Negate = false) { 105 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2); 106 uint32_t LHSVal, RHSVal; 107 if (getConstantValue(N->getOperand(0), LHSVal) && 108 getConstantValue(N->getOperand(1), RHSVal)) { 109 SDLoc SL(N); 110 uint32_t K = Negate ? 111 (-LHSVal & 0xffff) | (-RHSVal << 16) : 112 (LHSVal & 0xffff) | (RHSVal << 16); 113 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0), 114 DAG.getTargetConstant(K, SL, MVT::i32)); 115 } 116 117 return nullptr; 118 } 119 120 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) { 121 return packConstantV2I16(N, DAG, true); 122 } 123 124 /// AMDGPU specific code to select AMDGPU machine instructions for 125 /// SelectionDAG operations. 126 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 127 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 128 // make the right decision when generating code for different targets. 129 const GCNSubtarget *Subtarget; 130 131 // Default FP mode for the current function. 132 AMDGPU::SIModeRegisterDefaults Mode; 133 134 bool EnableLateStructurizeCFG; 135 136 public: 137 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 138 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 139 : SelectionDAGISel(*TM, OptLevel) { 140 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 141 } 142 ~AMDGPUDAGToDAGISel() override = default; 143 144 void getAnalysisUsage(AnalysisUsage &AU) const override { 145 AU.addRequired<AMDGPUArgumentUsageInfo>(); 146 AU.addRequired<LegacyDivergenceAnalysis>(); 147 #ifdef EXPENSIVE_CHECKS 148 AU.addRequired<DominatorTreeWrapperPass>(); 149 AU.addRequired<LoopInfoWrapperPass>(); 150 #endif 151 SelectionDAGISel::getAnalysisUsage(AU); 152 } 153 154 bool matchLoadD16FromBuildVector(SDNode *N) const; 155 156 bool runOnMachineFunction(MachineFunction &MF) override; 157 void PreprocessISelDAG() override; 158 void Select(SDNode *N) override; 159 StringRef getPassName() const override; 160 void PostprocessISelDAG() override; 161 162 protected: 163 void SelectBuildVector(SDNode *N, unsigned RegClassID); 164 165 private: 166 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 167 bool isNoNanSrc(SDValue N) const; 168 bool isInlineImmediate(const SDNode *N, bool Negated = false) const; 169 bool isNegInlineImmediate(const SDNode *N) const { 170 return isInlineImmediate(N, true); 171 } 172 173 bool isInlineImmediate16(int64_t Imm) const { 174 return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); 175 } 176 177 bool isInlineImmediate32(int64_t Imm) const { 178 return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); 179 } 180 181 bool isInlineImmediate64(int64_t Imm) const { 182 return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); 183 } 184 185 bool isInlineImmediate(const APFloat &Imm) const { 186 return Subtarget->getInstrInfo()->isInlineConstant(Imm); 187 } 188 189 bool isVGPRImm(const SDNode *N) const; 190 bool isUniformLoad(const SDNode *N) const; 191 bool isUniformBr(const SDNode *N) const; 192 193 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const; 194 195 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const; 196 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const; 197 SDNode *glueCopyToM0LDSInit(SDNode *N) const; 198 199 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 200 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 201 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 202 bool isDSOffsetLegal(SDValue Base, unsigned Offset, 203 unsigned OffsetBits) const; 204 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 205 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 206 SDValue &Offset1) const; 207 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 208 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 209 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC, 210 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 211 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 212 SDValue &SOffset, SDValue &Offset, SDValue &GLC, 213 SDValue &SLC, SDValue &TFE, SDValue &DLC, 214 SDValue &SWZ) const; 215 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 216 SDValue &VAddr, SDValue &SOffset, SDValue &Offset, 217 SDValue &SLC) const; 218 bool SelectMUBUFScratchOffen(SDNode *Parent, 219 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 220 SDValue &SOffset, SDValue &ImmOffset) const; 221 bool SelectMUBUFScratchOffset(SDNode *Parent, 222 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 223 SDValue &Offset) const; 224 225 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset, 226 SDValue &Offset, SDValue &GLC, SDValue &SLC, 227 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const; 228 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 229 SDValue &Offset, SDValue &SLC) const; 230 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 231 SDValue &Offset) const; 232 233 template <bool IsSigned> 234 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr, 235 SDValue &Offset, SDValue &SLC) const; 236 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr, 237 SDValue &Offset, SDValue &SLC) const; 238 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr, 239 SDValue &Offset, SDValue &SLC) const; 240 241 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 242 bool &Imm) const; 243 SDValue Expand32BitAddress(SDValue Addr) const; 244 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 245 bool &Imm) const; 246 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 247 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 248 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 249 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 250 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 251 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 252 253 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 254 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const; 255 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 256 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 257 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 258 SDValue &Clamp, SDValue &Omod) const; 259 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 260 SDValue &Clamp, SDValue &Omod) const; 261 262 bool SelectVOP3OMods(SDValue In, SDValue &Src, 263 SDValue &Clamp, SDValue &Omod) const; 264 265 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 266 267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 268 269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 271 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 272 273 SDValue getHi16Elt(SDValue In) const; 274 275 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const; 276 277 void SelectADD_SUB_I64(SDNode *N); 278 void SelectAddcSubb(SDNode *N); 279 void SelectUADDO_USUBO(SDNode *N); 280 void SelectDIV_SCALE(SDNode *N); 281 void SelectMAD_64_32(SDNode *N); 282 void SelectFMA_W_CHAIN(SDNode *N); 283 void SelectFMUL_W_CHAIN(SDNode *N); 284 285 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 286 uint32_t Offset, uint32_t Width); 287 void SelectS_BFEFromShifts(SDNode *N); 288 void SelectS_BFE(SDNode *N); 289 bool isCBranchSCC(const SDNode *N) const; 290 void SelectBRCOND(SDNode *N); 291 void SelectFMAD_FMA(SDNode *N); 292 void SelectATOMIC_CMP_SWAP(SDNode *N); 293 void SelectDSAppendConsume(SDNode *N, unsigned IntrID); 294 void SelectDS_GWS(SDNode *N, unsigned IntrID); 295 void SelectInterpP1F16(SDNode *N); 296 void SelectINTRINSIC_W_CHAIN(SDNode *N); 297 void SelectINTRINSIC_WO_CHAIN(SDNode *N); 298 void SelectINTRINSIC_VOID(SDNode *N); 299 300 protected: 301 // Include the pieces autogenerated from the target description. 302 #include "AMDGPUGenDAGISel.inc" 303 }; 304 305 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 306 const R600Subtarget *Subtarget; 307 308 bool isConstantLoad(const MemSDNode *N, int cbID) const; 309 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 310 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 311 SDValue& Offset); 312 public: 313 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 314 AMDGPUDAGToDAGISel(TM, OptLevel) {} 315 316 void Select(SDNode *N) override; 317 318 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 319 SDValue &Offset) override; 320 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 321 SDValue &Offset) override; 322 323 bool runOnMachineFunction(MachineFunction &MF) override; 324 325 void PreprocessISelDAG() override {} 326 327 protected: 328 // Include the pieces autogenerated from the target description. 329 #include "R600GenDAGISel.inc" 330 }; 331 332 static SDValue stripBitcast(SDValue Val) { 333 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 334 } 335 336 // Figure out if this is really an extract of the high 16-bits of a dword. 337 static bool isExtractHiElt(SDValue In, SDValue &Out) { 338 In = stripBitcast(In); 339 if (In.getOpcode() != ISD::TRUNCATE) 340 return false; 341 342 SDValue Srl = In.getOperand(0); 343 if (Srl.getOpcode() == ISD::SRL) { 344 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 345 if (ShiftAmt->getZExtValue() == 16) { 346 Out = stripBitcast(Srl.getOperand(0)); 347 return true; 348 } 349 } 350 } 351 352 return false; 353 } 354 355 // Look through operations that obscure just looking at the low 16-bits of the 356 // same register. 357 static SDValue stripExtractLoElt(SDValue In) { 358 if (In.getOpcode() == ISD::TRUNCATE) { 359 SDValue Src = In.getOperand(0); 360 if (Src.getValueType().getSizeInBits() == 32) 361 return stripBitcast(Src); 362 } 363 364 return In; 365 } 366 367 } // end anonymous namespace 368 369 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel", 370 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 371 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 372 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 373 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis) 374 #ifdef EXPENSIVE_CHECKS 375 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 376 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) 377 #endif 378 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel", 379 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 380 381 /// This pass converts a legalized DAG into a AMDGPU-specific 382 // DAG, ready for instruction scheduling. 383 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 384 CodeGenOpt::Level OptLevel) { 385 return new AMDGPUDAGToDAGISel(TM, OptLevel); 386 } 387 388 /// This pass converts a legalized DAG into a R600-specific 389 // DAG, ready for instruction scheduling. 390 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 391 CodeGenOpt::Level OptLevel) { 392 return new R600DAGToDAGISel(TM, OptLevel); 393 } 394 395 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 396 #ifdef EXPENSIVE_CHECKS 397 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 398 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 399 for (auto &L : LI->getLoopsInPreorder()) { 400 assert(L->isLCSSAForm(DT)); 401 } 402 #endif 403 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 404 Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction()); 405 return SelectionDAGISel::runOnMachineFunction(MF); 406 } 407 408 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const { 409 assert(Subtarget->d16PreservesUnusedBits()); 410 MVT VT = N->getValueType(0).getSimpleVT(); 411 if (VT != MVT::v2i16 && VT != MVT::v2f16) 412 return false; 413 414 SDValue Lo = N->getOperand(0); 415 SDValue Hi = N->getOperand(1); 416 417 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi)); 418 419 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo 420 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo 421 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo 422 423 // Need to check for possible indirect dependencies on the other half of the 424 // vector to avoid introducing a cycle. 425 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) { 426 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 427 428 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo); 429 SDValue Ops[] = { 430 LdHi->getChain(), LdHi->getBasePtr(), TiedIn 431 }; 432 433 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI; 434 if (LdHi->getMemoryVT() == MVT::i8) { 435 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ? 436 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8; 437 } else { 438 assert(LdHi->getMemoryVT() == MVT::i16); 439 } 440 441 SDValue NewLoadHi = 442 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList, 443 Ops, LdHi->getMemoryVT(), 444 LdHi->getMemOperand()); 445 446 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi); 447 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1)); 448 return true; 449 } 450 451 // build_vector (load ptr), hi -> load_d16_lo ptr, hi 452 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi 453 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi 454 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo)); 455 if (LdLo && Lo.hasOneUse()) { 456 SDValue TiedIn = getHi16Elt(Hi); 457 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode())) 458 return false; 459 460 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other); 461 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO; 462 if (LdLo->getMemoryVT() == MVT::i8) { 463 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ? 464 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8; 465 } else { 466 assert(LdLo->getMemoryVT() == MVT::i16); 467 } 468 469 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn); 470 471 SDValue Ops[] = { 472 LdLo->getChain(), LdLo->getBasePtr(), TiedIn 473 }; 474 475 SDValue NewLoadLo = 476 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList, 477 Ops, LdLo->getMemoryVT(), 478 LdLo->getMemOperand()); 479 480 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo); 481 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1)); 482 return true; 483 } 484 485 return false; 486 } 487 488 void AMDGPUDAGToDAGISel::PreprocessISelDAG() { 489 if (!Subtarget->d16PreservesUnusedBits()) 490 return; 491 492 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end(); 493 494 bool MadeChange = false; 495 while (Position != CurDAG->allnodes_begin()) { 496 SDNode *N = &*--Position; 497 if (N->use_empty()) 498 continue; 499 500 switch (N->getOpcode()) { 501 case ISD::BUILD_VECTOR: 502 MadeChange |= matchLoadD16FromBuildVector(N); 503 break; 504 default: 505 break; 506 } 507 } 508 509 if (MadeChange) { 510 CurDAG->RemoveDeadNodes(); 511 LLVM_DEBUG(dbgs() << "After PreProcess:\n"; 512 CurDAG->dump();); 513 } 514 } 515 516 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 517 if (TM.Options.NoNaNsFPMath) 518 return true; 519 520 // TODO: Move into isKnownNeverNaN 521 if (N->getFlags().isDefined()) 522 return N->getFlags().hasNoNaNs(); 523 524 return CurDAG->isKnownNeverNaN(N); 525 } 526 527 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N, 528 bool Negated) const { 529 if (N->isUndef()) 530 return true; 531 532 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 533 if (Negated) { 534 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 535 return TII->isInlineConstant(-C->getAPIntValue()); 536 537 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 538 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt()); 539 540 } else { 541 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 542 return TII->isInlineConstant(C->getAPIntValue()); 543 544 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 545 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 546 } 547 548 return false; 549 } 550 551 /// Determine the register class for \p OpNo 552 /// \returns The register class of the virtual register that will be used for 553 /// the given operand number \OpNo or NULL if the register class cannot be 554 /// determined. 555 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 556 unsigned OpNo) const { 557 if (!N->isMachineOpcode()) { 558 if (N->getOpcode() == ISD::CopyToReg) { 559 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 560 if (Register::isVirtualRegister(Reg)) { 561 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 562 return MRI.getRegClass(Reg); 563 } 564 565 const SIRegisterInfo *TRI 566 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 567 return TRI->getPhysRegClass(Reg); 568 } 569 570 return nullptr; 571 } 572 573 switch (N->getMachineOpcode()) { 574 default: { 575 const MCInstrDesc &Desc = 576 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 577 unsigned OpIdx = Desc.getNumDefs() + OpNo; 578 if (OpIdx >= Desc.getNumOperands()) 579 return nullptr; 580 int RegClass = Desc.OpInfo[OpIdx].RegClass; 581 if (RegClass == -1) 582 return nullptr; 583 584 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 585 } 586 case AMDGPU::REG_SEQUENCE: { 587 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 588 const TargetRegisterClass *SuperRC = 589 Subtarget->getRegisterInfo()->getRegClass(RCID); 590 591 SDValue SubRegOp = N->getOperand(OpNo + 1); 592 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 593 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 594 SubRegIdx); 595 } 596 } 597 } 598 599 SDNode *AMDGPUDAGToDAGISel::glueCopyToOp(SDNode *N, SDValue NewChain, 600 SDValue Glue) const { 601 SmallVector <SDValue, 8> Ops; 602 Ops.push_back(NewChain); // Replace the chain. 603 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i) 604 Ops.push_back(N->getOperand(i)); 605 606 Ops.push_back(Glue); 607 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 608 } 609 610 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const { 611 const SITargetLowering& Lowering = 612 *static_cast<const SITargetLowering*>(getTargetLowering()); 613 614 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain"); 615 616 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N), Val); 617 return glueCopyToOp(N, M0, M0.getValue(1)); 618 } 619 620 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const { 621 unsigned AS = cast<MemSDNode>(N)->getAddressSpace(); 622 if (AS == AMDGPUAS::LOCAL_ADDRESS) { 623 if (Subtarget->ldsRequiresM0Init()) 624 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 625 } else if (AS == AMDGPUAS::REGION_ADDRESS) { 626 MachineFunction &MF = CurDAG->getMachineFunction(); 627 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize(); 628 return 629 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32)); 630 } 631 return N; 632 } 633 634 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm, 635 EVT VT) const { 636 SDNode *Lo = CurDAG->getMachineNode( 637 AMDGPU::S_MOV_B32, DL, MVT::i32, 638 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32)); 639 SDNode *Hi = 640 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 641 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32)); 642 const SDValue Ops[] = { 643 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 644 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 645 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)}; 646 647 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops); 648 } 649 650 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 651 EVT VT = N->getValueType(0); 652 unsigned NumVectorElts = VT.getVectorNumElements(); 653 EVT EltVT = VT.getVectorElementType(); 654 SDLoc DL(N); 655 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 656 657 if (NumVectorElts == 1) { 658 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 659 RegClass); 660 return; 661 } 662 663 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not " 664 "supported yet"); 665 // 32 = Max Num Vector Elements 666 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 667 // 1 = Vector Register Class 668 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 669 670 bool IsGCN = CurDAG->getSubtarget().getTargetTriple().getArch() == 671 Triple::amdgcn; 672 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 673 bool IsRegSeq = true; 674 unsigned NOps = N->getNumOperands(); 675 for (unsigned i = 0; i < NOps; i++) { 676 // XXX: Why is this here? 677 if (isa<RegisterSDNode>(N->getOperand(i))) { 678 IsRegSeq = false; 679 break; 680 } 681 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 682 : R600RegisterInfo::getSubRegFromChannel(i); 683 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 684 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 685 } 686 if (NOps != NumVectorElts) { 687 // Fill in the missing undef elements if this was a scalar_to_vector. 688 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 689 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 690 DL, EltVT); 691 for (unsigned i = NOps; i < NumVectorElts; ++i) { 692 unsigned Sub = IsGCN ? SIRegisterInfo::getSubRegFromChannel(i) 693 : R600RegisterInfo::getSubRegFromChannel(i); 694 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 695 RegSeqArgs[1 + (2 * i) + 1] = 696 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 697 } 698 } 699 700 if (!IsRegSeq) 701 SelectCode(N); 702 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 703 } 704 705 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 706 unsigned int Opc = N->getOpcode(); 707 if (N->isMachineOpcode()) { 708 N->setNodeId(-1); 709 return; // Already selected. 710 } 711 712 // isa<MemSDNode> almost works but is slightly too permissive for some DS 713 // intrinsics. 714 if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) || 715 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 716 Opc == ISD::ATOMIC_LOAD_FADD || 717 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 718 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) { 719 N = glueCopyToM0LDSInit(N); 720 SelectCode(N); 721 return; 722 } 723 724 switch (Opc) { 725 default: 726 break; 727 // We are selecting i64 ADD here instead of custom lower it during 728 // DAG legalization, so we can fold some i64 ADDs used for address 729 // calculation into the LOAD and STORE instructions. 730 case ISD::ADDC: 731 case ISD::ADDE: 732 case ISD::SUBC: 733 case ISD::SUBE: { 734 if (N->getValueType(0) != MVT::i64) 735 break; 736 737 SelectADD_SUB_I64(N); 738 return; 739 } 740 case ISD::ADDCARRY: 741 case ISD::SUBCARRY: 742 if (N->getValueType(0) != MVT::i32) 743 break; 744 745 SelectAddcSubb(N); 746 return; 747 case ISD::UADDO: 748 case ISD::USUBO: { 749 SelectUADDO_USUBO(N); 750 return; 751 } 752 case AMDGPUISD::FMUL_W_CHAIN: { 753 SelectFMUL_W_CHAIN(N); 754 return; 755 } 756 case AMDGPUISD::FMA_W_CHAIN: { 757 SelectFMA_W_CHAIN(N); 758 return; 759 } 760 761 case ISD::SCALAR_TO_VECTOR: 762 case ISD::BUILD_VECTOR: { 763 EVT VT = N->getValueType(0); 764 unsigned NumVectorElts = VT.getVectorNumElements(); 765 if (VT.getScalarSizeInBits() == 16) { 766 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 767 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) { 768 ReplaceNode(N, Packed); 769 return; 770 } 771 } 772 773 break; 774 } 775 776 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 777 unsigned RegClassID = 778 SIRegisterInfo::getSGPRClassForBitWidth(NumVectorElts * 32)->getID(); 779 SelectBuildVector(N, RegClassID); 780 return; 781 } 782 case ISD::BUILD_PAIR: { 783 SDValue RC, SubReg0, SubReg1; 784 SDLoc DL(N); 785 if (N->getValueType(0) == MVT::i128) { 786 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32); 787 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 788 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 789 } else if (N->getValueType(0) == MVT::i64) { 790 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 791 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 792 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 793 } else { 794 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 795 } 796 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 797 N->getOperand(1), SubReg1 }; 798 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 799 N->getValueType(0), Ops)); 800 return; 801 } 802 803 case ISD::Constant: 804 case ISD::ConstantFP: { 805 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 806 break; 807 808 uint64_t Imm; 809 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 810 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 811 else { 812 ConstantSDNode *C = cast<ConstantSDNode>(N); 813 Imm = C->getZExtValue(); 814 } 815 816 SDLoc DL(N); 817 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0))); 818 return; 819 } 820 case AMDGPUISD::BFE_I32: 821 case AMDGPUISD::BFE_U32: { 822 // There is a scalar version available, but unlike the vector version which 823 // has a separate operand for the offset and width, the scalar version packs 824 // the width and offset into a single operand. Try to move to the scalar 825 // version if the offsets are constant, so that we can try to keep extended 826 // loads of kernel arguments in SGPRs. 827 828 // TODO: Technically we could try to pattern match scalar bitshifts of 829 // dynamic values, but it's probably not useful. 830 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 831 if (!Offset) 832 break; 833 834 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 835 if (!Width) 836 break; 837 838 bool Signed = Opc == AMDGPUISD::BFE_I32; 839 840 uint32_t OffsetVal = Offset->getZExtValue(); 841 uint32_t WidthVal = Width->getZExtValue(); 842 843 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 844 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 845 return; 846 } 847 case AMDGPUISD::DIV_SCALE: { 848 SelectDIV_SCALE(N); 849 return; 850 } 851 case AMDGPUISD::MAD_I64_I32: 852 case AMDGPUISD::MAD_U64_U32: { 853 SelectMAD_64_32(N); 854 return; 855 } 856 case ISD::CopyToReg: { 857 const SITargetLowering& Lowering = 858 *static_cast<const SITargetLowering*>(getTargetLowering()); 859 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 860 break; 861 } 862 case ISD::AND: 863 case ISD::SRL: 864 case ISD::SRA: 865 case ISD::SIGN_EXTEND_INREG: 866 if (N->getValueType(0) != MVT::i32) 867 break; 868 869 SelectS_BFE(N); 870 return; 871 case ISD::BRCOND: 872 SelectBRCOND(N); 873 return; 874 case ISD::FMAD: 875 case ISD::FMA: 876 SelectFMAD_FMA(N); 877 return; 878 case AMDGPUISD::ATOMIC_CMP_SWAP: 879 SelectATOMIC_CMP_SWAP(N); 880 return; 881 case AMDGPUISD::CVT_PKRTZ_F16_F32: 882 case AMDGPUISD::CVT_PKNORM_I16_F32: 883 case AMDGPUISD::CVT_PKNORM_U16_F32: 884 case AMDGPUISD::CVT_PK_U16_U32: 885 case AMDGPUISD::CVT_PK_I16_I32: { 886 // Hack around using a legal type if f16 is illegal. 887 if (N->getValueType(0) == MVT::i32) { 888 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16; 889 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT), 890 { N->getOperand(0), N->getOperand(1) }); 891 SelectCode(N); 892 return; 893 } 894 895 break; 896 } 897 case ISD::INTRINSIC_W_CHAIN: { 898 SelectINTRINSIC_W_CHAIN(N); 899 return; 900 } 901 case ISD::INTRINSIC_WO_CHAIN: { 902 SelectINTRINSIC_WO_CHAIN(N); 903 return; 904 } 905 case ISD::INTRINSIC_VOID: { 906 SelectINTRINSIC_VOID(N); 907 return; 908 } 909 } 910 911 SelectCode(N); 912 } 913 914 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 915 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 916 const Instruction *Term = BB->getTerminator(); 917 return Term->getMetadata("amdgpu.uniform") || 918 Term->getMetadata("structurizecfg.uniform"); 919 } 920 921 StringRef AMDGPUDAGToDAGISel::getPassName() const { 922 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 923 } 924 925 //===----------------------------------------------------------------------===// 926 // Complex Patterns 927 //===----------------------------------------------------------------------===// 928 929 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 930 SDValue &Offset) { 931 return false; 932 } 933 934 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 935 SDValue &Offset) { 936 ConstantSDNode *C; 937 SDLoc DL(Addr); 938 939 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 940 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 941 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 942 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 943 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 944 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 945 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 946 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 947 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 948 Base = Addr.getOperand(0); 949 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 950 } else { 951 Base = Addr; 952 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 953 } 954 955 return true; 956 } 957 958 SDValue AMDGPUDAGToDAGISel::getMaterializedScalarImm32(int64_t Val, 959 const SDLoc &DL) const { 960 SDNode *Mov = CurDAG->getMachineNode( 961 AMDGPU::S_MOV_B32, DL, MVT::i32, 962 CurDAG->getTargetConstant(Val, DL, MVT::i32)); 963 return SDValue(Mov, 0); 964 } 965 966 // FIXME: Should only handle addcarry/subcarry 967 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 968 SDLoc DL(N); 969 SDValue LHS = N->getOperand(0); 970 SDValue RHS = N->getOperand(1); 971 972 unsigned Opcode = N->getOpcode(); 973 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 974 bool ProduceCarry = 975 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 976 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 977 978 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 979 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 980 981 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 982 DL, MVT::i32, LHS, Sub0); 983 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 984 DL, MVT::i32, LHS, Sub1); 985 986 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 987 DL, MVT::i32, RHS, Sub0); 988 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 989 DL, MVT::i32, RHS, Sub1); 990 991 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 992 993 static const unsigned OpcMap[2][2][2] = { 994 {{AMDGPU::S_SUB_U32, AMDGPU::S_ADD_U32}, 995 {AMDGPU::V_SUB_I32_e32, AMDGPU::V_ADD_I32_e32}}, 996 {{AMDGPU::S_SUBB_U32, AMDGPU::S_ADDC_U32}, 997 {AMDGPU::V_SUBB_U32_e32, AMDGPU::V_ADDC_U32_e32}}}; 998 999 unsigned Opc = OpcMap[0][N->isDivergent()][IsAdd]; 1000 unsigned CarryOpc = OpcMap[1][N->isDivergent()][IsAdd]; 1001 1002 SDNode *AddLo; 1003 if (!ConsumeCarry) { 1004 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 1005 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 1006 } else { 1007 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 1008 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 1009 } 1010 SDValue AddHiArgs[] = { 1011 SDValue(Hi0, 0), 1012 SDValue(Hi1, 0), 1013 SDValue(AddLo, 1) 1014 }; 1015 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 1016 1017 SDValue RegSequenceArgs[] = { 1018 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 1019 SDValue(AddLo,0), 1020 Sub0, 1021 SDValue(AddHi,0), 1022 Sub1, 1023 }; 1024 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1025 MVT::i64, RegSequenceArgs); 1026 1027 if (ProduceCarry) { 1028 // Replace the carry-use 1029 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 1030 } 1031 1032 // Replace the remaining uses. 1033 ReplaceNode(N, RegSequence); 1034 } 1035 1036 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) { 1037 SDLoc DL(N); 1038 SDValue LHS = N->getOperand(0); 1039 SDValue RHS = N->getOperand(1); 1040 SDValue CI = N->getOperand(2); 1041 1042 if (N->isDivergent()) { 1043 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64 1044 : AMDGPU::V_SUBB_U32_e64; 1045 CurDAG->SelectNodeTo( 1046 N, Opc, N->getVTList(), 1047 {LHS, RHS, CI, 1048 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1049 } else { 1050 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::S_ADD_CO_PSEUDO 1051 : AMDGPU::S_SUB_CO_PSEUDO; 1052 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), {LHS, RHS, CI}); 1053 } 1054 } 1055 1056 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 1057 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 1058 // carry out despite the _i32 name. These were renamed in VI to _U32. 1059 // FIXME: We should probably rename the opcodes here. 1060 bool IsAdd = N->getOpcode() == ISD::UADDO; 1061 bool IsVALU = N->isDivergent(); 1062 1063 for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; 1064 ++UI) 1065 if (UI.getUse().getResNo() == 1) { 1066 if ((IsAdd && (UI->getOpcode() != ISD::ADDCARRY)) || 1067 (!IsAdd && (UI->getOpcode() != ISD::SUBCARRY))) { 1068 IsVALU = true; 1069 break; 1070 } 1071 } 1072 1073 if (IsVALU) { 1074 unsigned Opc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 1075 1076 CurDAG->SelectNodeTo( 1077 N, Opc, N->getVTList(), 1078 {N->getOperand(0), N->getOperand(1), 1079 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/}); 1080 } else { 1081 unsigned Opc = N->getOpcode() == ISD::UADDO ? AMDGPU::S_UADDO_PSEUDO 1082 : AMDGPU::S_USUBO_PSEUDO; 1083 1084 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), 1085 {N->getOperand(0), N->getOperand(1)}); 1086 } 1087 } 1088 1089 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 1090 SDLoc SL(N); 1091 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 1092 SDValue Ops[10]; 1093 1094 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 1095 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1096 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 1097 Ops[8] = N->getOperand(0); 1098 Ops[9] = N->getOperand(4); 1099 1100 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); 1101 } 1102 1103 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 1104 SDLoc SL(N); 1105 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 1106 SDValue Ops[8]; 1107 1108 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 1109 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 1110 Ops[6] = N->getOperand(0); 1111 Ops[7] = N->getOperand(3); 1112 1113 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 1114 } 1115 1116 // We need to handle this here because tablegen doesn't support matching 1117 // instructions with multiple outputs. 1118 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 1119 SDLoc SL(N); 1120 EVT VT = N->getValueType(0); 1121 1122 assert(VT == MVT::f32 || VT == MVT::f64); 1123 1124 unsigned Opc 1125 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32; 1126 1127 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; 1128 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1129 } 1130 1131 // We need to handle this here because tablegen doesn't support matching 1132 // instructions with multiple outputs. 1133 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 1134 SDLoc SL(N); 1135 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 1136 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32; 1137 1138 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 1139 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 1140 Clamp }; 1141 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 1142 } 1143 1144 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset, 1145 unsigned OffsetBits) const { 1146 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 1147 (OffsetBits == 8 && !isUInt<8>(Offset))) 1148 return false; 1149 1150 if (Subtarget->hasUsableDSOffset() || 1151 Subtarget->unsafeDSOffsetFoldingEnabled()) 1152 return true; 1153 1154 // On Southern Islands instruction with a negative base value and an offset 1155 // don't seem to work. 1156 return CurDAG->SignBitIsZero(Base); 1157 } 1158 1159 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 1160 SDValue &Offset) const { 1161 SDLoc DL(Addr); 1162 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1163 SDValue N0 = Addr.getOperand(0); 1164 SDValue N1 = Addr.getOperand(1); 1165 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1166 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) { 1167 // (add n0, c0) 1168 Base = N0; 1169 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1170 return true; 1171 } 1172 } else if (Addr.getOpcode() == ISD::SUB) { 1173 // sub C, x -> add (sub 0, x), C 1174 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1175 int64_t ByteOffset = C->getSExtValue(); 1176 if (isUInt<16>(ByteOffset)) { 1177 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1178 1179 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1180 // the known bits in isDSOffsetLegal. We need to emit the selected node 1181 // here, so this is thrown away. 1182 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1183 Zero, Addr.getOperand(1)); 1184 1185 if (isDSOffsetLegal(Sub, ByteOffset, 16)) { 1186 SmallVector<SDValue, 3> Opnds; 1187 Opnds.push_back(Zero); 1188 Opnds.push_back(Addr.getOperand(1)); 1189 1190 // FIXME: Select to VOP3 version for with-carry. 1191 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1192 if (Subtarget->hasAddNoCarry()) { 1193 SubOp = AMDGPU::V_SUB_U32_e64; 1194 Opnds.push_back( 1195 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1196 } 1197 1198 MachineSDNode *MachineSub = 1199 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1200 1201 Base = SDValue(MachineSub, 0); 1202 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 1203 return true; 1204 } 1205 } 1206 } 1207 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1208 // If we have a constant address, prefer to put the constant into the 1209 // offset. This can save moves to load the constant address since multiple 1210 // operations can share the zero base address register, and enables merging 1211 // into read2 / write2 instructions. 1212 1213 SDLoc DL(Addr); 1214 1215 if (isUInt<16>(CAddr->getZExtValue())) { 1216 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1217 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1218 DL, MVT::i32, Zero); 1219 Base = SDValue(MovZero, 0); 1220 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1221 return true; 1222 } 1223 } 1224 1225 // default case 1226 Base = Addr; 1227 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 1228 return true; 1229 } 1230 1231 // TODO: If offset is too big, put low 16-bit into offset. 1232 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 1233 SDValue &Offset0, 1234 SDValue &Offset1) const { 1235 SDLoc DL(Addr); 1236 1237 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1238 SDValue N0 = Addr.getOperand(0); 1239 SDValue N1 = Addr.getOperand(1); 1240 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1241 unsigned DWordOffset0 = C1->getZExtValue() / 4; 1242 unsigned DWordOffset1 = DWordOffset0 + 1; 1243 // (add n0, c0) 1244 if (isDSOffsetLegal(N0, DWordOffset1, 8)) { 1245 Base = N0; 1246 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1247 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1248 return true; 1249 } 1250 } else if (Addr.getOpcode() == ISD::SUB) { 1251 // sub C, x -> add (sub 0, x), C 1252 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 1253 unsigned DWordOffset0 = C->getZExtValue() / 4; 1254 unsigned DWordOffset1 = DWordOffset0 + 1; 1255 1256 if (isUInt<8>(DWordOffset0)) { 1257 SDLoc DL(Addr); 1258 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1259 1260 // XXX - This is kind of hacky. Create a dummy sub node so we can check 1261 // the known bits in isDSOffsetLegal. We need to emit the selected node 1262 // here, so this is thrown away. 1263 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 1264 Zero, Addr.getOperand(1)); 1265 1266 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) { 1267 SmallVector<SDValue, 3> Opnds; 1268 Opnds.push_back(Zero); 1269 Opnds.push_back(Addr.getOperand(1)); 1270 unsigned SubOp = AMDGPU::V_SUB_I32_e32; 1271 if (Subtarget->hasAddNoCarry()) { 1272 SubOp = AMDGPU::V_SUB_U32_e64; 1273 Opnds.push_back( 1274 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit 1275 } 1276 1277 MachineSDNode *MachineSub 1278 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds); 1279 1280 Base = SDValue(MachineSub, 0); 1281 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1282 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1283 return true; 1284 } 1285 } 1286 } 1287 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1288 unsigned DWordOffset0 = CAddr->getZExtValue() / 4; 1289 unsigned DWordOffset1 = DWordOffset0 + 1; 1290 assert(4 * DWordOffset0 == CAddr->getZExtValue()); 1291 1292 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) { 1293 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 1294 MachineSDNode *MovZero 1295 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1296 DL, MVT::i32, Zero); 1297 Base = SDValue(MovZero, 0); 1298 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 1299 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 1300 return true; 1301 } 1302 } 1303 1304 // default case 1305 1306 Base = Addr; 1307 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 1308 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 1309 return true; 1310 } 1311 1312 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, 1313 SDValue &VAddr, SDValue &SOffset, 1314 SDValue &Offset, SDValue &Offen, 1315 SDValue &Idxen, SDValue &Addr64, 1316 SDValue &GLC, SDValue &SLC, 1317 SDValue &TFE, SDValue &DLC, 1318 SDValue &SWZ) const { 1319 // Subtarget prefers to use flat instruction 1320 // FIXME: This should be a pattern predicate and not reach here 1321 if (Subtarget->useFlatForGlobal()) 1322 return false; 1323 1324 SDLoc DL(Addr); 1325 1326 if (!GLC.getNode()) 1327 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1328 if (!SLC.getNode()) 1329 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1330 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1); 1331 DLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 1332 SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1); 1333 1334 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1335 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1336 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1337 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1338 1339 ConstantSDNode *C1 = nullptr; 1340 SDValue N0 = Addr; 1341 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1342 C1 = cast<ConstantSDNode>(Addr.getOperand(1)); 1343 if (isUInt<32>(C1->getZExtValue())) 1344 N0 = Addr.getOperand(0); 1345 else 1346 C1 = nullptr; 1347 } 1348 1349 if (N0.getOpcode() == ISD::ADD) { 1350 // (add N2, N3) -> addr64, or 1351 // (add (add N2, N3), C1) -> addr64 1352 SDValue N2 = N0.getOperand(0); 1353 SDValue N3 = N0.getOperand(1); 1354 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1355 1356 if (N2->isDivergent()) { 1357 if (N3->isDivergent()) { 1358 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the 1359 // addr64, and construct the resource from a 0 address. 1360 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1361 VAddr = N0; 1362 } else { 1363 // N2 is divergent, N3 is not. 1364 Ptr = N3; 1365 VAddr = N2; 1366 } 1367 } else { 1368 // N2 is not divergent. 1369 Ptr = N2; 1370 VAddr = N3; 1371 } 1372 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1373 } else if (N0->isDivergent()) { 1374 // N0 is divergent. Use it as the addr64, and construct the resource from a 1375 // 0 address. 1376 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0); 1377 VAddr = N0; 1378 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1379 } else { 1380 // N0 -> offset, or 1381 // (N0 + C1) -> offset 1382 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1383 Ptr = N0; 1384 } 1385 1386 if (!C1) { 1387 // No offset. 1388 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1389 return true; 1390 } 1391 1392 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1393 // Legal offset for instruction. 1394 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1395 return true; 1396 } 1397 1398 // Illegal offset, store it in soffset. 1399 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1400 SOffset = 1401 SDValue(CurDAG->getMachineNode( 1402 AMDGPU::S_MOV_B32, DL, MVT::i32, 1403 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1404 0); 1405 return true; 1406 } 1407 1408 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1409 SDValue &VAddr, SDValue &SOffset, 1410 SDValue &Offset, SDValue &GLC, 1411 SDValue &SLC, SDValue &TFE, 1412 SDValue &DLC, SDValue &SWZ) const { 1413 SDValue Ptr, Offen, Idxen, Addr64; 1414 1415 // addr64 bit was removed for volcanic islands. 1416 // FIXME: This should be a pattern predicate and not reach here 1417 if (!Subtarget->hasAddr64()) 1418 return false; 1419 1420 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1421 GLC, SLC, TFE, DLC, SWZ)) 1422 return false; 1423 1424 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1425 if (C->getSExtValue()) { 1426 SDLoc DL(Addr); 1427 1428 const SITargetLowering& Lowering = 1429 *static_cast<const SITargetLowering*>(getTargetLowering()); 1430 1431 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1432 return true; 1433 } 1434 1435 return false; 1436 } 1437 1438 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1439 SDValue &VAddr, SDValue &SOffset, 1440 SDValue &Offset, 1441 SDValue &SLC) const { 1442 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1); 1443 SDValue GLC, TFE, DLC, SWZ; 1444 1445 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1446 } 1447 1448 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 1449 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 1450 return PSV && PSV->isStack(); 1451 } 1452 1453 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1454 SDLoc DL(N); 1455 const MachineFunction &MF = CurDAG->getMachineFunction(); 1456 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1457 1458 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) { 1459 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1460 FI->getValueType(0)); 1461 1462 // If we can resolve this to a frame index access, this will be relative to 1463 // either the stack or frame pointer SGPR. 1464 return std::make_pair( 1465 TFI, CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32)); 1466 } 1467 1468 // If we don't know this private access is a local stack object, it needs to 1469 // be relative to the entry point's scratch wave offset. 1470 return std::make_pair(N, CurDAG->getTargetConstant(0, DL, MVT::i32)); 1471 } 1472 1473 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1474 SDValue Addr, SDValue &Rsrc, 1475 SDValue &VAddr, SDValue &SOffset, 1476 SDValue &ImmOffset) const { 1477 1478 SDLoc DL(Addr); 1479 MachineFunction &MF = CurDAG->getMachineFunction(); 1480 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1481 1482 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1483 1484 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1485 int64_t Imm = CAddr->getSExtValue(); 1486 const int64_t NullPtr = 1487 AMDGPUTargetMachine::getNullPointerValue(AMDGPUAS::PRIVATE_ADDRESS); 1488 // Don't fold null pointer. 1489 if (Imm != NullPtr) { 1490 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1491 MachineSDNode *MovHighBits = CurDAG->getMachineNode( 1492 AMDGPU::V_MOV_B32_e32, DL, MVT::i32, HighBits); 1493 VAddr = SDValue(MovHighBits, 0); 1494 1495 // In a call sequence, stores to the argument stack area are relative to the 1496 // stack pointer. 1497 const MachinePointerInfo &PtrInfo 1498 = cast<MemSDNode>(Parent)->getPointerInfo(); 1499 SOffset = isStackPtrRelative(PtrInfo) 1500 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1501 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1502 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1503 return true; 1504 } 1505 } 1506 1507 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1508 // (add n0, c1) 1509 1510 SDValue N0 = Addr.getOperand(0); 1511 SDValue N1 = Addr.getOperand(1); 1512 1513 // Offsets in vaddr must be positive if range checking is enabled. 1514 // 1515 // The total computation of vaddr + soffset + offset must not overflow. If 1516 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1517 // overflowing. 1518 // 1519 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1520 // always perform a range check. If a negative vaddr base index was used, 1521 // this would fail the range check. The overall address computation would 1522 // compute a valid address, but this doesn't happen due to the range 1523 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1524 // 1525 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1526 // MUBUF vaddr, but not on older subtargets which can only do this if the 1527 // sign bit is known 0. 1528 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1529 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1530 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1531 CurDAG->SignBitIsZero(N0))) { 1532 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1533 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1534 return true; 1535 } 1536 } 1537 1538 // (node) 1539 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1540 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1541 return true; 1542 } 1543 1544 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1545 SDValue Addr, 1546 SDValue &SRsrc, 1547 SDValue &SOffset, 1548 SDValue &Offset) const { 1549 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr); 1550 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1551 return false; 1552 1553 SDLoc DL(Addr); 1554 MachineFunction &MF = CurDAG->getMachineFunction(); 1555 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1556 1557 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1558 1559 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1560 1561 // FIXME: Get from MachinePointerInfo? We should only be using the frame 1562 // offset if we know this is in a call sequence. 1563 SOffset = isStackPtrRelative(PtrInfo) 1564 ? CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32) 1565 : CurDAG->getTargetConstant(0, DL, MVT::i32); 1566 1567 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1568 return true; 1569 } 1570 1571 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1572 SDValue &SOffset, SDValue &Offset, 1573 SDValue &GLC, SDValue &SLC, 1574 SDValue &TFE, SDValue &DLC, 1575 SDValue &SWZ) const { 1576 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1577 const SIInstrInfo *TII = 1578 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1579 1580 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1581 GLC, SLC, TFE, DLC, SWZ)) 1582 return false; 1583 1584 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1585 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1586 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1587 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1588 APInt::getAllOnesValue(32).getZExtValue(); // Size 1589 SDLoc DL(Addr); 1590 1591 const SITargetLowering& Lowering = 1592 *static_cast<const SITargetLowering*>(getTargetLowering()); 1593 1594 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1595 return true; 1596 } 1597 return false; 1598 } 1599 1600 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1601 SDValue &Soffset, SDValue &Offset 1602 ) const { 1603 SDValue GLC, SLC, TFE, DLC, SWZ; 1604 1605 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1606 } 1607 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1608 SDValue &Soffset, SDValue &Offset, 1609 SDValue &SLC) const { 1610 SDValue GLC, TFE, DLC, SWZ; 1611 1612 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ); 1613 } 1614 1615 // Find a load or store from corresponding pattern root. 1616 // Roots may be build_vector, bitconvert or their combinations. 1617 static MemSDNode* findMemSDNode(SDNode *N) { 1618 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode(); 1619 if (MemSDNode *MN = dyn_cast<MemSDNode>(N)) 1620 return MN; 1621 assert(isa<BuildVectorSDNode>(N)); 1622 for (SDValue V : N->op_values()) 1623 if (MemSDNode *MN = 1624 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V))) 1625 return MN; 1626 llvm_unreachable("cannot find MemSDNode in the pattern!"); 1627 } 1628 1629 template <bool IsSigned> 1630 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N, 1631 SDValue Addr, 1632 SDValue &VAddr, 1633 SDValue &Offset, 1634 SDValue &SLC) const { 1635 int64_t OffsetVal = 0; 1636 1637 if (Subtarget->hasFlatInstOffsets() && 1638 (!Subtarget->hasFlatSegmentOffsetBug() || 1639 findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) && 1640 CurDAG->isBaseWithConstantOffset(Addr)) { 1641 SDValue N0 = Addr.getOperand(0); 1642 SDValue N1 = Addr.getOperand(1); 1643 uint64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1644 1645 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 1646 unsigned AS = findMemSDNode(N)->getAddressSpace(); 1647 if (TII->isLegalFLATOffset(COffsetVal, AS, IsSigned)) { 1648 Addr = N0; 1649 OffsetVal = COffsetVal; 1650 } else { 1651 // If the offset doesn't fit, put the low bits into the offset field and 1652 // add the rest. 1653 1654 SDLoc DL(N); 1655 uint64_t ImmField; 1656 const unsigned NumBits = TII->getNumFlatOffsetBits(AS, IsSigned); 1657 if (IsSigned) { 1658 ImmField = SignExtend64(COffsetVal, NumBits); 1659 1660 // Don't use a negative offset field if the base offset is positive. 1661 // Since the scheduler currently relies on the offset field, doing so 1662 // could result in strange scheduling decisions. 1663 1664 // TODO: Should we not do this in the opposite direction as well? 1665 if (static_cast<int64_t>(COffsetVal) > 0) { 1666 if (static_cast<int64_t>(ImmField) < 0) { 1667 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits - 1); 1668 ImmField = COffsetVal & OffsetMask; 1669 } 1670 } 1671 } else { 1672 // TODO: Should we do this for a negative offset? 1673 const uint64_t OffsetMask = maskTrailingOnes<uint64_t>(NumBits); 1674 ImmField = COffsetVal & OffsetMask; 1675 } 1676 1677 uint64_t RemainderOffset = COffsetVal - ImmField; 1678 1679 assert(TII->isLegalFLATOffset(ImmField, AS, IsSigned)); 1680 assert(RemainderOffset + ImmField == COffsetVal); 1681 1682 OffsetVal = ImmField; 1683 1684 // TODO: Should this try to use a scalar add pseudo if the base address is 1685 // uniform and saddr is usable? 1686 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 1687 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 1688 1689 SDNode *N0Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1690 DL, MVT::i32, N0, Sub0); 1691 SDNode *N0Hi = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 1692 DL, MVT::i32, N0, Sub1); 1693 1694 SDValue AddOffsetLo 1695 = getMaterializedScalarImm32(Lo_32(RemainderOffset), DL); 1696 SDValue AddOffsetHi 1697 = getMaterializedScalarImm32(Hi_32(RemainderOffset), DL); 1698 1699 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i1); 1700 SDValue Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1701 1702 SDNode *Add = CurDAG->getMachineNode( 1703 AMDGPU::V_ADD_I32_e64, DL, VTs, 1704 {AddOffsetLo, SDValue(N0Lo, 0), Clamp}); 1705 1706 SDNode *Addc = CurDAG->getMachineNode( 1707 AMDGPU::V_ADDC_U32_e64, DL, VTs, 1708 {AddOffsetHi, SDValue(N0Hi, 0), SDValue(Add, 1), Clamp}); 1709 1710 SDValue RegSequenceArgs[] = { 1711 CurDAG->getTargetConstant(AMDGPU::VReg_64RegClassID, DL, MVT::i32), 1712 SDValue(Add, 0), Sub0, SDValue(Addc, 0), Sub1 1713 }; 1714 1715 Addr = SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 1716 MVT::i64, RegSequenceArgs), 0); 1717 } 1718 } 1719 1720 VAddr = Addr; 1721 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1722 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); 1723 return true; 1724 } 1725 1726 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDNode *N, 1727 SDValue Addr, 1728 SDValue &VAddr, 1729 SDValue &Offset, 1730 SDValue &SLC) const { 1731 return SelectFlatOffset<false>(N, Addr, VAddr, Offset, SLC); 1732 } 1733 1734 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N, 1735 SDValue Addr, 1736 SDValue &VAddr, 1737 SDValue &Offset, 1738 SDValue &SLC) const { 1739 return SelectFlatOffset<true>(N, Addr, VAddr, Offset, SLC); 1740 } 1741 1742 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 1743 SDValue &Offset, bool &Imm) const { 1744 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 1745 if (!C) { 1746 if (ByteOffsetNode.getValueType().isScalarInteger() && 1747 ByteOffsetNode.getValueType().getSizeInBits() == 32) { 1748 Offset = ByteOffsetNode; 1749 Imm = false; 1750 return true; 1751 } 1752 if (ByteOffsetNode.getOpcode() == ISD::ZERO_EXTEND) { 1753 if (ByteOffsetNode.getOperand(0).getValueType().getSizeInBits() == 32) { 1754 Offset = ByteOffsetNode.getOperand(0); 1755 Imm = false; 1756 return true; 1757 } 1758 } 1759 return false; 1760 } 1761 1762 SDLoc SL(ByteOffsetNode); 1763 // GFX9 and GFX10 have signed byte immediate offsets. 1764 int64_t ByteOffset = C->getSExtValue(); 1765 Optional<int64_t> EncodedOffset = 1766 AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset, false); 1767 if (EncodedOffset) { 1768 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1769 Imm = true; 1770 return true; 1771 } 1772 1773 // SGPR and literal offsets are unsigned. 1774 if (ByteOffset < 0) 1775 return false; 1776 1777 EncodedOffset = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, ByteOffset); 1778 if (EncodedOffset) { 1779 Offset = CurDAG->getTargetConstant(*EncodedOffset, SL, MVT::i32); 1780 return true; 1781 } 1782 1783 if (!isUInt<32>(ByteOffset) && !isInt<32>(ByteOffset)) 1784 return false; 1785 1786 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 1787 Offset = SDValue( 1788 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, C32Bit), 0); 1789 1790 return true; 1791 } 1792 1793 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 1794 if (Addr.getValueType() != MVT::i32) 1795 return Addr; 1796 1797 // Zero-extend a 32-bit address. 1798 SDLoc SL(Addr); 1799 1800 const MachineFunction &MF = CurDAG->getMachineFunction(); 1801 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1802 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 1803 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 1804 1805 const SDValue Ops[] = { 1806 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 1807 Addr, 1808 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 1809 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 1810 0), 1811 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 1812 }; 1813 1814 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 1815 Ops), 0); 1816 } 1817 1818 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 1819 SDValue &Offset, bool &Imm) const { 1820 SDLoc SL(Addr); 1821 1822 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 1823 // wraparound, because s_load instructions perform the addition in 64 bits. 1824 if ((Addr.getValueType() != MVT::i32 || 1825 Addr->getFlags().hasNoUnsignedWrap()) && 1826 (CurDAG->isBaseWithConstantOffset(Addr) || 1827 Addr.getOpcode() == ISD::ADD)) { 1828 SDValue N0 = Addr.getOperand(0); 1829 SDValue N1 = Addr.getOperand(1); 1830 1831 if (SelectSMRDOffset(N1, Offset, Imm)) { 1832 SBase = Expand32BitAddress(N0); 1833 return true; 1834 } 1835 } 1836 SBase = Expand32BitAddress(Addr); 1837 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 1838 Imm = true; 1839 return true; 1840 } 1841 1842 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 1843 SDValue &Offset) const { 1844 bool Imm = false; 1845 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 1846 } 1847 1848 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 1849 SDValue &Offset) const { 1850 1851 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1852 1853 bool Imm = false; 1854 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 1855 return false; 1856 1857 return !Imm && isa<ConstantSDNode>(Offset); 1858 } 1859 1860 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 1861 SDValue &Offset) const { 1862 bool Imm = false; 1863 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 1864 !isa<ConstantSDNode>(Offset); 1865 } 1866 1867 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 1868 SDValue &Offset) const { 1869 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1870 // The immediate offset for S_BUFFER instructions is unsigned. 1871 if (auto Imm = 1872 AMDGPU::getSMRDEncodedOffset(*Subtarget, C->getZExtValue(), true)) { 1873 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1874 return true; 1875 } 1876 } 1877 1878 return false; 1879 } 1880 1881 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 1882 SDValue &Offset) const { 1883 assert(Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS); 1884 1885 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr)) { 1886 if (auto Imm = AMDGPU::getSMRDEncodedLiteralOffset32(*Subtarget, 1887 C->getZExtValue())) { 1888 Offset = CurDAG->getTargetConstant(*Imm, SDLoc(Addr), MVT::i32); 1889 return true; 1890 } 1891 } 1892 1893 return false; 1894 } 1895 1896 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 1897 SDValue &Base, 1898 SDValue &Offset) const { 1899 SDLoc DL(Index); 1900 1901 if (CurDAG->isBaseWithConstantOffset(Index)) { 1902 SDValue N0 = Index.getOperand(0); 1903 SDValue N1 = Index.getOperand(1); 1904 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1905 1906 // (add n0, c0) 1907 // Don't peel off the offset (c0) if doing so could possibly lead 1908 // the base (n0) to be negative. 1909 // (or n0, |c0|) can never change a sign given isBaseWithConstantOffset. 1910 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0) || 1911 (Index->getOpcode() == ISD::OR && C1->getSExtValue() >= 0)) { 1912 Base = N0; 1913 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 1914 return true; 1915 } 1916 } 1917 1918 if (isa<ConstantSDNode>(Index)) 1919 return false; 1920 1921 Base = Index; 1922 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1923 return true; 1924 } 1925 1926 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 1927 SDValue Val, uint32_t Offset, 1928 uint32_t Width) { 1929 // Transformation function, pack the offset and width of a BFE into 1930 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1931 // source, bits [5:0] contain the offset and bits [22:16] the width. 1932 uint32_t PackedVal = Offset | (Width << 16); 1933 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 1934 1935 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 1936 } 1937 1938 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 1939 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 1940 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 1941 // Predicate: 0 < b <= c < 32 1942 1943 const SDValue &Shl = N->getOperand(0); 1944 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 1945 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1946 1947 if (B && C) { 1948 uint32_t BVal = B->getZExtValue(); 1949 uint32_t CVal = C->getZExtValue(); 1950 1951 if (0 < BVal && BVal <= CVal && CVal < 32) { 1952 bool Signed = N->getOpcode() == ISD::SRA; 1953 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1954 1955 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 1956 32 - CVal)); 1957 return; 1958 } 1959 } 1960 SelectCode(N); 1961 } 1962 1963 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 1964 switch (N->getOpcode()) { 1965 case ISD::AND: 1966 if (N->getOperand(0).getOpcode() == ISD::SRL) { 1967 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 1968 // Predicate: isMask(mask) 1969 const SDValue &Srl = N->getOperand(0); 1970 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 1971 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1972 1973 if (Shift && Mask) { 1974 uint32_t ShiftVal = Shift->getZExtValue(); 1975 uint32_t MaskVal = Mask->getZExtValue(); 1976 1977 if (isMask_32(MaskVal)) { 1978 uint32_t WidthVal = countPopulation(MaskVal); 1979 1980 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1981 Srl.getOperand(0), ShiftVal, WidthVal)); 1982 return; 1983 } 1984 } 1985 } 1986 break; 1987 case ISD::SRL: 1988 if (N->getOperand(0).getOpcode() == ISD::AND) { 1989 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 1990 // Predicate: isMask(mask >> b) 1991 const SDValue &And = N->getOperand(0); 1992 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1993 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 1994 1995 if (Shift && Mask) { 1996 uint32_t ShiftVal = Shift->getZExtValue(); 1997 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 1998 1999 if (isMask_32(MaskVal)) { 2000 uint32_t WidthVal = countPopulation(MaskVal); 2001 2002 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 2003 And.getOperand(0), ShiftVal, WidthVal)); 2004 return; 2005 } 2006 } 2007 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 2008 SelectS_BFEFromShifts(N); 2009 return; 2010 } 2011 break; 2012 case ISD::SRA: 2013 if (N->getOperand(0).getOpcode() == ISD::SHL) { 2014 SelectS_BFEFromShifts(N); 2015 return; 2016 } 2017 break; 2018 2019 case ISD::SIGN_EXTEND_INREG: { 2020 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 2021 SDValue Src = N->getOperand(0); 2022 if (Src.getOpcode() != ISD::SRL) 2023 break; 2024 2025 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 2026 if (!Amt) 2027 break; 2028 2029 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 2030 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 2031 Amt->getZExtValue(), Width)); 2032 return; 2033 } 2034 } 2035 2036 SelectCode(N); 2037 } 2038 2039 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 2040 assert(N->getOpcode() == ISD::BRCOND); 2041 if (!N->hasOneUse()) 2042 return false; 2043 2044 SDValue Cond = N->getOperand(1); 2045 if (Cond.getOpcode() == ISD::CopyToReg) 2046 Cond = Cond.getOperand(2); 2047 2048 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 2049 return false; 2050 2051 MVT VT = Cond.getOperand(0).getSimpleValueType(); 2052 if (VT == MVT::i32) 2053 return true; 2054 2055 if (VT == MVT::i64) { 2056 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 2057 2058 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 2059 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 2060 } 2061 2062 return false; 2063 } 2064 2065 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 2066 SDValue Cond = N->getOperand(1); 2067 2068 if (Cond.isUndef()) { 2069 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 2070 N->getOperand(2), N->getOperand(0)); 2071 return; 2072 } 2073 2074 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget); 2075 const SIRegisterInfo *TRI = ST->getRegisterInfo(); 2076 2077 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 2078 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 2079 Register CondReg = UseSCCBr ? AMDGPU::SCC : TRI->getVCC(); 2080 SDLoc SL(N); 2081 2082 if (!UseSCCBr) { 2083 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 2084 // analyzed what generates the vcc value, so we do not know whether vcc 2085 // bits for disabled lanes are 0. Thus we need to mask out bits for 2086 // disabled lanes. 2087 // 2088 // For the case that we select S_CBRANCH_SCC1 and it gets 2089 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 2090 // SIInstrInfo::moveToVALU which inserts the S_AND). 2091 // 2092 // We could add an analysis of what generates the vcc value here and omit 2093 // the S_AND when is unnecessary. But it would be better to add a separate 2094 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 2095 // catches both cases. 2096 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32 2097 : AMDGPU::S_AND_B64, 2098 SL, MVT::i1, 2099 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO 2100 : AMDGPU::EXEC, 2101 MVT::i1), 2102 Cond), 2103 0); 2104 } 2105 2106 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 2107 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 2108 N->getOperand(2), // Basic Block 2109 VCC.getValue(0)); 2110 } 2111 2112 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 2113 MVT VT = N->getSimpleValueType(0); 2114 bool IsFMA = N->getOpcode() == ISD::FMA; 2115 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 2116 !Subtarget->hasFmaMixInsts()) || 2117 ((IsFMA && Subtarget->hasMadMixInsts()) || 2118 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 2119 SelectCode(N); 2120 return; 2121 } 2122 2123 SDValue Src0 = N->getOperand(0); 2124 SDValue Src1 = N->getOperand(1); 2125 SDValue Src2 = N->getOperand(2); 2126 unsigned Src0Mods, Src1Mods, Src2Mods; 2127 2128 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 2129 // using the conversion from f16. 2130 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 2131 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 2132 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 2133 2134 assert((IsFMA || !Mode.allFP32Denormals()) && 2135 "fmad selected with denormals enabled"); 2136 // TODO: We can select this with f32 denormals enabled if all the sources are 2137 // converted from f16 (in which case fmad isn't legal). 2138 2139 if (Sel0 || Sel1 || Sel2) { 2140 // For dummy operands. 2141 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2142 SDValue Ops[] = { 2143 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 2144 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 2145 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 2146 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 2147 Zero, Zero 2148 }; 2149 2150 CurDAG->SelectNodeTo(N, 2151 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 2152 MVT::f32, Ops); 2153 } else { 2154 SelectCode(N); 2155 } 2156 } 2157 2158 // This is here because there isn't a way to use the generated sub0_sub1 as the 2159 // subreg index to EXTRACT_SUBREG in tablegen. 2160 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 2161 MemSDNode *Mem = cast<MemSDNode>(N); 2162 unsigned AS = Mem->getAddressSpace(); 2163 if (AS == AMDGPUAS::FLAT_ADDRESS) { 2164 SelectCode(N); 2165 return; 2166 } 2167 2168 MVT VT = N->getSimpleValueType(0); 2169 bool Is32 = (VT == MVT::i32); 2170 SDLoc SL(N); 2171 2172 MachineSDNode *CmpSwap = nullptr; 2173 if (Subtarget->hasAddr64()) { 2174 SDValue SRsrc, VAddr, SOffset, Offset, SLC; 2175 2176 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) { 2177 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 2178 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 2179 SDValue CmpVal = Mem->getOperand(2); 2180 2181 // XXX - Do we care about glue operands? 2182 2183 SDValue Ops[] = { 2184 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2185 }; 2186 2187 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2188 } 2189 } 2190 2191 if (!CmpSwap) { 2192 SDValue SRsrc, SOffset, Offset, SLC; 2193 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) { 2194 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 2195 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 2196 2197 SDValue CmpVal = Mem->getOperand(2); 2198 SDValue Ops[] = { 2199 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain() 2200 }; 2201 2202 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 2203 } 2204 } 2205 2206 if (!CmpSwap) { 2207 SelectCode(N); 2208 return; 2209 } 2210 2211 MachineMemOperand *MMO = Mem->getMemOperand(); 2212 CurDAG->setNodeMemRefs(CmpSwap, {MMO}); 2213 2214 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 2215 SDValue Extract 2216 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 2217 2218 ReplaceUses(SDValue(N, 0), Extract); 2219 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 2220 CurDAG->RemoveDeadNode(N); 2221 } 2222 2223 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { 2224 // The address is assumed to be uniform, so if it ends up in a VGPR, it will 2225 // be copied to an SGPR with readfirstlane. 2226 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ? 2227 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME; 2228 2229 SDValue Chain = N->getOperand(0); 2230 SDValue Ptr = N->getOperand(2); 2231 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2232 MachineMemOperand *MMO = M->getMemOperand(); 2233 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; 2234 2235 SDValue Offset; 2236 if (CurDAG->isBaseWithConstantOffset(Ptr)) { 2237 SDValue PtrBase = Ptr.getOperand(0); 2238 SDValue PtrOffset = Ptr.getOperand(1); 2239 2240 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue(); 2241 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) { 2242 N = glueCopyToM0(N, PtrBase); 2243 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32); 2244 } 2245 } 2246 2247 if (!Offset) { 2248 N = glueCopyToM0(N, Ptr); 2249 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 2250 } 2251 2252 SDValue Ops[] = { 2253 Offset, 2254 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32), 2255 Chain, 2256 N->getOperand(N->getNumOperands() - 1) // New glue 2257 }; 2258 2259 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2260 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2261 } 2262 2263 static unsigned gwsIntrinToOpcode(unsigned IntrID) { 2264 switch (IntrID) { 2265 case Intrinsic::amdgcn_ds_gws_init: 2266 return AMDGPU::DS_GWS_INIT; 2267 case Intrinsic::amdgcn_ds_gws_barrier: 2268 return AMDGPU::DS_GWS_BARRIER; 2269 case Intrinsic::amdgcn_ds_gws_sema_v: 2270 return AMDGPU::DS_GWS_SEMA_V; 2271 case Intrinsic::amdgcn_ds_gws_sema_br: 2272 return AMDGPU::DS_GWS_SEMA_BR; 2273 case Intrinsic::amdgcn_ds_gws_sema_p: 2274 return AMDGPU::DS_GWS_SEMA_P; 2275 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2276 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL; 2277 default: 2278 llvm_unreachable("not a gws intrinsic"); 2279 } 2280 } 2281 2282 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) { 2283 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all && 2284 !Subtarget->hasGWSSemaReleaseAll()) { 2285 // Let this error. 2286 SelectCode(N); 2287 return; 2288 } 2289 2290 // Chain, intrinsic ID, vsrc, offset 2291 const bool HasVSrc = N->getNumOperands() == 4; 2292 assert(HasVSrc || N->getNumOperands() == 3); 2293 2294 SDLoc SL(N); 2295 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2); 2296 int ImmOffset = 0; 2297 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); 2298 MachineMemOperand *MMO = M->getMemOperand(); 2299 2300 // Don't worry if the offset ends up in a VGPR. Only one lane will have 2301 // effect, so SIFixSGPRCopies will validly insert readfirstlane. 2302 2303 // The resource id offset is computed as (<isa opaque base> + M0[21:16] + 2304 // offset field) % 64. Some versions of the programming guide omit the m0 2305 // part, or claim it's from offset 0. 2306 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) { 2307 // If we have a constant offset, try to use the 0 in m0 as the base. 2308 // TODO: Look into changing the default m0 initialization value. If the 2309 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to 2310 // the immediate offset. 2311 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32)); 2312 ImmOffset = ConstOffset->getZExtValue(); 2313 } else { 2314 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) { 2315 ImmOffset = BaseOffset.getConstantOperandVal(1); 2316 BaseOffset = BaseOffset.getOperand(0); 2317 } 2318 2319 // Prefer to do the shift in an SGPR since it should be possible to use m0 2320 // as the result directly. If it's already an SGPR, it will be eliminated 2321 // later. 2322 SDNode *SGPROffset 2323 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32, 2324 BaseOffset); 2325 // Shift to offset in m0 2326 SDNode *M0Base 2327 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32, 2328 SDValue(SGPROffset, 0), 2329 CurDAG->getTargetConstant(16, SL, MVT::i32)); 2330 glueCopyToM0(N, SDValue(M0Base, 0)); 2331 } 2332 2333 SDValue Chain = N->getOperand(0); 2334 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32); 2335 2336 // TODO: Can this just be removed from the instruction? 2337 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1); 2338 2339 const unsigned Opc = gwsIntrinToOpcode(IntrID); 2340 SmallVector<SDValue, 5> Ops; 2341 if (HasVSrc) 2342 Ops.push_back(N->getOperand(2)); 2343 Ops.push_back(OffsetField); 2344 Ops.push_back(GDS); 2345 Ops.push_back(Chain); 2346 2347 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 2348 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); 2349 } 2350 2351 void AMDGPUDAGToDAGISel::SelectInterpP1F16(SDNode *N) { 2352 if (Subtarget->getLDSBankCount() != 16) { 2353 // This is a single instruction with a pattern. 2354 SelectCode(N); 2355 return; 2356 } 2357 2358 SDLoc DL(N); 2359 2360 // This requires 2 instructions. It is possible to write a pattern to support 2361 // this, but the generated isel emitter doesn't correctly deal with multiple 2362 // output instructions using the same physical register input. The copy to m0 2363 // is incorrectly placed before the second instruction. 2364 // 2365 // TODO: Match source modifiers. 2366 // 2367 // def : Pat < 2368 // (int_amdgcn_interp_p1_f16 2369 // (VOP3Mods f32:$src0, i32:$src0_modifiers), 2370 // (i32 timm:$attrchan), (i32 timm:$attr), 2371 // (i1 timm:$high), M0), 2372 // (V_INTERP_P1LV_F16 $src0_modifiers, VGPR_32:$src0, timm:$attr, 2373 // timm:$attrchan, 0, 2374 // (V_INTERP_MOV_F32 2, timm:$attr, timm:$attrchan), timm:$high)> { 2375 // let Predicates = [has16BankLDS]; 2376 // } 2377 2378 // 16 bank LDS 2379 SDValue ToM0 = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, AMDGPU::M0, 2380 N->getOperand(5), SDValue()); 2381 2382 SDVTList VTs = CurDAG->getVTList(MVT::f32, MVT::Other); 2383 2384 SDNode *InterpMov = 2385 CurDAG->getMachineNode(AMDGPU::V_INTERP_MOV_F32, DL, VTs, { 2386 CurDAG->getTargetConstant(2, DL, MVT::i32), // P0 2387 N->getOperand(3), // Attr 2388 N->getOperand(2), // Attrchan 2389 ToM0.getValue(1) // In glue 2390 }); 2391 2392 SDNode *InterpP1LV = 2393 CurDAG->getMachineNode(AMDGPU::V_INTERP_P1LV_F16, DL, MVT::f32, { 2394 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src0_modifiers 2395 N->getOperand(1), // Src0 2396 N->getOperand(3), // Attr 2397 N->getOperand(2), // Attrchan 2398 CurDAG->getTargetConstant(0, DL, MVT::i32), // $src2_modifiers 2399 SDValue(InterpMov, 0), // Src2 - holds two f16 values selected by high 2400 N->getOperand(4), // high 2401 CurDAG->getTargetConstant(0, DL, MVT::i1), // $clamp 2402 CurDAG->getTargetConstant(0, DL, MVT::i32), // $omod 2403 SDValue(InterpMov, 1) 2404 }); 2405 2406 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(InterpP1LV, 0)); 2407 } 2408 2409 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { 2410 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2411 switch (IntrID) { 2412 case Intrinsic::amdgcn_ds_append: 2413 case Intrinsic::amdgcn_ds_consume: { 2414 if (N->getValueType(0) != MVT::i32) 2415 break; 2416 SelectDSAppendConsume(N, IntrID); 2417 return; 2418 } 2419 } 2420 2421 SelectCode(N); 2422 } 2423 2424 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) { 2425 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 2426 unsigned Opcode; 2427 switch (IntrID) { 2428 case Intrinsic::amdgcn_wqm: 2429 Opcode = AMDGPU::WQM; 2430 break; 2431 case Intrinsic::amdgcn_softwqm: 2432 Opcode = AMDGPU::SOFT_WQM; 2433 break; 2434 case Intrinsic::amdgcn_wwm: 2435 Opcode = AMDGPU::WWM; 2436 break; 2437 case Intrinsic::amdgcn_interp_p1_f16: 2438 SelectInterpP1F16(N); 2439 return; 2440 default: 2441 SelectCode(N); 2442 return; 2443 } 2444 2445 SDValue Src = N->getOperand(1); 2446 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src}); 2447 } 2448 2449 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) { 2450 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2451 switch (IntrID) { 2452 case Intrinsic::amdgcn_ds_gws_init: 2453 case Intrinsic::amdgcn_ds_gws_barrier: 2454 case Intrinsic::amdgcn_ds_gws_sema_v: 2455 case Intrinsic::amdgcn_ds_gws_sema_br: 2456 case Intrinsic::amdgcn_ds_gws_sema_p: 2457 case Intrinsic::amdgcn_ds_gws_sema_release_all: 2458 SelectDS_GWS(N, IntrID); 2459 return; 2460 default: 2461 break; 2462 } 2463 2464 SelectCode(N); 2465 } 2466 2467 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 2468 unsigned &Mods) const { 2469 Mods = 0; 2470 Src = In; 2471 2472 if (Src.getOpcode() == ISD::FNEG) { 2473 Mods |= SISrcMods::NEG; 2474 Src = Src.getOperand(0); 2475 } 2476 2477 if (Src.getOpcode() == ISD::FABS) { 2478 Mods |= SISrcMods::ABS; 2479 Src = Src.getOperand(0); 2480 } 2481 2482 return true; 2483 } 2484 2485 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 2486 SDValue &SrcMods) const { 2487 unsigned Mods; 2488 if (SelectVOP3ModsImpl(In, Src, Mods)) { 2489 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2490 return true; 2491 } 2492 2493 return false; 2494 } 2495 2496 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 2497 SDValue &SrcMods) const { 2498 SelectVOP3Mods(In, Src, SrcMods); 2499 return isNoNanSrc(Src); 2500 } 2501 2502 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 2503 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 2504 return false; 2505 2506 Src = In; 2507 return true; 2508 } 2509 2510 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 2511 SDValue &SrcMods, SDValue &Clamp, 2512 SDValue &Omod) const { 2513 SDLoc DL(In); 2514 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2515 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2516 2517 return SelectVOP3Mods(In, Src, SrcMods); 2518 } 2519 2520 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 2521 SDValue &Clamp, SDValue &Omod) const { 2522 Src = In; 2523 2524 SDLoc DL(In); 2525 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 2526 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 2527 2528 return true; 2529 } 2530 2531 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 2532 SDValue &SrcMods) const { 2533 unsigned Mods = 0; 2534 Src = In; 2535 2536 if (Src.getOpcode() == ISD::FNEG) { 2537 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 2538 Src = Src.getOperand(0); 2539 } 2540 2541 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 2542 unsigned VecMods = Mods; 2543 2544 SDValue Lo = stripBitcast(Src.getOperand(0)); 2545 SDValue Hi = stripBitcast(Src.getOperand(1)); 2546 2547 if (Lo.getOpcode() == ISD::FNEG) { 2548 Lo = stripBitcast(Lo.getOperand(0)); 2549 Mods ^= SISrcMods::NEG; 2550 } 2551 2552 if (Hi.getOpcode() == ISD::FNEG) { 2553 Hi = stripBitcast(Hi.getOperand(0)); 2554 Mods ^= SISrcMods::NEG_HI; 2555 } 2556 2557 if (isExtractHiElt(Lo, Lo)) 2558 Mods |= SISrcMods::OP_SEL_0; 2559 2560 if (isExtractHiElt(Hi, Hi)) 2561 Mods |= SISrcMods::OP_SEL_1; 2562 2563 Lo = stripExtractLoElt(Lo); 2564 Hi = stripExtractLoElt(Hi); 2565 2566 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 2567 // Really a scalar input. Just select from the low half of the register to 2568 // avoid packing. 2569 2570 Src = Lo; 2571 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2572 return true; 2573 } 2574 2575 Mods = VecMods; 2576 } 2577 2578 // Packed instructions do not have abs modifiers. 2579 Mods |= SISrcMods::OP_SEL_1; 2580 2581 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2582 return true; 2583 } 2584 2585 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2586 SDValue &SrcMods) const { 2587 Src = In; 2588 // FIXME: Handle op_sel 2589 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2590 return true; 2591 } 2592 2593 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2594 SDValue &SrcMods) const { 2595 // FIXME: Handle op_sel 2596 return SelectVOP3Mods(In, Src, SrcMods); 2597 } 2598 2599 // The return value is not whether the match is possible (which it always is), 2600 // but whether or not it a conversion is really used. 2601 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2602 unsigned &Mods) const { 2603 Mods = 0; 2604 SelectVOP3ModsImpl(In, Src, Mods); 2605 2606 if (Src.getOpcode() == ISD::FP_EXTEND) { 2607 Src = Src.getOperand(0); 2608 assert(Src.getValueType() == MVT::f16); 2609 Src = stripBitcast(Src); 2610 2611 // Be careful about folding modifiers if we already have an abs. fneg is 2612 // applied last, so we don't want to apply an earlier fneg. 2613 if ((Mods & SISrcMods::ABS) == 0) { 2614 unsigned ModsTmp; 2615 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2616 2617 if ((ModsTmp & SISrcMods::NEG) != 0) 2618 Mods ^= SISrcMods::NEG; 2619 2620 if ((ModsTmp & SISrcMods::ABS) != 0) 2621 Mods |= SISrcMods::ABS; 2622 } 2623 2624 // op_sel/op_sel_hi decide the source type and source. 2625 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2626 // If the sources's op_sel is set, it picks the high half of the source 2627 // register. 2628 2629 Mods |= SISrcMods::OP_SEL_1; 2630 if (isExtractHiElt(Src, Src)) { 2631 Mods |= SISrcMods::OP_SEL_0; 2632 2633 // TODO: Should we try to look for neg/abs here? 2634 } 2635 2636 return true; 2637 } 2638 2639 return false; 2640 } 2641 2642 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2643 SDValue &SrcMods) const { 2644 unsigned Mods = 0; 2645 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2646 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2647 return true; 2648 } 2649 2650 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const { 2651 if (In.isUndef()) 2652 return CurDAG->getUNDEF(MVT::i32); 2653 2654 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2655 SDLoc SL(In); 2656 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32); 2657 } 2658 2659 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 2660 SDLoc SL(In); 2661 return CurDAG->getConstant( 2662 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 2663 } 2664 2665 SDValue Src; 2666 if (isExtractHiElt(In, Src)) 2667 return Src; 2668 2669 return SDValue(); 2670 } 2671 2672 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const { 2673 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn); 2674 2675 const SIRegisterInfo *SIRI = 2676 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); 2677 const SIInstrInfo * SII = 2678 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 2679 2680 unsigned Limit = 0; 2681 bool AllUsesAcceptSReg = true; 2682 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end(); 2683 Limit < 10 && U != E; ++U, ++Limit) { 2684 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo()); 2685 2686 // If the register class is unknown, it could be an unknown 2687 // register class that needs to be an SGPR, e.g. an inline asm 2688 // constraint 2689 if (!RC || SIRI->isSGPRClass(RC)) 2690 return false; 2691 2692 if (RC != &AMDGPU::VS_32RegClass) { 2693 AllUsesAcceptSReg = false; 2694 SDNode * User = *U; 2695 if (User->isMachineOpcode()) { 2696 unsigned Opc = User->getMachineOpcode(); 2697 MCInstrDesc Desc = SII->get(Opc); 2698 if (Desc.isCommutable()) { 2699 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo(); 2700 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex; 2701 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) { 2702 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs(); 2703 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo); 2704 if (CommutedRC == &AMDGPU::VS_32RegClass) 2705 AllUsesAcceptSReg = true; 2706 } 2707 } 2708 } 2709 // If "AllUsesAcceptSReg == false" so far we haven't suceeded 2710 // commuting current user. This means have at least one use 2711 // that strictly require VGPR. Thus, we will not attempt to commute 2712 // other user instructions. 2713 if (!AllUsesAcceptSReg) 2714 break; 2715 } 2716 } 2717 return !AllUsesAcceptSReg && (Limit < 10); 2718 } 2719 2720 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const { 2721 auto Ld = cast<LoadSDNode>(N); 2722 2723 return Ld->getAlignment() >= 4 && 2724 ( 2725 ( 2726 ( 2727 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2728 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT 2729 ) 2730 && 2731 !N->isDivergent() 2732 ) 2733 || 2734 ( 2735 Subtarget->getScalarizeGlobalBehavior() && 2736 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && 2737 Ld->isSimple() && 2738 !N->isDivergent() && 2739 static_cast<const SITargetLowering *>( 2740 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N) 2741 ) 2742 ); 2743 } 2744 2745 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 2746 const AMDGPUTargetLowering& Lowering = 2747 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 2748 bool IsModified = false; 2749 do { 2750 IsModified = false; 2751 2752 // Go over all selected nodes and try to fold them a bit more 2753 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 2754 while (Position != CurDAG->allnodes_end()) { 2755 SDNode *Node = &*Position++; 2756 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 2757 if (!MachineNode) 2758 continue; 2759 2760 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 2761 if (ResNode != Node) { 2762 if (ResNode) 2763 ReplaceUses(Node, ResNode); 2764 IsModified = true; 2765 } 2766 } 2767 CurDAG->RemoveDeadNodes(); 2768 } while (IsModified); 2769 } 2770 2771 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 2772 Subtarget = &MF.getSubtarget<R600Subtarget>(); 2773 return SelectionDAGISel::runOnMachineFunction(MF); 2774 } 2775 2776 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 2777 if (!N->readMem()) 2778 return false; 2779 if (CbId == -1) 2780 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || 2781 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; 2782 2783 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId; 2784 } 2785 2786 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 2787 SDValue& IntPtr) { 2788 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 2789 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 2790 true); 2791 return true; 2792 } 2793 return false; 2794 } 2795 2796 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 2797 SDValue& BaseReg, SDValue &Offset) { 2798 if (!isa<ConstantSDNode>(Addr)) { 2799 BaseReg = Addr; 2800 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 2801 return true; 2802 } 2803 return false; 2804 } 2805 2806 void R600DAGToDAGISel::Select(SDNode *N) { 2807 unsigned int Opc = N->getOpcode(); 2808 if (N->isMachineOpcode()) { 2809 N->setNodeId(-1); 2810 return; // Already selected. 2811 } 2812 2813 switch (Opc) { 2814 default: break; 2815 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 2816 case ISD::SCALAR_TO_VECTOR: 2817 case ISD::BUILD_VECTOR: { 2818 EVT VT = N->getValueType(0); 2819 unsigned NumVectorElts = VT.getVectorNumElements(); 2820 unsigned RegClassID; 2821 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 2822 // that adds a 128 bits reg copy when going through TwoAddressInstructions 2823 // pass. We want to avoid 128 bits copies as much as possible because they 2824 // can't be bundled by our scheduler. 2825 switch(NumVectorElts) { 2826 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 2827 case 4: 2828 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 2829 RegClassID = R600::R600_Reg128VerticalRegClassID; 2830 else 2831 RegClassID = R600::R600_Reg128RegClassID; 2832 break; 2833 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 2834 } 2835 SelectBuildVector(N, RegClassID); 2836 return; 2837 } 2838 } 2839 2840 SelectCode(N); 2841 } 2842 2843 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 2844 SDValue &Offset) { 2845 ConstantSDNode *C; 2846 SDLoc DL(Addr); 2847 2848 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 2849 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2850 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2851 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 2852 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 2853 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2854 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2855 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 2856 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 2857 Base = Addr.getOperand(0); 2858 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2859 } else { 2860 Base = Addr; 2861 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2862 } 2863 2864 return true; 2865 } 2866 2867 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 2868 SDValue &Offset) { 2869 ConstantSDNode *IMMOffset; 2870 2871 if (Addr.getOpcode() == ISD::ADD 2872 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 2873 && isInt<16>(IMMOffset->getZExtValue())) { 2874 2875 Base = Addr.getOperand(0); 2876 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2877 MVT::i32); 2878 return true; 2879 // If the pointer address is constant, we can move it to the offset field. 2880 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 2881 && isInt<16>(IMMOffset->getZExtValue())) { 2882 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 2883 SDLoc(CurDAG->getEntryNode()), 2884 R600::ZERO, MVT::i32); 2885 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2886 MVT::i32); 2887 return true; 2888 } 2889 2890 // Default case, no offset 2891 Base = Addr; 2892 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 2893 return true; 2894 } 2895